rpms
/
koji
20
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update to git tarball

Browse Source
epel9
Dennis Gilmore 10 years ago
parent 8511f62424
commit 4ce429fa93
17 changed files with 5 additions and 1714 deletions
Show Stats Download Patch File Download Diff File

41
0001-correctly-call-pykickstarts-makeVersion.patch
Unescape View File

@ -1,41 +0,0 @@
From 049ab0dd94544bb0770defa54b1322812e6d3082 Mon Sep 17 00:00:00 2001
From: Dennis Gilmore <dennis@ausil.us>
Date: Mon, 4 Aug 2014 05:02:08 -0500
Subject: [PATCH] correctly call pykickstarts makeVersion
---
builder/kojid | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/builder/kojid b/builder/kojid
index b23e9ce..2451ea6 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -2387,9 +2387,9 @@ class ImageTask(BaseTaskHandler):
# may not be a problem if the included kickstarts are present
# in the repository we checked out.
if opts.get('ksversion'):
- version = ksparser.makeVersion(ksparser.stringToVersion(opts['ksversion']))
+ version = ksparser.version.makeVersion(ksparser.stringToVersion(opts['ksversion']))
else:
- version = ksparser.makeVersion()
+ version = ksparser.version.makeVersion()
self.ks = ksparser.KickstartParser(version)
try:
self.ks.readKickstart(kspath)
@@ -2768,10 +2768,10 @@ class OzImageTask(BaseTaskHandler):
# macros, Oz will fail because it can only handle flat files.
# We require users to flatten their kickstart file.
if self.opts.get('ksversion'):
- version = ksparser.makeVersion(
+ version = ksparser.version.makeVersion(
ksparser.stringToVersion(self.opts['ksversion']))
else:
- version = ksparser.makeVersion()
+ version = ksparser.version.makeVersion()
ks = ksparser.KickstartParser(version)
self.logger.debug('attempting to read kickstart: %s' % kspath)
try:
--
2.0.4

28
0001-fix-distro-behavior-for-rhel-5.patch
Unescape View File

@ -1,28 +0,0 @@
From 2e587154eb101ae4b2015a3b47bbb18e53d393dd Mon Sep 17 00:00:00 2001
From: Jay Greguske <jgregusk@redhat.com>
Date: Tue, 15 Apr 2014 14:31:10 -0400
Subject: [PATCH 1/8] fix distro behavior for rhel 5
---
builder/kojid | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/builder/kojid b/builder/kojid
index 4163f49..1691a54 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -2654,7 +2654,10 @@ class OzImageTask(BaseTaskHandler):
image on.
"""
if distro.startswith('RHEL'):
- return distro.split('.')
+ major, minor = distro.split('.')
+ if major == 'RHEL-5':
+ minor = 'U' + minor
+ return major, minor
elif distro.startswith('Fedora'):
return distro.split('-')
elif distro.startswith('CentOS'):
--
2.0.4

28
0001-move-workdir-from-tmp-koji-to-var-tmp-koji.patch
Unescape View File

@ -1,28 +0,0 @@
From 28b685087b620946c81ea72c04a018a174b1362f Mon Sep 17 00:00:00 2001
From: Dennis Gilmore <dennis@ausil.us>
Date: Thu, 29 May 2014 21:39:44 -0500
Subject: [PATCH 1/2] move workdir from /tmp/koji/ to /var/tmp/koji/
moving the workdir is needed as /tmp can be quite small with /tmp on tmpfs
Signed-off-by: Dennis Gilmore <dennis@ausil.us>
---
builder/kojid | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/builder/kojid b/builder/kojid
index f101e0f..14309bb 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -3657,7 +3657,7 @@ def get_options():
'log_level': None,
'topdir': '/mnt/koji',
'topurl': None,
- 'workdir': '/tmp/koji',
+ 'workdir': '/var/tmp/koji',
'pluginpath': '/usr/lib/koji-builder-plugins',
'mockdir': '/var/lib/mock',
'mockuser': 'kojibuilder',
--
2.0.0

694
0001-refactor-image-build-handlers-in-kojid.patch
Unescape View File

@ -1,694 +0,0 @@
From 440ec22696a5f65f43c042570abb8b39dec5d7ae Mon Sep 17 00:00:00 2001
From: Jay Greguske <jgregusk@redhat.com>
Date: Mon, 2 Jun 2014 15:54:42 -0400
Subject: [PATCH 1/3] refactor image-build handlers in kojid
---
builder/kojid | 477 ++++++++++++++++++++++++++++++++++++++++++----------------
1 file changed, 345 insertions(+), 132 deletions(-)
diff --git a/builder/kojid b/builder/kojid
index aece387..2ea0105 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -2711,7 +2711,7 @@ class OzImageTask(BaseTaskHandler):
def fetchKickstart(self):
"""
Retrieve the kickstart file we were given (locally or remotely) and
- upload it.
+ upload it to the hub.
Note that if the KS file existed locally, then "ksfile" is a relative
path to it in the /mnt/koji/work directory. If not, then it is still
@@ -2720,7 +2720,8 @@ class OzImageTask(BaseTaskHandler):
url with --ksurl.
@args: None, use self.opts for options
- @returns: absolute path to the retrieved kickstart file
+ @returns:
+ absolute path to the retrieved kickstart file
"""
ksfile = self.opts.get('kickstart')
self.logger.debug("ksfile = %s" % ksfile)
@@ -2751,14 +2752,8 @@ class OzImageTask(BaseTaskHandler):
@returns: None
"""
# XXX: If the ks file came from a local path and has %include
- # macros, *-creator will fail because the included
- # kickstarts were not copied into the chroot. For now we
- # require users to flatten their kickstart file if submitting
- # the task with a local path.
- #
- # Note that if an SCM URL was used instead, %include macros
- # may not be a problem if the included kickstarts are present
- # in the repository we checked out.
+ # macros, Oz will fail because it can only handle flat files.
+ # We require users to flatten their kickstart file.
if self.opts.get('ksversion'):
version = ksparser.makeVersion(
ksparser.stringToVersion(self.opts['ksversion']))
@@ -2775,16 +2770,15 @@ class OzImageTask(BaseTaskHandler):
raise koji.BuildError("Failed to parse kickstart file "
"'%s' : %s" % (kspath, e))
- def prepareKickstart(self, repo_info, target_info, arch):
+ def prepareKickstart(self, repo_info, target_info):
"""
Process the ks file to be used for controlled image generation. This
method also uploads the modified kickstart file to the task output
- area.
+ area on the hub..
@args:
target_info: a sesion.getBuildTarget() object
repo_info: session.getRepo() object
- arch: canonical architecture name
@returns:
absolute path to a processed kickstart file
"""
@@ -2794,12 +2788,13 @@ class OzImageTask(BaseTaskHandler):
# repo associated with the target passed in initially.
self.ks.handler.repo.repoList = [] # delete whatever the ks file told us
repo_class = kscontrol.dataMap[self.ks.version]['RepoData']
+ # TODO: sensibly use "url" and "repo" commands in kickstart
if self.opts.get('repo'):
# the user used --repo at least once
user_repos = self.opts.get('repo')
index = 0
for user_repo in user_repos:
- repo_url = user_repo.replace('$arch', arch)
+ repo_url = user_repo.replace('$arch', self.arch)
self.ks.handler.repo.repoList.append(repo_class(
baseurl=repo_url, name='koji-override-%i' % index))
index += 1
@@ -2808,7 +2803,7 @@ class OzImageTask(BaseTaskHandler):
path_info = koji.PathInfo(topdir=self.options.topurl)
repopath = path_info.repo(repo_info['id'],
target_info['build_tag_name'])
- baseurl = '%s/%s' % (repopath, arch)
+ baseurl = '%s/%s' % (repopath, self.arch)
self.logger.debug('BASEURL: %s' % baseurl)
self.ks.handler.repo.repoList.append(repo_class(
baseurl=baseurl, name='koji-override-0'))
@@ -2831,7 +2826,14 @@ class OzImageTask(BaseTaskHandler):
def makeConfig(self):
"""
Generate a configuration dict for ImageFactory. This will override
- anything in the /etc config files.
+ anything in the /etc config files. We do this forcibly so that it is
+ impossible for Koji to use any image caches or leftover metadata from
+ other images created by the service.
+
+ @args: none
+ @returns:
+ a dictionary used for configuring ImageFactory to built an image
+ the way we want
"""
return {
#Oz specific
@@ -2852,9 +2854,14 @@ class OzImageTask(BaseTaskHandler):
'storage_path': os.path.join(self.workdir, 'output_image')},
}
- def makeTemplate(self, imgname, arch, inst_tree):
+ def makeTemplate(self, inst_tree):
"""
- Generate a simple template for ImageFactory
+ Generate a simple "TDL" for ImageFactory to build an image with.
+
+ @args:
+ inst_tree - a string, a URL to the install tree (a compose)
+ @returns:
+ An XML string that imagefactory can consume
"""
# we have to split this up so the variable substitution works
distname, distver = self.parseDistro(self.opts.get('distro'))
@@ -2867,7 +2874,7 @@ class OzImageTask(BaseTaskHandler):
<install type='url'>
<url>%s</url>
</install>
- """ % (imgname, distname, distver, arch, inst_tree)
+ """ % (self.imgname, distname, distver, self.arch, inst_tree)
template += """<icicle>
<extra_command>rpm -qa --qf '%{NAME},%{VERSION},%{RELEASE},%{ARCH},%{EPOCH},%{SIZE},%{SIGMD5},%{BUILDTIME}\n'</extra_command>
</icicle>
@@ -2878,13 +2885,54 @@ class OzImageTask(BaseTaskHandler):
<size>%sG</size>
</disk>
</template>
-""" % (imgname, self.opts.get('disk_size'))
+""" % (self.imgname, self.opts.get('disk_size'))
+ return template
+
+ def makeDockerUtilTemplate(self, inst_tree, pkg_group):
+ """
+ Generate a "TDL" for ImageFactory to build a utility image to run
+ docker commands on a docker image.
+
+ @args:
+ inst_tree - a string, a URL to the install tree (a compose)
+ @returns:
+ An XML string that imagefactory can consume
+ """
+ distname, distver = self.parseDistro(self.opts.get('distro'))
+ template = """<template>
+ <name>koji-%s-utility</name>
+ <os>
+ <name>%s</name>
+ <version>%s</version>
+ <arch>%s</arch>
+ <install type='url'>
+ <url>%s</url>
+ </install>
+ """ % (self.id, distname, distver, self.arch, inst_tree)
+ template += """<icicle>
+ <extra_command>rpm -qa --qf '%{NAME},%{VERSION},%{RELEASE},%{ARCH},%{EPOCH},%{SIZE},%{SIGMD5},%{BUILDTIME}\n'</extra_command>
+ </icicle>
+ """
+ # TODO: this should be defined by a docker-build package group
+ template += """</os>
+ <description>koji-%s-utility</description>
+ <packages>
+ <package name='docker'/>
+ <package name='libguestfs-tools'/>
+ </packages>
+</template>
+""" % self.id
return template
def parseDistro(self, distro):
"""
Figure out the distribution name and version we are going to build an
image on.
+
+ args:
+ a string of the form: RHEL-X.Y, Fedora-NN, CentOS-X.Y, or SL-X.Y
+ returns:
+ a 2-element list, depends on the distro where the split happened
"""
if distro.startswith('RHEL'):
major, minor = distro.split('.')
@@ -2900,7 +2948,8 @@ class OzImageTask(BaseTaskHandler):
else:
raise BuildError('Unknown or supported distro given: %s' % distro)
- def fixImageXML(self, format, imgname, filename, xmltext):
+ def fixImageXML(self, format, filename, xmltext):
+
"""
The XML generated by Oz/ImageFactory knows nothing about the name
or image format conversions Koji does. We fix those values in the
@@ -2909,18 +2958,18 @@ class OzImageTask(BaseTaskHandler):
@args:
format = raw, qcow2, vmdk, etc... a string representation
- name = the (file) name of the image
+ filename = the name of the XML file we will save this too
xmltext = the libvirt XML to start with
@return:
an absolute path to the modified XML
"""
newxml = xml.dom.minidom.parseString(xmltext)
ename = newxml.getElementsByTagName('name')[0]
- ename.firstChild.nodeValue = imgname
+ ename.firstChild.nodeValue = self.imgname
esources = newxml.getElementsByTagName('source')
for e in esources:
if e.hasAttribute('file'):
- e.setAttribute('file', '%s.%s' % (imgname, format))
+ e.setAttribute('file', '%s.%s' % (self.imgname, format))
edriver = newxml.getElementsByTagName('driver')[0]
edriver.setAttribute('type', format)
xml_path = os.path.join(self.workdir, filename)
@@ -2933,6 +2982,9 @@ class OzImageTask(BaseTaskHandler):
"""
Locate a screenshot taken by libvirt in the case of build failure,
if it exists. If it does, return the path, else return None.
+
+ @args: none
+ @returns: a path to a screenshot take by libvirt
"""
shotdir = os.path.join(self.workdir, 'oz_screenshots')
screenshot = None
@@ -2947,7 +2999,7 @@ class OzImageTask(BaseTaskHandler):
class BaseImageTask(OzImageTask):
Methods = ['createImage']
- _taskWeight = 1.5
+ _taskWeight = 2.0
def getRootDevice(self):
"""
@@ -2959,7 +3011,38 @@ class BaseImageTask(OzImageTask):
return part.disk
raise koji.ApplianceError, 'kickstart lacks a "/" mountpoint'
- def format_deps(self, formats):
+ def _makeDockerCmds(self, tags):
+ """
+ When building a docker image, we call docker commands on it from
+ within the "utility" image. ImageFactory accepts an XML string that
+ describes what commands to run, which is what this method returns.
+
+ @args: None
+ tags- a list of tags to apply to the docker image
+ @returns:
+ an XML string with docker commands
+ """
+ # TODO: set up the file name correctly
+ # TODO: set up the image id in docker correctly
+ cmds = """<template>
+ <commands>
+ <command name='mount'>mount /dev/vdb1 /mnt</command>
+ <command name='dockerstart'>/usr/bin/systemctl start docker</command>
+ <command name='taroutimport'>LIBGUESTFS_BACKEND=direct virt-tar-out -a /mnt/input_image.raw / - | docker import - </command>"""
+ idx = 0
+ for tag in tags:
+ cmds += """
+ <command name='docker-tag%s'>docker tag %s ...</command>""" % (idx, tag)
+ idx += 1
+ cmds += """
+ <command name='save'>docker save %s > /mnt/docker-output.img</command>
+ </commands>
+</template>
+""" % self.imgname
+ self.logger.debug('docker command template: %s' % cmds)
+ return cmds
+
+ def _format_deps(self, formats):
"""
Return a dictionary where the keys are the image formats we need to
build/convert, and the values are booleans that indicate whether the
@@ -2988,136 +3071,266 @@ class BaseImageTask(OzImageTask):
self.logger.debug('Image delivery plan: %s' % f_dict)
return f_dict
- def do_images(self, arch, template, ozlog, imgname):
+ def do_images(self, template, inst_tree):
"""
Call out to ImageFactory to build the image(s) we want. Returns a dict
of details for each image type we had to ask ImageFactory to build
- {format: dispatcher object}
"""
- def do_target_image(base_id, image_type, ova_opts={}):
- self.logger.debug('ova_opts: %s' % ova_opts)
- try:
- target = bd.builder_for_target_image(image_type,
- image_id=base_id, template=None, parameters=ova_opts)
- target.target_thread.join()
- except:
- tlog.removeHandler(fhandler)
- self.uploadFile(ozlog)
- self.logger.debug(
- 'Target image results: %s' % target.target_image.status)
- if target.target_image.status == 'FAILED':
- if not self.session.checkUpload('', os.path.basename(ozlog)):
- tlog.removeHandler(fhandler)
- self.uploadFile(ozlog)
- raise koji.ApplianceError('Image status is %s: %s' % (
- target.target_image.status,
- target.target_image.status_detail))
- return target
-
- fhandler = logging.FileHandler(ozlog)
- bd = BuildDispatcher()
- tlog = logging.getLogger()
- tlog.setLevel(logging.DEBUG)
- tlog.addHandler(fhandler)
+ fcalls = {'raw': self._buildBase,
+ 'vmdk': self._buildConvert,
+ 'vdi': self._buildConvert,
+ 'qcow': self._buildConvert,
+ 'qcow2': self._buildConvert,
+ 'rhevm-ova': self._buildOVA,
+ 'vsphere-ova': self._buildOVA,
+ 'docker': self._buildIndirect
+ }
+ # add a handler to the logger so that we capture ImageFactory's logging
+ self.fhandler = logging.FileHandler(self.ozlog)
+ self.bd = BuildDispatcher()
+ self.tlog = logging.getLogger()
+ self.tlog.setLevel(logging.DEBUG)
+ self.tlog.addHandler(self.fhandler)
images = {}
+ random.seed() # necessary to ensure a unique mac address
+ # if we need a utility image for the indirection plugin we create it
+ # we do not join() this until later
+ # Future: 'livecd' will be in here
+ if 'docker' in self.formats:
+ self.session.host.setTaskWeight(self.id, 3.0)
+ self.util_img = self._buildDockerUtility(inst_tree)
params = {'install_script': str(self.ks.handler),
'offline_icicle': True}
- random.seed() # necessary to ensure a unique mac address
- try:
- base = bd.builder_for_base_image(template, parameters=params)
- base.base_thread.join()
- except:
- # upload log even if we failed to help diagnose an issue
- tlog.removeHandler(fhandler)
- self.uploadFile(ozlog)
- self.logger.debug('Base image results: %s' % base.base_image.status)
- if base.base_image.status == 'FAILED':
+ # build the base (raw) image
+ self.base_img = self._buildBase(template, params)
+ images['raw'] = {'image': self.base_img.base_image.data,
+ 'icicle': self.base_img.base_image.icicle}
+ # Do the rest of the image types (everything but raw)
+ for format in self.formats:
+ if format == 'raw':
+ continue
+ self.logger.info('dispatching %s image builder' % format)
+ images[format] = fcalls[format](format)
+ imginfo = self._processXML(images)
+ self.tlog.removeHandler(self.fhandler)
+ self.uploadFile(self.ozlog)
+ return imginfo
+
+ def _processXML(self, images):
+ """
+ Produce XML that libvirt can import to create a domain based on image(s)
+ we produced. We save the location of the XML file in the dictionary
+ it corresponds to here.
+
+ @args:
+ images - a dict where the keys are image formats, and the values
+ are dicts with details about the image (location, icicle, etc)
+ @returns:
+ a dictionary just like "images" but with a new key called "libvirt"
+ that points to the path of the XML file for that image
+ """
+ imginfo = {}
+ for fmt in images.keys():
+ imginfo[fmt] = images[fmt]
+ lxml = self.fixImageXML(fmt, 'libvirt-%s-%s.xml' % (fmt, self.arch),
+ self.base_img.base_image.parameters['libvirt_xml'])
+ imginfo[fmt]['libvirt'] = lxml
+ return imginfo
+
+ def _checkImageState(self, image):
+ """
+ Query ImageFactory for details of a dispatched image build. If it is
+ FAILED we raise an exception.
+
+ @args:
+ image - a build dispatcher object returned by a BuildDispatcher
+ @returns: nothing
+ """
+ if image.target_image:
+ status = image.target_image.status
+ details = image.target_image.status_detail
+ else:
+ status = image.base_image.status
+ details = image.base_image.status_detail
+ self.logger.debug('check image results: %s' % status)
+ if status == 'FAILED':
scrnshot = self.getScreenshot()
if scrnshot:
ext = scrnshot[-3:]
self.uploadFile(scrnshot, remoteName='screenshot.%s' % ext)
- base.os_plugin.abort() # forcibly tear down the VM
+ image.os_plugin.abort() # forcibly tear down the VM
# TODO abort when a task is CANCELLED
- if not self.session.checkUpload('', os.path.basename(ozlog)):
- tlog.removeHandler(fhandler)
- self.uploadFile(ozlog)
-
+ if not self.session.checkUpload('', os.path.basename(self.ozlog)):
+ self.tlog.removeHandler(self.fhandler)
+ self.uploadFile(self.ozlog)
raise koji.ApplianceError('Image status is %s: %s' %
- (base.base_image.status, base.base_image.status_detail))
- lxml = self.fixImageXML('raw', imgname,
- 'libvirt-%s-%s.xml' % ('raw', arch),
- base.base_image.parameters['libvirt_xml'])
- images['raw'] = {'image': base.base_image.data, 'libvirt': lxml,
- 'icicle': base.base_image.icicle}
-
- # target-image type images
- if 'docker' in self.formats:
- targ = do_target_image(base.base_image.identifier, 'docker',
- ova_opts={'compress': 'gzip'})
- images['docker'] = {'image': targ.target_image.data}
+ (status, details))
+
+ def _buildDockerUtility(self, inst_tree):
+ """
+ Build a utility image used for the indirection plugin later. Docker
+ and eventually liveCDs will use this. The utility image provides an
+ environment where we will run post-build commands on the base image
+ we generated.
- ova_opts = {}
+ @args:
+ inst_tree - a string URL to an install tree (a compose)
+ @returns:
+ a dict with some details about the image
+ """
+ #dtemp = self.makeDockerUtilTemplate(inst_tree, 'docker-build')
+ dtemp = self.makeDockerUtilTemplate('http://download.lab.bos.redhat.com/rel-eng/RHEL-7.0-20140507.0/compose/Server/x86_64/os', 'docker-build')
+ # TODO: enable this and store it properly
+ # pkgs = [x['packagelist'] for x in brew.getTagGroups('rhel-7.0-build') if x['name'] == 'livecd-build'][0]
+ # print '\n'.join([p['package'] for p in pkgs])
+ dparams = {'generate_icicle': False}
+ utilname = 'koji-%s-util' % self.id
+ return self._buildBase(dtemp, dparams, wait=False)
+
+ def _buildBase(self, template, params, wait=True):
+ """
+ Build a base image using ImageFactory. This is a "raw" image.
+
+ @args:
+ template - an XML string for the TDL
+ params - a dict that controls some ImageFactory settings
+ wait - call join() on the building thread if True
+ @returns:
+ a dict with some metadata about the image (includes an icicle)
+ """
+ # TODO: test the failure case where IF itself throws an exception
+ # ungracefully (missing a plugin for example)
+ # may need to still upload ozlog and remove the log handler
+ self.logger.info('dispatching a baseimg builder')
+ self.logger.debug('templates: %s' % template)
+ self.logger.debug('params: %s' % params)
+ base = self.bd.builder_for_base_image(template, parameters=params)
+ if wait:
+ base.base_thread.join()
+ self._checkImageState(base)
+ return base
+
+ def _buildOVA(self, format):
+ """
+ Build an OVA target image. This is a format supported by RHEV and
+ vSphere
+
+ @args:
+ format - a string representing the image format, "rhevm-ova"
+ @returns
+ a dict with some metadata about the image
+ """
+ img_opts = {}
if self.opts.get('ova_option'):
- ova_opts = dict([o.split('=') for o in self.opts.get('ova_option')])
- for format in ('rhevm-ova', 'vsphere-ova'):
- # assumes self.formats is pre-populated with rhevm/vsphere if the
- # "ova" format for them was requested with --format only
- if format not in self.formats:
- continue
- targ = do_target_image(base.base_image.identifier,
- format.replace('-ova', ''))
- # Target images do not have their own modified libvirt xml files.
- # They may not even be bootable with libvirt.
- lxml = self.fixImageXML(format, imgname,
- 'libvirt-%s-%s.xml' % (format, arch),
- base.base_image.parameters['libvirt_xml'])
- targ2 = do_target_image(targ.target_image.identifier, 'OVA',
- ova_opts=ova_opts)
- images[format] = {'libvirt': lxml, 'image': targ2.target_image.data}
- tlog.removeHandler(fhandler)
- self.uploadFile(ozlog)
-
- # qemu-img conversions
- for format in ('qcow', 'qcow2', 'vdi', 'vmdk'):
- if format not in self.formats:
- continue
- newimg = os.path.join(self.workdir, imgname + '.%s' % format)
- cmd = ['/usr/bin/qemu-img', 'convert', '-f', 'raw', '-O',
- format, base.base_image.data, newimg]
- if format in ('qcow', 'qcow2'):
- cmd.insert(2, '-c') # enable compression for qcow images
- conlog = os.path.join(self.workdir,
- 'qemu-img-%s-%s.log' % (format, arch))
- log_output(self.session, cmd[0], cmd, conlog,
- self.getUploadDir(), logerror=1)
- lxml = self.fixImageXML(format, imgname,
- 'libvirt-%s-%s.xml' % (format, arch),
- base.base_image.parameters['libvirt_xml'])
- images[format] = {'image': newimg, 'libvirt': lxml}
-
- return images
+ img_opts = dict([o.split('=') for o in self.opts.get('ova_option')])
+ targ = self._do_target_image(self.base_img.base_image.identifier,
+ format.replace('-ova', ''))
+ targ2 = self._do_target_image(targ.target_image.identifier, 'OVA',
+ img_opts=img_opts)
+ return {'image': targ2.target_image.data}
+
+ def _do_target_image(self, base_id, image_type, img_opts={}):
+ """
+ A generic method for building what ImageFactory calls "target images".
+ These are images based on a raw disk that was built before using the
+ _buildBase method.
+
+ @args:
+ base_id - a string ID of the image to build off of
+ image_type - a string representing the target type. ImageFactory
+ uses this to figure out what plugin to run
+ img_opts - a dict of additional options that specific to the target
+ type we pass in via image_type
+ @returns:
+ A Builder() object from ImageFactory that contains information
+ about the image building include state and progress.
+ """
+ # TODO: test the failure case where IF itself throws an exception
+ # ungracefully (missing a plugin for example)
+ # may need to still upload ozlog and remove the log handler
+ self.logger.debug('img_opts: %s' % img_opts)
+ target = self.bd.builder_for_target_image(image_type,
+ image_id=base_id, template=None, parameters=img_opts)
+ target.target_thread.join()
+ self._checkImageState(target)
+ return target
+
+ def _buildConvert(self, format):
+ """
+ Build an image by converting the format using qemu-img. This is method
+ enables a variety of formats like qcow, qcow2, vmdk, and vdi.
+
+ @args:
+ format - a string representing the image format, "qcow2"
+ @returns
+ a dict with some metadata about the image
+ """
+
+ newimg = os.path.join(self.workdir, self.imgname + '.%s' % format)
+ cmd = ['/usr/bin/qemu-img', 'convert', '-f', 'raw', '-O',
+ format, self.base_img.base_image.data, newimg]
+ if format in ('qcow', 'qcow2'):
+ cmd.insert(2, '-c') # enable compression for qcow images
+ conlog = os.path.join(self.workdir,
+ 'qemu-img-%s-%s.log' % (format, self.arch))
+ log_output(self.session, cmd[0], cmd, conlog,
+ self.getUploadDir(), logerror=1)
+ return {'image': newimg}
+
+ def _buildIndirect(self, format):
+ """
+ "Indirect" images are target images that use the indirection plugin.
+ This plugin makes use of an additional guest, and launches that to run
+ commands on the base image we built earlier. The image that backs the
+ modifying guest is called the "utility" image, which we built before.
+
+ @args:
+ format - a string representing the image format, "qcow2"
+ @returns
+ a dict with some metadata about the image
+ """
+ # Future: livecd will be introduced here too
+ # we should have waited for and checked the base image already
+ # TODO: log the shit out of this new stuff
+ # TODO: test the failure case where IF itself throws an exception
+ # ungracefully (missing a plugin for example)
+ # may need to still upload ozlog and remove the log handler
+ self.util_img.base_thread.join()
+ self._checkImageState(self.util_img)
+ params = {
+ 'compress': 'gzip',
+ 'utility_image': self.util_img.base_image.identifier,
+ 'utility_customizations': self._makeDockerCmds([self.imgname])
+ }
+ targ = self.bd.builder_for_target_image('indirection',
+ image_id=self.base.base_image.identifier,
+ template=None, parameters=params)
+ targ.target_thread.join()
+ self._checkImageState(targ)
+ return {'image': targ.target_image.data}
def handler(self, name, version, release, arch, target_info, build_tag, repo_info, inst_tree, opts=None):
if opts == None:
opts = {}
+ self.arch = arch
self.opts = opts
- self.formats = self.format_deps(opts.get('format'))
+ self.formats = self._format_deps(opts.get('format'))
# First, prepare the kickstart to use the repos we tell it
kspath = self.fetchKickstart()
self.readKickstart(kspath)
- kskoji = self.prepareKickstart(repo_info, target_info, arch)
+ kskoji = self.prepareKickstart(repo_info, target_info)
# auto-generate a TDL file and config dict for ImageFactory
- imgname = '%s-%s-%s.%s' % (name, version, release, arch)
- template = self.makeTemplate(imgname, arch, inst_tree)
+ self.imgname = '%s-%s-%s.%s' % (name, version, release, self.arch)
+ template = self.makeTemplate(inst_tree)
self.logger.debug('oz template: %s' % template)
config = self.makeConfig()
self.logger.debug('IF config object: %s' % config)
ApplicationConfiguration(configuration=config)
- tdl_path = os.path.join(self.workdir, 'tdl-%s.xml' % arch)
+ tdl_path = os.path.join(self.workdir, 'tdl-%s.xml' % self.arch)
tdl = open(tdl_path, 'w')
tdl.write(template)
tdl.close()
@@ -3129,16 +3342,16 @@ class BaseImageTask(OzImageTask):
# the likelihood of image tasks clashing here is very small)
rm = ReservationManager()
rm._listen_port = rm.MIN_PORT + self.id % (rm.MAX_PORT - rm.MIN_PORT)
- ozlogname = 'oz-%s.log' % arch
- ozlog = os.path.join(self.workdir, ozlogname)
+ ozlogname = 'oz-%s.log' % self.arch
+ self.ozlog = os.path.join(self.workdir, ozlogname)
# invoke the image builds
- images = self.do_images(arch, template, ozlog, imgname)
+ images = self.do_images(template, inst_tree)
images['raw']['tdl'] = os.path.basename(tdl_path),
# structure the results to pass back to the hub:
imgdata = {
- 'arch': arch,
+ 'arch': self.arch,
'task_id': self.id,
'logs': [ozlogname],
'name': name,
@@ -3175,7 +3388,7 @@ class BaseImageTask(OzImageTask):
rpm['epoch'] = int(bits[4])
imgdata['rpmlist'].append(rpm)
# TODO: hack to make this work for now, need to refactor
- br = BuildRoot(self.session, self.options, build_tag, arch,
+ br = BuildRoot(self.session, self.options, build_tag, self.arch,
self.id, repo_id=repo_info['id'])
br.markExternalRPMs(imgdata['rpmlist'])
@@ -3183,11 +3396,11 @@ class BaseImageTask(OzImageTask):
for format in (f for f in self.formats.keys() if self.formats[f]):
newimg = images[format]['image']
if 'ova' in format:
- newname = imgname + '.' + format.replace('-', '.')
+ newname = self.imgname + '.' + format.replace('-', '.')
elif format == 'docker':
- newname = imgname + '.' + 'tar.gz'
+ newname = self.imgname + '.' + 'tar.gz'
else:
- newname = imgname + '.' + format
+ newname = self.imgname + '.' + format
if format != 'docker':
lxml = images[format]['libvirt']
imgdata['files'].append(os.path.basename(lxml))
--
2.0.4

35
0001-use-TLSv1.-https-bugzilla.redhat.com-show_bug.cgi-id.patch
Unescape View File

@ -1,35 +0,0 @@
From 07d1eb810930fa1de7c89bad817ccca68b9ec7bc Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Wed, 15 Oct 2014 10:54:31 -0400
Subject: [PATCH] use TLSv1.
https://bugzilla.redhat.com/show_bug.cgi?id=1152823
---
koji/ssl/SSLCommon.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/koji/ssl/SSLCommon.py b/koji/ssl/SSLCommon.py
index 1a3b3d6..014fbf6 100644
--- a/koji/ssl/SSLCommon.py
+++ b/koji/ssl/SSLCommon.py
@@ -37,7 +37,7 @@ def CreateSSLContext(certs):
if f and not os.access(f, os.R_OK):
raise StandardError, "%s does not exist or is not readable" % f
- ctx = SSL.Context(SSL.SSLv3_METHOD) # SSLv3 only
+ ctx = SSL.Context(SSL.TLSv1_METHOD) # TLS v1 only
ctx.use_certificate_file(key_and_cert)
ctx.use_privatekey_file(key_and_cert)
ctx.load_client_ca(ca_cert)
@@ -45,7 +45,7 @@ def CreateSSLContext(certs):
verify = SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT
ctx.set_verify(verify, our_verify)
ctx.set_verify_depth(10)
- ctx.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_TLSv1)
+ ctx.set_options(SSL.OP_NO_SSLv3 | SSL.OP_NO_SSLv2 | SSL.OP_NO_TLSv1)
return ctx
--
2.2.0

26
0002-compress-docker-tarball-properly.patch
Unescape View File

@ -1,26 +0,0 @@
From 073bdb69e73e8943f3fb031f56b3d3ca66412edc Mon Sep 17 00:00:00 2001
From: Jay Greguske <jgregusk@redhat.com>
Date: Tue, 15 Apr 2014 14:45:40 -0400
Subject: [PATCH 2/8] compress docker tarball properly
---
builder/kojid | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/builder/kojid b/builder/kojid
index 1691a54..cee7637 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -2824,7 +2824,8 @@ class BaseImageTask(OzImageTask):
# target-image type images
if 'docker' in self.formats:
- targ = do_target_image(base.base_image.identifier, 'docker')
+ targ = do_target_image(base.base_image.identifier, 'docker',
+ ova_opts={'compress': 'gzip'})
images['docker'] = {'image': targ.target_image.data}
ova_opts = {}
--
2.0.4

439
0002-refactor-do_images.patch
Unescape View File

@ -1,439 +0,0 @@
From ef2e41a672fecf6168dca5a177d61c8e77427279 Mon Sep 17 00:00:00 2001
From: Jay Greguske <jgregusk@redhat.com>
Date: Fri, 25 Jul 2014 13:34:10 -0400
Subject: [PATCH 2/3] refactor do_images
---
builder/kojid | 248 +++++++++++++++++-----------------------------------------
1 file changed, 73 insertions(+), 175 deletions(-)
diff --git a/builder/kojid b/builder/kojid
index 2ea0105..d3446b9 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -2749,7 +2749,8 @@ class OzImageTask(BaseTaskHandler):
@args:
kspath: path to a kickstart file
- @returns: None
+ @returns:
+ a kickstart object returned by pykickstart
"""
# XXX: If the ks file came from a local path and has %include
# macros, Oz will fail because it can only handle flat files.
@@ -2759,35 +2760,36 @@ class OzImageTask(BaseTaskHandler):
ksparser.stringToVersion(self.opts['ksversion']))
else:
version = ksparser.makeVersion()
- self.ks = ksparser.KickstartParser(version)
+ ks = ksparser.KickstartParser(version)
self.logger.debug('attempting to read kickstart: %s' % kspath)
try:
- self.ks.readKickstart(kspath)
+ ks.readKickstart(kspath)
except IOError, e:
raise koji.BuildError("Failed to read kickstart file "
"'%s' : %s" % (kspath, e))
except kserrors.KickstartError, e:
raise koji.BuildError("Failed to parse kickstart file "
"'%s' : %s" % (kspath, e))
+ return ks
- def prepareKickstart(self, repo_info, target_info):
+ def prepareKickstart(self, kspath):
"""
Process the ks file to be used for controlled image generation. This
method also uploads the modified kickstart file to the task output
- area on the hub..
+ area on the hub.
@args:
- target_info: a sesion.getBuildTarget() object
- repo_info: session.getRepo() object
+ kspath: a path to a kickstart file
@returns:
- absolute path to a processed kickstart file
+ a kickstart object with koji-specific modifications
"""
+ ks = self.readKickstart(kspath)
# Now we do some kickstart manipulation. If the user passed in a repo
# url with --repo, then we substitute that in for the repo(s) specified
# in the kickstart file. If --repo wasn't specified, then we use the
# repo associated with the target passed in initially.
- self.ks.handler.repo.repoList = [] # delete whatever the ks file told us
- repo_class = kscontrol.dataMap[self.ks.version]['RepoData']
+ ks.handler.repo.repoList = [] # delete whatever the ks file told us
+ repo_class = kscontrol.dataMap[ks.version]['RepoData']
# TODO: sensibly use "url" and "repo" commands in kickstart
if self.opts.get('repo'):
# the user used --repo at least once
@@ -2795,33 +2797,41 @@ class OzImageTask(BaseTaskHandler):
index = 0
for user_repo in user_repos:
repo_url = user_repo.replace('$arch', self.arch)
- self.ks.handler.repo.repoList.append(repo_class(
+ ks.handler.repo.repoList.append(repo_class(
baseurl=repo_url, name='koji-override-%i' % index))
index += 1
else:
# --repo was not given, so we use the target's build repo
path_info = koji.PathInfo(topdir=self.options.topurl)
- repopath = path_info.repo(repo_info['id'],
- target_info['build_tag_name'])
+ repopath = path_info.repo(self.repo_info['id'],
+ self.target_info['build_tag_name'])
baseurl = '%s/%s' % (repopath, self.arch)
self.logger.debug('BASEURL: %s' % baseurl)
- self.ks.handler.repo.repoList.append(repo_class(
+ ks.handler.repo.repoList.append(repo_class(
baseurl=baseurl, name='koji-override-0'))
+ return ks
- # Write out the new ks file. Note that things may not be in the same
- # order and comments in the original ks file may be lost.
- kskoji = os.path.join(self.workdir, 'koji-image-%s-%i.ks' %
- (target_info['build_tag_name'], self.id))
- self.logger.debug('modified ks file: %s' % kskoji)
- outfile = open(kskoji, 'w')
- outfile.write(str(self.ks.handler))
+ def writeKickstart(self, ksobj, ksname):
+ """
+ Write out the new ks file. Note that things may not be in the same
+ order and comments in the original ks file may be lost.
+
+ @args:
+ ksobj: a pykickstart object of what we want to write
+ ksname: file name for the kickstart
+ @returns:
+ an absolute path to the kickstart file we wrote
+ """
+ kspath = os.path.join(self.workdir, ksname)
+ outfile = open(kspath, 'w')
+ outfile.write(str(ksobj.handler))
outfile.close()
# put the new ksfile in the output directory
- if not os.path.exists(kskoji):
- raise koji.BuildError, "KS file missing: %s" % kskoji
- self.uploadFile(kskoji) # upload the modified ks file
- return kskoji
+ if not os.path.exists(kspath):
+ raise koji.BuildError, "KS file missing: %s" % kspath
+ self.uploadFile(kspath) # upload the modified ks file
+ return kspath
def makeConfig(self):
"""
@@ -2854,16 +2864,20 @@ class OzImageTask(BaseTaskHandler):
'storage_path': os.path.join(self.workdir, 'output_image')},
}
- def makeTemplate(self, inst_tree):
+ def makeTemplate(self, name, inst_tree):
"""
Generate a simple "TDL" for ImageFactory to build an image with.
@args:
- inst_tree - a string, a URL to the install tree (a compose)
+ name: a name for the image
+ inst_tree: a string, a URL to the install tree (a compose)
@returns:
An XML string that imagefactory can consume
"""
# we have to split this up so the variable substitution works
+ # XXX: using a packages section (which we don't) will have IF boot the
+ # image and attempt to ssh in. This breaks docker image creation.
+ # TODO: intelligently guess the distro based on the install tree URL
distname, distver = self.parseDistro(self.opts.get('distro'))
template = """<template>
<name>%s</name>
@@ -2874,54 +2888,19 @@ class OzImageTask(BaseTaskHandler):
<install type='url'>
<url>%s</url>
</install>
- """ % (self.imgname, distname, distver, self.arch, inst_tree)
+ """ % (name, distname, distver, self.arch, inst_tree)
template += """<icicle>
<extra_command>rpm -qa --qf '%{NAME},%{VERSION},%{RELEASE},%{ARCH},%{EPOCH},%{SIZE},%{SIGMD5},%{BUILDTIME}\n'</extra_command>
</icicle>
"""
+ # TODO: intelligently guess the size based on the kickstart file
template += """</os>
<description>%s OS</description>
<disk>
<size>%sG</size>
</disk>
</template>
-""" % (self.imgname, self.opts.get('disk_size'))
- return template
-
- def makeDockerUtilTemplate(self, inst_tree, pkg_group):
- """
- Generate a "TDL" for ImageFactory to build a utility image to run
- docker commands on a docker image.
-
- @args:
- inst_tree - a string, a URL to the install tree (a compose)
- @returns:
- An XML string that imagefactory can consume
- """
- distname, distver = self.parseDistro(self.opts.get('distro'))
- template = """<template>
- <name>koji-%s-utility</name>
- <os>
- <name>%s</name>
- <version>%s</version>
- <arch>%s</arch>
- <install type='url'>
- <url>%s</url>
- </install>
- """ % (self.id, distname, distver, self.arch, inst_tree)
- template += """<icicle>
- <extra_command>rpm -qa --qf '%{NAME},%{VERSION},%{RELEASE},%{ARCH},%{EPOCH},%{SIZE},%{SIGMD5},%{BUILDTIME}\n'</extra_command>
- </icicle>
- """
- # TODO: this should be defined by a docker-build package group
- template += """</os>
- <description>koji-%s-utility</description>
- <packages>
- <package name='docker'/>
- <package name='libguestfs-tools'/>
- </packages>
-</template>
-""" % self.id
+""" % (name, self.opts.get('disk_size'))
return template
def parseDistro(self, distro):
@@ -3001,47 +2980,6 @@ class BaseImageTask(OzImageTask):
Methods = ['createImage']
_taskWeight = 2.0
- def getRootDevice(self):
- """
- Return the device name for the / partition, as specified in the
- kickstart file. Appliances should have this defined.
- """
- for part in self.ks.handler.partition.partitions:
- if part.mountpoint == '/':
- return part.disk
- raise koji.ApplianceError, 'kickstart lacks a "/" mountpoint'
-
- def _makeDockerCmds(self, tags):
- """
- When building a docker image, we call docker commands on it from
- within the "utility" image. ImageFactory accepts an XML string that
- describes what commands to run, which is what this method returns.
-
- @args: None
- tags- a list of tags to apply to the docker image
- @returns:
- an XML string with docker commands
- """
- # TODO: set up the file name correctly
- # TODO: set up the image id in docker correctly
- cmds = """<template>
- <commands>
- <command name='mount'>mount /dev/vdb1 /mnt</command>
- <command name='dockerstart'>/usr/bin/systemctl start docker</command>
- <command name='taroutimport'>LIBGUESTFS_BACKEND=direct virt-tar-out -a /mnt/input_image.raw / - | docker import - </command>"""
- idx = 0
- for tag in tags:
- cmds += """
- <command name='docker-tag%s'>docker tag %s ...</command>""" % (idx, tag)
- idx += 1
- cmds += """
- <command name='save'>docker save %s > /mnt/docker-output.img</command>
- </commands>
-</template>
-""" % self.imgname
- self.logger.debug('docker command template: %s' % cmds)
- return cmds
-
def _format_deps(self, formats):
"""
Return a dictionary where the keys are the image formats we need to
@@ -3049,8 +2987,7 @@ class BaseImageTask(OzImageTask):
output should be included in the task results.
Some image formats require others to be processed first, which is why
- we have to do this. rhevm-ova requires rhevm, but if the user did not
- request it, we should not pass it back up.
+ we have to do this. raw files in particular may not be kept.
"""
supported = ('raw', 'vmdk', 'qcow', 'qcow2', 'vdi', 'rhevm-ova', 'vsphere-ova', 'docker')
for f in formats:
@@ -3071,7 +3008,7 @@ class BaseImageTask(OzImageTask):
self.logger.debug('Image delivery plan: %s' % f_dict)
return f_dict
- def do_images(self, template, inst_tree):
+ def do_images(self, ks, template, inst_tree):
"""
Call out to ImageFactory to build the image(s) we want. Returns a dict
of details for each image type we had to ask ImageFactory to build
@@ -3083,7 +3020,7 @@ class BaseImageTask(OzImageTask):
'qcow2': self._buildConvert,
'rhevm-ova': self._buildOVA,
'vsphere-ova': self._buildOVA,
- 'docker': self._buildIndirect
+ 'docker': self._buildDocker
}
# add a handler to the logger so that we capture ImageFactory's logging
self.fhandler = logging.FileHandler(self.ozlog)
@@ -3093,13 +3030,7 @@ class BaseImageTask(OzImageTask):
self.tlog.addHandler(self.fhandler)
images = {}
random.seed() # necessary to ensure a unique mac address
- # if we need a utility image for the indirection plugin we create it
- # we do not join() this until later
- # Future: 'livecd' will be in here
- if 'docker' in self.formats:
- self.session.host.setTaskWeight(self.id, 3.0)
- self.util_img = self._buildDockerUtility(inst_tree)
- params = {'install_script': str(self.ks.handler),
+ params = {'install_script': str(ks.handler),
'offline_icicle': True}
# build the base (raw) image
self.base_img = self._buildBase(template, params)
@@ -3166,26 +3097,6 @@ class BaseImageTask(OzImageTask):
raise koji.ApplianceError('Image status is %s: %s' %
(status, details))
- def _buildDockerUtility(self, inst_tree):
- """
- Build a utility image used for the indirection plugin later. Docker
- and eventually liveCDs will use this. The utility image provides an
- environment where we will run post-build commands on the base image
- we generated.
-
- @args:
- inst_tree - a string URL to an install tree (a compose)
- @returns:
- a dict with some details about the image
- """
- #dtemp = self.makeDockerUtilTemplate(inst_tree, 'docker-build')
- dtemp = self.makeDockerUtilTemplate('http://download.lab.bos.redhat.com/rel-eng/RHEL-7.0-20140507.0/compose/Server/x86_64/os', 'docker-build')
- # TODO: enable this and store it properly
- # pkgs = [x['packagelist'] for x in brew.getTagGroups('rhel-7.0-build') if x['name'] == 'livecd-build'][0]
- # print '\n'.join([p['package'] for p in pkgs])
- dparams = {'generate_icicle': False}
- utilname = 'koji-%s-util' % self.id
- return self._buildBase(dtemp, dparams, wait=False)
def _buildBase(self, template, params, wait=True):
"""
@@ -3217,7 +3128,7 @@ class BaseImageTask(OzImageTask):
@args:
format - a string representing the image format, "rhevm-ova"
- @returns
+ @returns:
a dict with some metadata about the image
"""
img_opts = {}
@@ -3229,6 +3140,21 @@ class BaseImageTask(OzImageTask):
img_opts=img_opts)
return {'image': targ2.target_image.data}
+ def _buildDocker(self, format):
+ """
+ Build a base docker image. This image will be tagged with the NVR.A
+ automatically because we name it that way in the ImageFactory TDL.
+
+ @args:
+ format - the string "docker"
+ @returns:
+ a dict with some metadata about the image
+ """
+ img_opts = {'compress': 'gzip'}
+ targ = self._do_target_image(self.base_img.base_image.identifier,
+ 'docker', img_opts=img_opts)
+ return {'image': targ.target_image.data}
+
def _do_target_image(self, base_id, image_type, img_opts={}):
"""
A generic method for building what ImageFactory calls "target images".
@@ -3277,54 +3203,26 @@ class BaseImageTask(OzImageTask):
self.getUploadDir(), logerror=1)
return {'image': newimg}
- def _buildIndirect(self, format):
- """
- "Indirect" images are target images that use the indirection plugin.
- This plugin makes use of an additional guest, and launches that to run
- commands on the base image we built earlier. The image that backs the
- modifying guest is called the "utility" image, which we built before.
-
- @args:
- format - a string representing the image format, "qcow2"
- @returns
- a dict with some metadata about the image
- """
- # Future: livecd will be introduced here too
- # we should have waited for and checked the base image already
- # TODO: log the shit out of this new stuff
- # TODO: test the failure case where IF itself throws an exception
- # ungracefully (missing a plugin for example)
- # may need to still upload ozlog and remove the log handler
- self.util_img.base_thread.join()
- self._checkImageState(self.util_img)
- params = {
- 'compress': 'gzip',
- 'utility_image': self.util_img.base_image.identifier,
- 'utility_customizations': self._makeDockerCmds([self.imgname])
- }
- targ = self.bd.builder_for_target_image('indirection',
- image_id=self.base.base_image.identifier,
- template=None, parameters=params)
- targ.target_thread.join()
- self._checkImageState(targ)
- return {'image': targ.target_image.data}
-
def handler(self, name, version, release, arch, target_info, build_tag, repo_info, inst_tree, opts=None):
if opts == None:
opts = {}
self.arch = arch
+ self.target_info = target_info
+ self.repo_info = repo_info
self.opts = opts
self.formats = self._format_deps(opts.get('format'))
# First, prepare the kickstart to use the repos we tell it
kspath = self.fetchKickstart()
- self.readKickstart(kspath)
- kskoji = self.prepareKickstart(repo_info, target_info)
+ ks = self.prepareKickstart(kspath)
+ kskoji = self.writeKickstart(ks,
+ os.path.join(self.workdir, 'koji-%s-%i-base.ks' %
+ (self.target_info['build_tag_name'], self.id)))
# auto-generate a TDL file and config dict for ImageFactory
self.imgname = '%s-%s-%s.%s' % (name, version, release, self.arch)
- template = self.makeTemplate(inst_tree)
+ template = self.makeTemplate(self.imgname, inst_tree)
self.logger.debug('oz template: %s' % template)
config = self.makeConfig()
self.logger.debug('IF config object: %s' % config)
@@ -3346,7 +3244,7 @@ class BaseImageTask(OzImageTask):
self.ozlog = os.path.join(self.workdir, ozlogname)
# invoke the image builds
- images = self.do_images(template, inst_tree)
+ images = self.do_images(ks, template, inst_tree)
images['raw']['tdl'] = os.path.basename(tdl_path),
# structure the results to pass back to the hub:
@@ -3389,7 +3287,7 @@ class BaseImageTask(OzImageTask):
imgdata['rpmlist'].append(rpm)
# TODO: hack to make this work for now, need to refactor
br = BuildRoot(self.session, self.options, build_tag, self.arch,
- self.id, repo_id=repo_info['id'])
+ self.id, repo_id=self.repo_info['id'])
br.markExternalRPMs(imgdata['rpmlist'])
# upload the results
--
2.0.4

148
0003-add-raw-xz-option.patch
Unescape View File

@ -1,148 +0,0 @@
From c1b42e0c67b1050c00c922f882cc192dbc571edc Mon Sep 17 00:00:00 2001
From: Jay Greguske <jgregusk@redhat.com>
Date: Tue, 29 Jul 2014 21:33:49 -0400
Subject: [PATCH 3/3] add raw-xz option
---
builder/kojid | 50 +++++++++++++++++++++++++++++++++++++++++++-------
cli/koji | 2 +-
2 files changed, 44 insertions(+), 8 deletions(-)
diff --git a/builder/kojid b/builder/kojid
index d3446b9..b23e9ce 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -2096,6 +2096,18 @@ class BuildBaseImageTask(BuildImageTask):
spec_url, subtask, target_info, bld_info,
repo_info['id'])
+ # make sure we only import the user-submitted kickstart file one
+ # time, otherwise we will have collisions. Remove it from exactly
+ # 1 results hash from the subtasks
+ if opts.has_key('kickstart'):
+ saw_ks = False
+ for arch in results.keys():
+ ks = os.path.basename(opts.get('kickstart'))
+ if ks in results[arch]['files']:
+ if saw_ks:
+ results[arch]['files'].remove(ks)
+ saw_ks = True
+
self.logger.debug('Image Results for hub: %s' % results)
if opts.get('scratch'):
self.session.host.moveImageBuildToScratch(self.id, results)
@@ -2728,7 +2740,7 @@ class OzImageTask(BaseTaskHandler):
if self.opts.get('ksurl'):
scm = SCM(self.opts['ksurl'])
scm.assert_allowed(self.options.allowed_scms)
- logfile = os.path.join(self.workdir, 'checkout.log')
+ logfile = os.path.join(self.workdir, 'checkout-%s.log' % self.arch)
scmsrcdir = scm.checkout(self.workdir, self.session,
self.getUploadDir(), logfile)
kspath = os.path.join(scmsrcdir, os.path.basename(ksfile))
@@ -2989,7 +3001,7 @@ class BaseImageTask(OzImageTask):
Some image formats require others to be processed first, which is why
we have to do this. raw files in particular may not be kept.
"""
- supported = ('raw', 'vmdk', 'qcow', 'qcow2', 'vdi', 'rhevm-ova', 'vsphere-ova', 'docker')
+ supported = ('raw', 'raw-xz', 'vmdk', 'qcow', 'qcow2', 'vdi', 'rhevm-ova', 'vsphere-ova', 'docker')
for f in formats:
if f not in supported:
raise koji.ApplianceError('Invalid format: %s' % f)
@@ -3014,6 +3026,7 @@ class BaseImageTask(OzImageTask):
of details for each image type we had to ask ImageFactory to build
"""
fcalls = {'raw': self._buildBase,
+ 'raw-xz': self._buildXZ,
'vmdk': self._buildConvert,
'vdi': self._buildConvert,
'qcow': self._buildConvert,
@@ -3079,10 +3092,10 @@ class BaseImageTask(OzImageTask):
"""
if image.target_image:
status = image.target_image.status
- details = image.target_image.status_detail
+ details = image.target_image.status_detail['error']
else:
status = image.base_image.status
- details = image.base_image.status_detail
+ details = image.base_image.status_detail['error']
self.logger.debug('check image results: %s' % status)
if status == 'FAILED':
scrnshot = self.getScreenshot()
@@ -3094,10 +3107,11 @@ class BaseImageTask(OzImageTask):
if not self.session.checkUpload('', os.path.basename(self.ozlog)):
self.tlog.removeHandler(self.fhandler)
self.uploadFile(self.ozlog)
+ if 'No disk activity' in details:
+ details = 'Automated install failed or prompted for input. See the screenshot in the task results for more information.'
raise koji.ApplianceError('Image status is %s: %s' %
(status, details))
-
def _buildBase(self, template, params, wait=True):
"""
Build a base image using ImageFactory. This is a "raw" image.
@@ -3121,6 +3135,29 @@ class BaseImageTask(OzImageTask):
self._checkImageState(base)
return base
+ def _buildXZ(self, format):
+ """
+ Use xz to compress a raw disk image. This is very straightforward.
+
+ @args:
+ format - a string representing the image format, "raw-xz"
+ @returns:
+ a dict with some metadata about the image
+ """
+ newimg = os.path.join(self.workdir, self.imgname + '.raw.xz')
+ rawimg = os.path.join(self.workdir, self.imgname + '.raw')
+ cmd = ['/bin/cp', self.base_img.base_image.data, rawimg]
+ conlog = os.path.join(self.workdir,
+ 'xz-cp-%s-%s.log' % (format, self.arch))
+ log_output(self.session, cmd[0], cmd, conlog, self.getUploadDir(),
+ logerror=1)
+ cmd = ['/usr/bin/xz', '-z', rawimg]
+ conlog = os.path.join(self.workdir,
+ 'xz-%s-%s.log' % (format, self.arch))
+ log_output(self.session, cmd[0], cmd, conlog, self.getUploadDir(),
+ logerror=1)
+ return {'image': newimg}
+
def _buildOVA(self, format):
"""
Build an OVA target image. This is a format supported by RHEV and
@@ -3191,7 +3228,6 @@ class BaseImageTask(OzImageTask):
@returns
a dict with some metadata about the image
"""
-
newimg = os.path.join(self.workdir, self.imgname + '.%s' % format)
cmd = ['/usr/bin/qemu-img', 'convert', '-f', 'raw', '-O',
format, self.base_img.base_image.data, newimg]
@@ -3293,7 +3329,7 @@ class BaseImageTask(OzImageTask):
# upload the results
for format in (f for f in self.formats.keys() if self.formats[f]):
newimg = images[format]['image']
- if 'ova' in format:
+ if 'ova' in format or format == 'raw-xz':
newname = self.imgname + '.' + format.replace('-', '.')
elif format == 'docker':
newname = self.imgname + '.' + 'tar.gz'
diff --git a/cli/koji b/cli/koji
index 8722f01..9451963 100755
--- a/cli/koji
+++ b/cli/koji
@@ -5130,7 +5130,7 @@ def handle_spin_appliance(options, session, args):
def handle_image_build(options, session, args):
"""Create a disk image given an install tree"""
formats = ('vmdk', 'qcow', 'qcow2', 'vdi', 'rhevm-ova', 'vsphere-ova',
- 'docker')
+ 'docker', 'raw-xz')
usage = _("usage: %prog image-build [options] <name> <version> " +
"<target> <install-tree-url> <arch> [<arch>...]")
usage += _("\n %prog image-build --config FILE")
--
2.0.4

54
0003-upload-the-tdl-priority-to-building.patch
Unescape View File

@ -1,54 +0,0 @@
From b566eba6f83b151fcc058906a0630f4d9e303490 Mon Sep 17 00:00:00 2001
From: Jay Greguske <jgregusk@redhat.com>
Date: Wed, 16 Apr 2014 11:18:53 -0400
Subject: [PATCH 3/8] upload the tdl priority to building
---
builder/kojid | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/builder/kojid b/builder/kojid
index cee7637..32400c6 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -2813,13 +2813,7 @@ class BaseImageTask(OzImageTask):
lxml = self.fixImageXML('raw', imgname,
'libvirt-%s-%s.xml' % ('raw', arch),
base.base_image.parameters['libvirt_xml'])
- tdl_path = os.path.join(self.workdir, 'tdl-%s.xml' % arch)
- tdl = open(tdl_path, 'w')
- tdl.write(base.base_image.template)
- tdl.close()
- self.uploadFile(tdl_path)
images['raw'] = {'image': base.base_image.data, 'libvirt': lxml,
- 'tdl': os.path.basename(tdl_path),
'icicle': base.base_image.icicle}
# target-image type images
@@ -2889,6 +2883,12 @@ class BaseImageTask(OzImageTask):
self.logger.debug('IF config object: %s' % config)
ApplicationConfiguration(configuration=config)
+ tdl_path = os.path.join(self.workdir, 'tdl-%s.xml' % arch)
+ tdl = open(tdl_path, 'w')
+ tdl.write(template)
+ tdl.close()
+ self.uploadFile(tdl_path)
+
# ImageFactory picks a port to the guest VM using a rolling integer.
# This is a problem for concurrency, so we override the port it picks
# here using the task ID. (not a perfect solution but good enough:
@@ -2900,9 +2900,9 @@ class BaseImageTask(OzImageTask):
# invoke the image builds
images = self.do_images(arch, template, ozlog, imgname)
+ images['raw']['tdl'] = os.path.basename(tdl_path),
# structure the results to pass back to the hub:
- tdl_path = images['raw']['tdl']
imgdata = {
'arch': arch,
'task_id': self.id,
--
2.0.4

33
0004-define-bld_info-before-the-try-in-case-of-an-earlier.patch
Unescape View File

@ -1,33 +0,0 @@
From 7a1d6c4cd42f6efe8a3833a77a4475b6cfe9da43 Mon Sep 17 00:00:00 2001
From: Jay Greguske <jgregusk@redhat.com>
Date: Wed, 16 Apr 2014 12:00:33 -0400
Subject: [PATCH 4/8] define bld_info before the try in case of an earlier
exception
---
builder/kojid | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/builder/kojid b/builder/kojid
index 32400c6..fa5daca 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -1824,6 +1824,7 @@ class BuildBaseImageTask(BuildImageTask):
raise koji.ApplianceError, 'ImageFactory functions not available'
# build image(s)
+ bld_info = None
try:
release = opts.get('release')
if not release:
@@ -1832,7 +1833,6 @@ class BuildBaseImageTask(BuildImageTask):
raise koji.ApplianceError('The Version may not have a hyphen')
if '-' in release:
raise koji.ApplianceError('The Release may not have a hyphen')
- bld_info = None
if not opts.get('scratch'):
bld_info = self.initImageBuild(name, version, release,
target_info, opts)
--
2.0.4

39
0005-put-ova-options-in-its-own-section.patch
Unescape View File

@ -1,39 +0,0 @@
From 20bb3e15265499d129cb85b6cd41a8724e037082 Mon Sep 17 00:00:00 2001
From: Jay Greguske <jgregusk@redhat.com>
Date: Wed, 16 Apr 2014 14:00:17 -0400
Subject: [PATCH 5/8] put ova options in its own section
---
cli/koji | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
diff --git a/cli/koji b/cli/koji
index 088c505..504b4ba 100755
--- a/cli/koji
+++ b/cli/koji
@@ -5074,14 +5074,17 @@ def handle_image_build(options, session, args):
if config.has_option(section, arg):
setattr(task_options, arg, config.get(section, arg).split(','))
config.remove_option(section, arg)
- # ova_option is newline separated, handle that
- if config.has_option(section, 'ova_option'):
- task_options.ova_option = \
- config.get(section, 'ova_option').splitlines()
- config.remove_option(section, 'ova_option')
# handle everything else
for k, v in config.items(section):
setattr(task_options, k, v)
+
+ # ova-options belong in their own section
+ section = 'ova-options'
+ if config.has_section(section):
+ task_options.ova_option = []
+ for k, v in config.items(section):
+ task_options.ova_option.append('%s=%s' % (k, v))
+
else:
if len(args) < 5:
parser.error(_("At least five arguments are required: a name, " +
--
2.0.4

26
0006-use-offline-icicle-generation.patch
Unescape View File

@ -1,26 +0,0 @@
From 61c1737652544150fe962921b9dfd3708e1d256e Mon Sep 17 00:00:00 2001
From: Jay Greguske <jgregusk@redhat.com>
Date: Thu, 17 Apr 2014 18:59:41 -0400
Subject: [PATCH 6/8] use offline icicle generation
---
builder/kojid | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/builder/kojid b/builder/kojid
index fa5daca..4712cc5 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -2787,7 +2787,8 @@ class BaseImageTask(OzImageTask):
tlog.setLevel(logging.DEBUG)
tlog.addHandler(fhandler)
images = {}
- params = {'install_script': str(self.ks.handler)} #ks contents
+ params = {'install_script': str(self.ks.handler),
+ 'offline_icicle': True}
random.seed() # necessary to ensure a unique mac address
try:
base = bd.builder_for_base_image(template, parameters=params)
--
2.0.4

44
0007-include-multilib-packages-in-repos.patch
Unescape View File

@ -1,44 +0,0 @@
From 794abb71b1cd5662354475e2beafbab0f287b409 Mon Sep 17 00:00:00 2001
From: Jay Greguske <jgregusk@redhat.com>
Date: Thu, 17 Apr 2014 19:00:08 -0400
Subject: [PATCH 7/8] include multilib packages in repos
---
builder/mergerepos | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/builder/mergerepos b/builder/mergerepos
index b873ccd..fecbc4b 100755
--- a/builder/mergerepos
+++ b/builder/mergerepos
@@ -48,6 +48,12 @@ EXPAND_ARCHES = {
'sh4': ['sh4a']
}
+MULTILIB_ARCHES = {
+ 'x86_64': 'i386',
+ 'ppc64': 'ppc',
+ 's390x': 's390'
+ }
+
def parse_args(args):
"""Parse our opts/args"""
usage = """
@@ -78,6 +84,14 @@ def parse_args(args):
if EXPAND_ARCHES.has_key(arch):
opts.arches.extend(EXPAND_ARCHES[arch])
+ # support multilib repos
+ for arch in opts.arches[:]:
+ multilib_arch = MULTILIB_ARCHES.get(arch)
+ if multilib_arch:
+ opts.arches.append(multilib_arch)
+ if multilib_arch in EXPAND_ARCHES:
+ opts.arches.extend(EXPAND_ARCHES[multilib_arch])
+
# always include noarch
if not 'noarch' in opts.arches:
opts.arches.append('noarch')
--
2.0.4

25
0008-bump-install-timeout-to-2-hours.patch
Unescape View File

@ -1,25 +0,0 @@
From 1fbbf825ab7962e68d0059ab86b0955312997035 Mon Sep 17 00:00:00 2001
From: Jay Greguske <jgregusk@redhat.com>
Date: Thu, 8 May 2014 11:34:51 -0400
Subject: [PATCH 8/8] bump install timeout to 2 hours
---
builder/kojid | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/builder/kojid b/builder/kojid
index 4712cc5..90ff2bd 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -2608,7 +2608,7 @@ class OzImageTask(BaseTaskHandler):
'imgdir': os.path.join(self.workdir, 'scratch_images'),
'tmpdir': os.path.join(self.workdir, 'oz-tmp'),
'verbose' : True,
- 'timeout': 3600,
+ 'timeout': 7200,
'output': 'log',
'raw': False,
'debug': True,
--
2.0.4

24
compress-docker.patch
Unescape View File

@ -1,24 +0,0 @@
From 073bdb69e73e8943f3fb031f56b3d3ca66412edc Mon Sep 17 00:00:00 2001
From: Jay Greguske <jgregusk@redhat.com>
Date: Tue, 15 Apr 2014 14:45:40 -0400
Subject: compress docker tarball properly
diff --git a/builder/kojid b/builder/kojid
index 1691a54..cee7637 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -2824,7 +2824,8 @@ class BaseImageTask(OzImageTask):
# target-image type images
if 'docker' in self.formats:
- targ = do_target_image(base.base_image.identifier, 'docker')
+ targ = do_target_image(base.base_image.identifier, 'docker',
+ ova_opts={'compress': 'gzip'})
images['docker'] = {'image': targ.target_image.data}
ova_opts = {}
--
cgit v0.10.2

33
koji.spec
Unescape View File

@ -2,27 +2,13 @@
Name: koji
Version: 1.9.0
Release: 9%{?dist}
Release: 10%{?dist}.gitcd45e886
License: LGPLv2 and GPLv2+
# koji.ssl libs (from plague) are GPLv2+
Summary: Build system tools
Group: Applications/System
URL: https://fedorahosted.org/koji
Patch0: fedora-config.patch
Patch1: 0001-move-workdir-from-tmp-koji-to-var-tmp-koji.patch
Patch2: 0001-fix-distro-behavior-for-rhel-5.patch
Patch3: 0002-compress-docker-tarball-properly.patch
Patch4: 0003-upload-the-tdl-priority-to-building.patch
Patch5: 0004-define-bld_info-before-the-try-in-case-of-an-earlier.patch
Patch6: 0005-put-ova-options-in-its-own-section.patch
Patch7: 0006-use-offline-icicle-generation.patch
Patch8: 0007-include-multilib-packages-in-repos.patch
Patch9: 0008-bump-install-timeout-to-2-hours.patch
Patch10: 0001-refactor-image-build-handlers-in-kojid.patch
Patch11: 0002-refactor-do_images.patch
Patch12: 0003-add-raw-xz-option.patch
Patch13: 0001-correctly-call-pykickstarts-makeVersion.patch
Patch14: 0001-use-TLSv1.-https-bugzilla.redhat.com-show_bug.cgi-id.patch
Source: https://fedorahosted.org/released/koji/koji-%{version}.tar.bz2
BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
@ -136,20 +122,6 @@ koji-web is a web UI to the Koji system.
%prep
%setup -q
%patch0 -p1 -b .orig
%patch1 -p1
%patch2 -p1
%patch3 -p1
%patch4 -p1
%patch5 -p1
%patch6 -p1
%patch7 -p1
%patch8 -p1
%patch9 -p1
%patch10 -p1
%patch11 -p1
%patch12 -p1
%patch13 -p1
%patch14 -p1
%build
@ -253,6 +225,9 @@ if [ $1 = 0 ]; then
fi
%changelog
* Tue Jan 27 2015 Dennis Gilmore <dennis@ausil.us> - 1.9.0-10.gitcd45e886
- update to git tarball
* Thu Dec 11 2014 Dennis Gilmore <dennis@ausil.us> - 1.9.0-9
- add upstream patch switching to TLS1 from sslv3

2
sources
Unescape View File

@ -1 +1 @@
0ce900022f67324858551622f9f75c73 koji-1.9.0.tar.bz2
b6596198479fbf4923ea428b013cb1d1 koji-1.9.0.tar.bz2

Write Preview
Loading…
Cancel
Save