diff --git a/0001-refactor-image-build-handlers-in-kojid.patch b/0001-refactor-image-build-handlers-in-kojid.patch new file mode 100644 index 0000000..22d4f0e --- /dev/null +++ b/0001-refactor-image-build-handlers-in-kojid.patch @@ -0,0 +1,694 @@ +From 440ec22696a5f65f43c042570abb8b39dec5d7ae Mon Sep 17 00:00:00 2001 +From: Jay Greguske +Date: Mon, 2 Jun 2014 15:54:42 -0400 +Subject: [PATCH 1/3] refactor image-build handlers in kojid + +--- + builder/kojid | 477 ++++++++++++++++++++++++++++++++++++++++++---------------- + 1 file changed, 345 insertions(+), 132 deletions(-) + +diff --git a/builder/kojid b/builder/kojid +index aece387..2ea0105 100755 +--- a/builder/kojid ++++ b/builder/kojid +@@ -2711,7 +2711,7 @@ class OzImageTask(BaseTaskHandler): + def fetchKickstart(self): + """ + Retrieve the kickstart file we were given (locally or remotely) and +- upload it. ++ upload it to the hub. + + Note that if the KS file existed locally, then "ksfile" is a relative + path to it in the /mnt/koji/work directory. If not, then it is still +@@ -2720,7 +2720,8 @@ class OzImageTask(BaseTaskHandler): + url with --ksurl. + + @args: None, use self.opts for options +- @returns: absolute path to the retrieved kickstart file ++ @returns: ++ absolute path to the retrieved kickstart file + """ + ksfile = self.opts.get('kickstart') + self.logger.debug("ksfile = %s" % ksfile) +@@ -2751,14 +2752,8 @@ class OzImageTask(BaseTaskHandler): + @returns: None + """ + # XXX: If the ks file came from a local path and has %include +- # macros, *-creator will fail because the included +- # kickstarts were not copied into the chroot. For now we +- # require users to flatten their kickstart file if submitting +- # the task with a local path. +- # +- # Note that if an SCM URL was used instead, %include macros +- # may not be a problem if the included kickstarts are present +- # in the repository we checked out. ++ # macros, Oz will fail because it can only handle flat files. ++ # We require users to flatten their kickstart file. + if self.opts.get('ksversion'): + version = ksparser.makeVersion( + ksparser.stringToVersion(self.opts['ksversion'])) +@@ -2775,16 +2770,15 @@ class OzImageTask(BaseTaskHandler): + raise koji.BuildError("Failed to parse kickstart file " + "'%s' : %s" % (kspath, e)) + +- def prepareKickstart(self, repo_info, target_info, arch): ++ def prepareKickstart(self, repo_info, target_info): + """ + Process the ks file to be used for controlled image generation. This + method also uploads the modified kickstart file to the task output +- area. ++ area on the hub.. + + @args: + target_info: a sesion.getBuildTarget() object + repo_info: session.getRepo() object +- arch: canonical architecture name + @returns: + absolute path to a processed kickstart file + """ +@@ -2794,12 +2788,13 @@ class OzImageTask(BaseTaskHandler): + # repo associated with the target passed in initially. + self.ks.handler.repo.repoList = [] # delete whatever the ks file told us + repo_class = kscontrol.dataMap[self.ks.version]['RepoData'] ++ # TODO: sensibly use "url" and "repo" commands in kickstart + if self.opts.get('repo'): + # the user used --repo at least once + user_repos = self.opts.get('repo') + index = 0 + for user_repo in user_repos: +- repo_url = user_repo.replace('$arch', arch) ++ repo_url = user_repo.replace('$arch', self.arch) + self.ks.handler.repo.repoList.append(repo_class( + baseurl=repo_url, name='koji-override-%i' % index)) + index += 1 +@@ -2808,7 +2803,7 @@ class OzImageTask(BaseTaskHandler): + path_info = koji.PathInfo(topdir=self.options.topurl) + repopath = path_info.repo(repo_info['id'], + target_info['build_tag_name']) +- baseurl = '%s/%s' % (repopath, arch) ++ baseurl = '%s/%s' % (repopath, self.arch) + self.logger.debug('BASEURL: %s' % baseurl) + self.ks.handler.repo.repoList.append(repo_class( + baseurl=baseurl, name='koji-override-0')) +@@ -2831,7 +2826,14 @@ class OzImageTask(BaseTaskHandler): + def makeConfig(self): + """ + Generate a configuration dict for ImageFactory. This will override +- anything in the /etc config files. ++ anything in the /etc config files. We do this forcibly so that it is ++ impossible for Koji to use any image caches or leftover metadata from ++ other images created by the service. ++ ++ @args: none ++ @returns: ++ a dictionary used for configuring ImageFactory to built an image ++ the way we want + """ + return { + #Oz specific +@@ -2852,9 +2854,14 @@ class OzImageTask(BaseTaskHandler): + 'storage_path': os.path.join(self.workdir, 'output_image')}, + } + +- def makeTemplate(self, imgname, arch, inst_tree): ++ def makeTemplate(self, inst_tree): + """ +- Generate a simple template for ImageFactory ++ Generate a simple "TDL" for ImageFactory to build an image with. ++ ++ @args: ++ inst_tree - a string, a URL to the install tree (a compose) ++ @returns: ++ An XML string that imagefactory can consume + """ + # we have to split this up so the variable substitution works + distname, distver = self.parseDistro(self.opts.get('distro')) +@@ -2867,7 +2874,7 @@ class OzImageTask(BaseTaskHandler): + + %s + +- """ % (imgname, distname, distver, arch, inst_tree) ++ """ % (self.imgname, distname, distver, self.arch, inst_tree) + template += """ + rpm -qa --qf '%{NAME},%{VERSION},%{RELEASE},%{ARCH},%{EPOCH},%{SIZE},%{SIGMD5},%{BUILDTIME}\n' + +@@ -2878,13 +2885,54 @@ class OzImageTask(BaseTaskHandler): + %sG + + +-""" % (imgname, self.opts.get('disk_size')) ++""" % (self.imgname, self.opts.get('disk_size')) ++ return template ++ ++ def makeDockerUtilTemplate(self, inst_tree, pkg_group): ++ """ ++ Generate a "TDL" for ImageFactory to build a utility image to run ++ docker commands on a docker image. ++ ++ @args: ++ inst_tree - a string, a URL to the install tree (a compose) ++ @returns: ++ An XML string that imagefactory can consume ++ """ ++ distname, distver = self.parseDistro(self.opts.get('distro')) ++ template = """ ++""" % self.id + return template + + def parseDistro(self, distro): + """ + Figure out the distribution name and version we are going to build an + image on. ++ ++ args: ++ a string of the form: RHEL-X.Y, Fedora-NN, CentOS-X.Y, or SL-X.Y ++ returns: ++ a 2-element list, depends on the distro where the split happened + """ + if distro.startswith('RHEL'): + major, minor = distro.split('.') +@@ -2900,7 +2948,8 @@ class OzImageTask(BaseTaskHandler): + else: + raise BuildError('Unknown or supported distro given: %s' % distro) + +- def fixImageXML(self, format, imgname, filename, xmltext): ++ def fixImageXML(self, format, filename, xmltext): ++ + """ + The XML generated by Oz/ImageFactory knows nothing about the name + or image format conversions Koji does. We fix those values in the +@@ -2909,18 +2958,18 @@ class OzImageTask(BaseTaskHandler): + + @args: + format = raw, qcow2, vmdk, etc... a string representation +- name = the (file) name of the image ++ filename = the name of the XML file we will save this too + xmltext = the libvirt XML to start with + @return: + an absolute path to the modified XML + """ + newxml = xml.dom.minidom.parseString(xmltext) + ename = newxml.getElementsByTagName('name')[0] +- ename.firstChild.nodeValue = imgname ++ ename.firstChild.nodeValue = self.imgname + esources = newxml.getElementsByTagName('source') + for e in esources: + if e.hasAttribute('file'): +- e.setAttribute('file', '%s.%s' % (imgname, format)) ++ e.setAttribute('file', '%s.%s' % (self.imgname, format)) + edriver = newxml.getElementsByTagName('driver')[0] + edriver.setAttribute('type', format) + xml_path = os.path.join(self.workdir, filename) +@@ -2933,6 +2982,9 @@ class OzImageTask(BaseTaskHandler): + """ + Locate a screenshot taken by libvirt in the case of build failure, + if it exists. If it does, return the path, else return None. ++ ++ @args: none ++ @returns: a path to a screenshot take by libvirt + """ + shotdir = os.path.join(self.workdir, 'oz_screenshots') + screenshot = None +@@ -2947,7 +2999,7 @@ class OzImageTask(BaseTaskHandler): + class BaseImageTask(OzImageTask): + + Methods = ['createImage'] +- _taskWeight = 1.5 ++ _taskWeight = 2.0 + + def getRootDevice(self): + """ +@@ -2959,7 +3011,38 @@ class BaseImageTask(OzImageTask): + return part.disk + raise koji.ApplianceError, 'kickstart lacks a "/" mountpoint' + +- def format_deps(self, formats): ++ def _makeDockerCmds(self, tags): ++ """ ++ When building a docker image, we call docker commands on it from ++ within the "utility" image. ImageFactory accepts an XML string that ++ describes what commands to run, which is what this method returns. ++ ++ @args: None ++ tags- a list of tags to apply to the docker image ++ @returns: ++ an XML string with docker commands ++ """ ++ # TODO: set up the file name correctly ++ # TODO: set up the image id in docker correctly ++ cmds = """ ++""" % self.imgname ++ self.logger.debug('docker command template: %s' % cmds) ++ return cmds ++ ++ def _format_deps(self, formats): + """ + Return a dictionary where the keys are the image formats we need to + build/convert, and the values are booleans that indicate whether the +@@ -2988,136 +3071,266 @@ class BaseImageTask(OzImageTask): + self.logger.debug('Image delivery plan: %s' % f_dict) + return f_dict + +- def do_images(self, arch, template, ozlog, imgname): ++ def do_images(self, template, inst_tree): + """ + Call out to ImageFactory to build the image(s) we want. Returns a dict + of details for each image type we had to ask ImageFactory to build +- {format: dispatcher object} + """ +- def do_target_image(base_id, image_type, ova_opts={}): +- self.logger.debug('ova_opts: %s' % ova_opts) +- try: +- target = bd.builder_for_target_image(image_type, +- image_id=base_id, template=None, parameters=ova_opts) +- target.target_thread.join() +- except: +- tlog.removeHandler(fhandler) +- self.uploadFile(ozlog) +- self.logger.debug( +- 'Target image results: %s' % target.target_image.status) +- if target.target_image.status == 'FAILED': +- if not self.session.checkUpload('', os.path.basename(ozlog)): +- tlog.removeHandler(fhandler) +- self.uploadFile(ozlog) +- raise koji.ApplianceError('Image status is %s: %s' % ( +- target.target_image.status, +- target.target_image.status_detail)) +- return target +- +- fhandler = logging.FileHandler(ozlog) +- bd = BuildDispatcher() +- tlog = logging.getLogger() +- tlog.setLevel(logging.DEBUG) +- tlog.addHandler(fhandler) ++ fcalls = {'raw': self._buildBase, ++ 'vmdk': self._buildConvert, ++ 'vdi': self._buildConvert, ++ 'qcow': self._buildConvert, ++ 'qcow2': self._buildConvert, ++ 'rhevm-ova': self._buildOVA, ++ 'vsphere-ova': self._buildOVA, ++ 'docker': self._buildIndirect ++ } ++ # add a handler to the logger so that we capture ImageFactory's logging ++ self.fhandler = logging.FileHandler(self.ozlog) ++ self.bd = BuildDispatcher() ++ self.tlog = logging.getLogger() ++ self.tlog.setLevel(logging.DEBUG) ++ self.tlog.addHandler(self.fhandler) + images = {} ++ random.seed() # necessary to ensure a unique mac address ++ # if we need a utility image for the indirection plugin we create it ++ # we do not join() this until later ++ # Future: 'livecd' will be in here ++ if 'docker' in self.formats: ++ self.session.host.setTaskWeight(self.id, 3.0) ++ self.util_img = self._buildDockerUtility(inst_tree) + params = {'install_script': str(self.ks.handler), + 'offline_icicle': True} +- random.seed() # necessary to ensure a unique mac address +- try: +- base = bd.builder_for_base_image(template, parameters=params) +- base.base_thread.join() +- except: +- # upload log even if we failed to help diagnose an issue +- tlog.removeHandler(fhandler) +- self.uploadFile(ozlog) +- self.logger.debug('Base image results: %s' % base.base_image.status) +- if base.base_image.status == 'FAILED': ++ # build the base (raw) image ++ self.base_img = self._buildBase(template, params) ++ images['raw'] = {'image': self.base_img.base_image.data, ++ 'icicle': self.base_img.base_image.icicle} ++ # Do the rest of the image types (everything but raw) ++ for format in self.formats: ++ if format == 'raw': ++ continue ++ self.logger.info('dispatching %s image builder' % format) ++ images[format] = fcalls[format](format) ++ imginfo = self._processXML(images) ++ self.tlog.removeHandler(self.fhandler) ++ self.uploadFile(self.ozlog) ++ return imginfo ++ ++ def _processXML(self, images): ++ """ ++ Produce XML that libvirt can import to create a domain based on image(s) ++ we produced. We save the location of the XML file in the dictionary ++ it corresponds to here. ++ ++ @args: ++ images - a dict where the keys are image formats, and the values ++ are dicts with details about the image (location, icicle, etc) ++ @returns: ++ a dictionary just like "images" but with a new key called "libvirt" ++ that points to the path of the XML file for that image ++ """ ++ imginfo = {} ++ for fmt in images.keys(): ++ imginfo[fmt] = images[fmt] ++ lxml = self.fixImageXML(fmt, 'libvirt-%s-%s.xml' % (fmt, self.arch), ++ self.base_img.base_image.parameters['libvirt_xml']) ++ imginfo[fmt]['libvirt'] = lxml ++ return imginfo ++ ++ def _checkImageState(self, image): ++ """ ++ Query ImageFactory for details of a dispatched image build. If it is ++ FAILED we raise an exception. ++ ++ @args: ++ image - a build dispatcher object returned by a BuildDispatcher ++ @returns: nothing ++ """ ++ if image.target_image: ++ status = image.target_image.status ++ details = image.target_image.status_detail ++ else: ++ status = image.base_image.status ++ details = image.base_image.status_detail ++ self.logger.debug('check image results: %s' % status) ++ if status == 'FAILED': + scrnshot = self.getScreenshot() + if scrnshot: + ext = scrnshot[-3:] + self.uploadFile(scrnshot, remoteName='screenshot.%s' % ext) +- base.os_plugin.abort() # forcibly tear down the VM ++ image.os_plugin.abort() # forcibly tear down the VM + # TODO abort when a task is CANCELLED +- if not self.session.checkUpload('', os.path.basename(ozlog)): +- tlog.removeHandler(fhandler) +- self.uploadFile(ozlog) +- ++ if not self.session.checkUpload('', os.path.basename(self.ozlog)): ++ self.tlog.removeHandler(self.fhandler) ++ self.uploadFile(self.ozlog) + raise koji.ApplianceError('Image status is %s: %s' % +- (base.base_image.status, base.base_image.status_detail)) +- lxml = self.fixImageXML('raw', imgname, +- 'libvirt-%s-%s.xml' % ('raw', arch), +- base.base_image.parameters['libvirt_xml']) +- images['raw'] = {'image': base.base_image.data, 'libvirt': lxml, +- 'icicle': base.base_image.icicle} +- +- # target-image type images +- if 'docker' in self.formats: +- targ = do_target_image(base.base_image.identifier, 'docker', +- ova_opts={'compress': 'gzip'}) +- images['docker'] = {'image': targ.target_image.data} ++ (status, details)) ++ ++ def _buildDockerUtility(self, inst_tree): ++ """ ++ Build a utility image used for the indirection plugin later. Docker ++ and eventually liveCDs will use this. The utility image provides an ++ environment where we will run post-build commands on the base image ++ we generated. + +- ova_opts = {} ++ @args: ++ inst_tree - a string URL to an install tree (a compose) ++ @returns: ++ a dict with some details about the image ++ """ ++ #dtemp = self.makeDockerUtilTemplate(inst_tree, 'docker-build') ++ dtemp = self.makeDockerUtilTemplate('http://download.lab.bos.redhat.com/rel-eng/RHEL-7.0-20140507.0/compose/Server/x86_64/os', 'docker-build') ++ # TODO: enable this and store it properly ++ # pkgs = [x['packagelist'] for x in brew.getTagGroups('rhel-7.0-build') if x['name'] == 'livecd-build'][0] ++ # print '\n'.join([p['package'] for p in pkgs]) ++ dparams = {'generate_icicle': False} ++ utilname = 'koji-%s-util' % self.id ++ return self._buildBase(dtemp, dparams, wait=False) ++ ++ def _buildBase(self, template, params, wait=True): ++ """ ++ Build a base image using ImageFactory. This is a "raw" image. ++ ++ @args: ++ template - an XML string for the TDL ++ params - a dict that controls some ImageFactory settings ++ wait - call join() on the building thread if True ++ @returns: ++ a dict with some metadata about the image (includes an icicle) ++ """ ++ # TODO: test the failure case where IF itself throws an exception ++ # ungracefully (missing a plugin for example) ++ # may need to still upload ozlog and remove the log handler ++ self.logger.info('dispatching a baseimg builder') ++ self.logger.debug('templates: %s' % template) ++ self.logger.debug('params: %s' % params) ++ base = self.bd.builder_for_base_image(template, parameters=params) ++ if wait: ++ base.base_thread.join() ++ self._checkImageState(base) ++ return base ++ ++ def _buildOVA(self, format): ++ """ ++ Build an OVA target image. This is a format supported by RHEV and ++ vSphere ++ ++ @args: ++ format - a string representing the image format, "rhevm-ova" ++ @returns ++ a dict with some metadata about the image ++ """ ++ img_opts = {} + if self.opts.get('ova_option'): +- ova_opts = dict([o.split('=') for o in self.opts.get('ova_option')]) +- for format in ('rhevm-ova', 'vsphere-ova'): +- # assumes self.formats is pre-populated with rhevm/vsphere if the +- # "ova" format for them was requested with --format only +- if format not in self.formats: +- continue +- targ = do_target_image(base.base_image.identifier, +- format.replace('-ova', '')) +- # Target images do not have their own modified libvirt xml files. +- # They may not even be bootable with libvirt. +- lxml = self.fixImageXML(format, imgname, +- 'libvirt-%s-%s.xml' % (format, arch), +- base.base_image.parameters['libvirt_xml']) +- targ2 = do_target_image(targ.target_image.identifier, 'OVA', +- ova_opts=ova_opts) +- images[format] = {'libvirt': lxml, 'image': targ2.target_image.data} +- tlog.removeHandler(fhandler) +- self.uploadFile(ozlog) +- +- # qemu-img conversions +- for format in ('qcow', 'qcow2', 'vdi', 'vmdk'): +- if format not in self.formats: +- continue +- newimg = os.path.join(self.workdir, imgname + '.%s' % format) +- cmd = ['/usr/bin/qemu-img', 'convert', '-f', 'raw', '-O', +- format, base.base_image.data, newimg] +- if format in ('qcow', 'qcow2'): +- cmd.insert(2, '-c') # enable compression for qcow images +- conlog = os.path.join(self.workdir, +- 'qemu-img-%s-%s.log' % (format, arch)) +- log_output(self.session, cmd[0], cmd, conlog, +- self.getUploadDir(), logerror=1) +- lxml = self.fixImageXML(format, imgname, +- 'libvirt-%s-%s.xml' % (format, arch), +- base.base_image.parameters['libvirt_xml']) +- images[format] = {'image': newimg, 'libvirt': lxml} +- +- return images ++ img_opts = dict([o.split('=') for o in self.opts.get('ova_option')]) ++ targ = self._do_target_image(self.base_img.base_image.identifier, ++ format.replace('-ova', '')) ++ targ2 = self._do_target_image(targ.target_image.identifier, 'OVA', ++ img_opts=img_opts) ++ return {'image': targ2.target_image.data} ++ ++ def _do_target_image(self, base_id, image_type, img_opts={}): ++ """ ++ A generic method for building what ImageFactory calls "target images". ++ These are images based on a raw disk that was built before using the ++ _buildBase method. ++ ++ @args: ++ base_id - a string ID of the image to build off of ++ image_type - a string representing the target type. ImageFactory ++ uses this to figure out what plugin to run ++ img_opts - a dict of additional options that specific to the target ++ type we pass in via image_type ++ @returns: ++ A Builder() object from ImageFactory that contains information ++ about the image building include state and progress. ++ """ ++ # TODO: test the failure case where IF itself throws an exception ++ # ungracefully (missing a plugin for example) ++ # may need to still upload ozlog and remove the log handler ++ self.logger.debug('img_opts: %s' % img_opts) ++ target = self.bd.builder_for_target_image(image_type, ++ image_id=base_id, template=None, parameters=img_opts) ++ target.target_thread.join() ++ self._checkImageState(target) ++ return target ++ ++ def _buildConvert(self, format): ++ """ ++ Build an image by converting the format using qemu-img. This is method ++ enables a variety of formats like qcow, qcow2, vmdk, and vdi. ++ ++ @args: ++ format - a string representing the image format, "qcow2" ++ @returns ++ a dict with some metadata about the image ++ """ ++ ++ newimg = os.path.join(self.workdir, self.imgname + '.%s' % format) ++ cmd = ['/usr/bin/qemu-img', 'convert', '-f', 'raw', '-O', ++ format, self.base_img.base_image.data, newimg] ++ if format in ('qcow', 'qcow2'): ++ cmd.insert(2, '-c') # enable compression for qcow images ++ conlog = os.path.join(self.workdir, ++ 'qemu-img-%s-%s.log' % (format, self.arch)) ++ log_output(self.session, cmd[0], cmd, conlog, ++ self.getUploadDir(), logerror=1) ++ return {'image': newimg} ++ ++ def _buildIndirect(self, format): ++ """ ++ "Indirect" images are target images that use the indirection plugin. ++ This plugin makes use of an additional guest, and launches that to run ++ commands on the base image we built earlier. The image that backs the ++ modifying guest is called the "utility" image, which we built before. ++ ++ @args: ++ format - a string representing the image format, "qcow2" ++ @returns ++ a dict with some metadata about the image ++ """ ++ # Future: livecd will be introduced here too ++ # we should have waited for and checked the base image already ++ # TODO: log the shit out of this new stuff ++ # TODO: test the failure case where IF itself throws an exception ++ # ungracefully (missing a plugin for example) ++ # may need to still upload ozlog and remove the log handler ++ self.util_img.base_thread.join() ++ self._checkImageState(self.util_img) ++ params = { ++ 'compress': 'gzip', ++ 'utility_image': self.util_img.base_image.identifier, ++ 'utility_customizations': self._makeDockerCmds([self.imgname]) ++ } ++ targ = self.bd.builder_for_target_image('indirection', ++ image_id=self.base.base_image.identifier, ++ template=None, parameters=params) ++ targ.target_thread.join() ++ self._checkImageState(targ) ++ return {'image': targ.target_image.data} + + def handler(self, name, version, release, arch, target_info, build_tag, repo_info, inst_tree, opts=None): + + if opts == None: + opts = {} ++ self.arch = arch + self.opts = opts +- self.formats = self.format_deps(opts.get('format')) ++ self.formats = self._format_deps(opts.get('format')) + + # First, prepare the kickstart to use the repos we tell it + kspath = self.fetchKickstart() + self.readKickstart(kspath) +- kskoji = self.prepareKickstart(repo_info, target_info, arch) ++ kskoji = self.prepareKickstart(repo_info, target_info) + + # auto-generate a TDL file and config dict for ImageFactory +- imgname = '%s-%s-%s.%s' % (name, version, release, arch) +- template = self.makeTemplate(imgname, arch, inst_tree) ++ self.imgname = '%s-%s-%s.%s' % (name, version, release, self.arch) ++ template = self.makeTemplate(inst_tree) + self.logger.debug('oz template: %s' % template) + config = self.makeConfig() + self.logger.debug('IF config object: %s' % config) + ApplicationConfiguration(configuration=config) + +- tdl_path = os.path.join(self.workdir, 'tdl-%s.xml' % arch) ++ tdl_path = os.path.join(self.workdir, 'tdl-%s.xml' % self.arch) + tdl = open(tdl_path, 'w') + tdl.write(template) + tdl.close() +@@ -3129,16 +3342,16 @@ class BaseImageTask(OzImageTask): + # the likelihood of image tasks clashing here is very small) + rm = ReservationManager() + rm._listen_port = rm.MIN_PORT + self.id % (rm.MAX_PORT - rm.MIN_PORT) +- ozlogname = 'oz-%s.log' % arch +- ozlog = os.path.join(self.workdir, ozlogname) ++ ozlogname = 'oz-%s.log' % self.arch ++ self.ozlog = os.path.join(self.workdir, ozlogname) + + # invoke the image builds +- images = self.do_images(arch, template, ozlog, imgname) ++ images = self.do_images(template, inst_tree) + images['raw']['tdl'] = os.path.basename(tdl_path), + + # structure the results to pass back to the hub: + imgdata = { +- 'arch': arch, ++ 'arch': self.arch, + 'task_id': self.id, + 'logs': [ozlogname], + 'name': name, +@@ -3175,7 +3388,7 @@ class BaseImageTask(OzImageTask): + rpm['epoch'] = int(bits[4]) + imgdata['rpmlist'].append(rpm) + # TODO: hack to make this work for now, need to refactor +- br = BuildRoot(self.session, self.options, build_tag, arch, ++ br = BuildRoot(self.session, self.options, build_tag, self.arch, + self.id, repo_id=repo_info['id']) + br.markExternalRPMs(imgdata['rpmlist']) + +@@ -3183,11 +3396,11 @@ class BaseImageTask(OzImageTask): + for format in (f for f in self.formats.keys() if self.formats[f]): + newimg = images[format]['image'] + if 'ova' in format: +- newname = imgname + '.' + format.replace('-', '.') ++ newname = self.imgname + '.' + format.replace('-', '.') + elif format == 'docker': +- newname = imgname + '.' + 'tar.gz' ++ newname = self.imgname + '.' + 'tar.gz' + else: +- newname = imgname + '.' + format ++ newname = self.imgname + '.' + format + if format != 'docker': + lxml = images[format]['libvirt'] + imgdata['files'].append(os.path.basename(lxml)) +-- +2.0.4 + diff --git a/0002-image-support-xz-compressed-raw-files.patch b/0002-image-support-xz-compressed-raw-files.patch deleted file mode 100644 index 60a8f63..0000000 --- a/0002-image-support-xz-compressed-raw-files.patch +++ /dev/null @@ -1,119 +0,0 @@ -From ce89836df875f17ba94c9d47144e89fda22612ce Mon Sep 17 00:00:00 2001 -From: Dennis Gilmore -Date: Thu, 29 May 2014 23:05:07 -0500 -Subject: [PATCH 2/2] image: support xz compressed raw files - -as we publish the raw files on the mirrors we want to be able to request -xz compressed versions of theraw image. - -Signed-off-by: Dennis Gilmore ---- - builder/kojid | 25 ++++++++++++++++++++++++- - cli/koji | 4 ++-- - docs/schema-upgrade-1.9-next.sql | 9 +++++++++ - docs/schema.sql | 1 + - 4 files changed, 36 insertions(+), 3 deletions(-) - create mode 100644 docs/schema-upgrade-1.9-next.sql - -diff --git a/builder/kojid b/builder/kojid -index 14309bb..34c62d8 100755 ---- a/builder/kojid -+++ b/builder/kojid -@@ -2737,7 +2737,7 @@ class BaseImageTask(OzImageTask): - we have to do this. rhevm-ova requires rhevm, but if the user did not - request it, we should not pass it back up. - """ -- supported = ('raw', 'vmdk', 'qcow', 'qcow2', 'vdi', 'rhevm-ova', 'vsphere-ova', 'docker') -+ supported = ('raw', 'raw-xz', 'vmdk', 'qcow', 'qcow2', 'vdi', 'rhevm-ova', 'vsphere-ova', 'docker') - for f in formats: - if f not in supported: - raise koji.ApplianceError('Invalid format: %s' % f) -@@ -2863,6 +2863,27 @@ class BaseImageTask(OzImageTask): - base.base_image.parameters['libvirt_xml']) - images[format] = {'image': newimg, 'libvirt': lxml} - -+ # xz compress the raw disk image if asked for -+ for format in ('raw-xz',): -+ if format not in self.formats: -+ continue -+ newimg = os.path.join(self.workdir, imgname + 'raw.xz') -+ rawimg = os.path.join(self.workdir, imgname + 'raw') -+ cmd = ['/bin/cp', base.base_image.data, rawimg] -+ conlog = os.path.join(self.workdir, -+ 'xz-cp-%s-%s.log' % (format, arch)) -+ log_output(self.session, cmd[0], cmd, conlog, -+ self.getUploadDir(), logerror=1) -+ cmd = ['/usr/bin/xz', '-z', rawimg] -+ conlog = os.path.join(self.workdir, -+ 'xz-%s-%s.log' % (format, arch)) -+ log_output(self.session, cmd[0], cmd, conlog, -+ self.getUploadDir(), logerror=1) -+ lxml = self.fixImageXML(format, imgname, -+ 'libvirt-%s-%s.xml' % (format, arch), -+ base.base_image.parameters['libvirt_xml']) -+ images[format] = {'image': newimg, 'libvirt': lxml} -+ - return images - - def handler(self, name, version, release, arch, target_info, build_tag, repo_info, inst_tree, opts=None): -@@ -2954,6 +2975,8 @@ class BaseImageTask(OzImageTask): - newname = imgname + '.' + format.replace('-', '.') - elif format == 'docker': - newname = imgname + '.' + 'tar.gz' -+ elif format == 'raw-xz': -+ newname = imgname + '.' + 'raw.xz' - else: - newname = imgname + '.' + format - if format != 'docker': -diff --git a/cli/koji b/cli/koji -index 504b4ba..1ba273f 100755 ---- a/cli/koji -+++ b/cli/koji -@@ -4981,7 +4981,7 @@ def handle_spin_appliance(options, session, args): - help=_("Set the number of virtual cpus in the appliance, " + - "default is 1")) - parser.add_option("--format", metavar="DISK_FORMAT", default='raw', -- help=_("Disk format, default is raw. Other options are qcow, " + -+ help=_("Disk format, default is raw. Other options are raw-xz, qcow, " + - "qcow2, and vmx.")) - - (task_options, args) = parser.parse_args(args) -@@ -4998,7 +4998,7 @@ def handle_spin_appliance(options, session, args): - def handle_image_build(options, session, args): - """Create a disk image given an install tree""" - formats = ('vmdk', 'qcow', 'qcow2', 'vdi', 'rhevm-ova', 'vsphere-ova', -- 'docker') -+ 'docker', 'raw-xz') - usage = _("usage: %prog image-build [options] " + - " [...]") - usage += _("\n %prog image-build --config FILE") -diff --git a/docs/schema-upgrade-1.9-next.sql b/docs/schema-upgrade-1.9-next.sql -new file mode 100644 -index 0000000..7d45e91 ---- /dev/null -+++ b/docs/schema-upgrade-1.9-next.sql -@@ -0,0 +1,9 @@ -+-- schema migration from version 1.9 to next -+-- note: this update will require additional steps, please see the migration doc -+ -+BEGIN; -+ -+-- new archive types -+insert into archivetypes (name, description, extensions) values ('raw-xz', 'xz compressed raw disk image', 'raw.xz'); -+ -+COMMIT; -diff --git a/docs/schema.sql b/docs/schema.sql -index 56418c9..91bcfd2 100644 ---- a/docs/schema.sql -+++ b/docs/schema.sql -@@ -713,6 +713,7 @@ insert into archivetypes (name, description, extensions) values ('pdb', 'Windows - insert into archivetypes (name, description, extensions) values ('oem', 'Windows driver oem file', 'oem'); - insert into archivetypes (name, description, extensions) values ('iso', 'CD/DVD Image', 'iso'); - insert into archivetypes (name, description, extensions) values ('raw', 'Raw disk image', 'raw'); -+insert into archivetypes (name, description, extensions) values ('raw-xz', 'xz compressed raw disk image', 'raw.xz'); - insert into archivetypes (name, description, extensions) values ('qcow', 'QCOW image', 'qcow'); - insert into archivetypes (name, description, extensions) values ('qcow2', 'QCOW2 image', 'qcow2'); - insert into archivetypes (name, description, extensions) values ('vmdk', 'vSphere image', 'vmdk'); --- -2.0.0 - diff --git a/0002-refactor-do_images.patch b/0002-refactor-do_images.patch new file mode 100644 index 0000000..2b6a73d --- /dev/null +++ b/0002-refactor-do_images.patch @@ -0,0 +1,439 @@ +From ef2e41a672fecf6168dca5a177d61c8e77427279 Mon Sep 17 00:00:00 2001 +From: Jay Greguske +Date: Fri, 25 Jul 2014 13:34:10 -0400 +Subject: [PATCH 2/3] refactor do_images + +--- + builder/kojid | 248 +++++++++++++++++----------------------------------------- + 1 file changed, 73 insertions(+), 175 deletions(-) + +diff --git a/builder/kojid b/builder/kojid +index 2ea0105..d3446b9 100755 +--- a/builder/kojid ++++ b/builder/kojid +@@ -2749,7 +2749,8 @@ class OzImageTask(BaseTaskHandler): + + @args: + kspath: path to a kickstart file +- @returns: None ++ @returns: ++ a kickstart object returned by pykickstart + """ + # XXX: If the ks file came from a local path and has %include + # macros, Oz will fail because it can only handle flat files. +@@ -2759,35 +2760,36 @@ class OzImageTask(BaseTaskHandler): + ksparser.stringToVersion(self.opts['ksversion'])) + else: + version = ksparser.makeVersion() +- self.ks = ksparser.KickstartParser(version) ++ ks = ksparser.KickstartParser(version) + self.logger.debug('attempting to read kickstart: %s' % kspath) + try: +- self.ks.readKickstart(kspath) ++ ks.readKickstart(kspath) + except IOError, e: + raise koji.BuildError("Failed to read kickstart file " + "'%s' : %s" % (kspath, e)) + except kserrors.KickstartError, e: + raise koji.BuildError("Failed to parse kickstart file " + "'%s' : %s" % (kspath, e)) ++ return ks + +- def prepareKickstart(self, repo_info, target_info): ++ def prepareKickstart(self, kspath): + """ + Process the ks file to be used for controlled image generation. This + method also uploads the modified kickstart file to the task output +- area on the hub.. ++ area on the hub. + + @args: +- target_info: a sesion.getBuildTarget() object +- repo_info: session.getRepo() object ++ kspath: a path to a kickstart file + @returns: +- absolute path to a processed kickstart file ++ a kickstart object with koji-specific modifications + """ ++ ks = self.readKickstart(kspath) + # Now we do some kickstart manipulation. If the user passed in a repo + # url with --repo, then we substitute that in for the repo(s) specified + # in the kickstart file. If --repo wasn't specified, then we use the + # repo associated with the target passed in initially. +- self.ks.handler.repo.repoList = [] # delete whatever the ks file told us +- repo_class = kscontrol.dataMap[self.ks.version]['RepoData'] ++ ks.handler.repo.repoList = [] # delete whatever the ks file told us ++ repo_class = kscontrol.dataMap[ks.version]['RepoData'] + # TODO: sensibly use "url" and "repo" commands in kickstart + if self.opts.get('repo'): + # the user used --repo at least once +@@ -2795,33 +2797,41 @@ class OzImageTask(BaseTaskHandler): + index = 0 + for user_repo in user_repos: + repo_url = user_repo.replace('$arch', self.arch) +- self.ks.handler.repo.repoList.append(repo_class( ++ ks.handler.repo.repoList.append(repo_class( + baseurl=repo_url, name='koji-override-%i' % index)) + index += 1 + else: + # --repo was not given, so we use the target's build repo + path_info = koji.PathInfo(topdir=self.options.topurl) +- repopath = path_info.repo(repo_info['id'], +- target_info['build_tag_name']) ++ repopath = path_info.repo(self.repo_info['id'], ++ self.target_info['build_tag_name']) + baseurl = '%s/%s' % (repopath, self.arch) + self.logger.debug('BASEURL: %s' % baseurl) +- self.ks.handler.repo.repoList.append(repo_class( ++ ks.handler.repo.repoList.append(repo_class( + baseurl=baseurl, name='koji-override-0')) ++ return ks + +- # Write out the new ks file. Note that things may not be in the same +- # order and comments in the original ks file may be lost. +- kskoji = os.path.join(self.workdir, 'koji-image-%s-%i.ks' % +- (target_info['build_tag_name'], self.id)) +- self.logger.debug('modified ks file: %s' % kskoji) +- outfile = open(kskoji, 'w') +- outfile.write(str(self.ks.handler)) ++ def writeKickstart(self, ksobj, ksname): ++ """ ++ Write out the new ks file. Note that things may not be in the same ++ order and comments in the original ks file may be lost. ++ ++ @args: ++ ksobj: a pykickstart object of what we want to write ++ ksname: file name for the kickstart ++ @returns: ++ an absolute path to the kickstart file we wrote ++ """ ++ kspath = os.path.join(self.workdir, ksname) ++ outfile = open(kspath, 'w') ++ outfile.write(str(ksobj.handler)) + outfile.close() + + # put the new ksfile in the output directory +- if not os.path.exists(kskoji): +- raise koji.BuildError, "KS file missing: %s" % kskoji +- self.uploadFile(kskoji) # upload the modified ks file +- return kskoji ++ if not os.path.exists(kspath): ++ raise koji.BuildError, "KS file missing: %s" % kspath ++ self.uploadFile(kspath) # upload the modified ks file ++ return kspath + + def makeConfig(self): + """ +@@ -2854,16 +2864,20 @@ class OzImageTask(BaseTaskHandler): + 'storage_path': os.path.join(self.workdir, 'output_image')}, + } + +- def makeTemplate(self, inst_tree): ++ def makeTemplate(self, name, inst_tree): + """ + Generate a simple "TDL" for ImageFactory to build an image with. + + @args: +- inst_tree - a string, a URL to the install tree (a compose) ++ name: a name for the image ++ inst_tree: a string, a URL to the install tree (a compose) + @returns: + An XML string that imagefactory can consume + """ + # we have to split this up so the variable substitution works ++ # XXX: using a packages section (which we don't) will have IF boot the ++ # image and attempt to ssh in. This breaks docker image creation. ++ # TODO: intelligently guess the distro based on the install tree URL + distname, distver = self.parseDistro(self.opts.get('distro')) + template = """ +-""" % (self.imgname, self.opts.get('disk_size')) +- return template +- +- def makeDockerUtilTemplate(self, inst_tree, pkg_group): +- """ +- Generate a "TDL" for ImageFactory to build a utility image to run +- docker commands on a docker image. +- +- @args: +- inst_tree - a string, a URL to the install tree (a compose) +- @returns: +- An XML string that imagefactory can consume +- """ +- distname, distver = self.parseDistro(self.opts.get('distro')) +- template = """ +-""" % self.id ++""" % (name, self.opts.get('disk_size')) + return template + + def parseDistro(self, distro): +@@ -3001,47 +2980,6 @@ class BaseImageTask(OzImageTask): + Methods = ['createImage'] + _taskWeight = 2.0 + +- def getRootDevice(self): +- """ +- Return the device name for the / partition, as specified in the +- kickstart file. Appliances should have this defined. +- """ +- for part in self.ks.handler.partition.partitions: +- if part.mountpoint == '/': +- return part.disk +- raise koji.ApplianceError, 'kickstart lacks a "/" mountpoint' +- +- def _makeDockerCmds(self, tags): +- """ +- When building a docker image, we call docker commands on it from +- within the "utility" image. ImageFactory accepts an XML string that +- describes what commands to run, which is what this method returns. +- +- @args: None +- tags- a list of tags to apply to the docker image +- @returns: +- an XML string with docker commands +- """ +- # TODO: set up the file name correctly +- # TODO: set up the image id in docker correctly +- cmds = """ +-""" % self.imgname +- self.logger.debug('docker command template: %s' % cmds) +- return cmds +- + def _format_deps(self, formats): + """ + Return a dictionary where the keys are the image formats we need to +@@ -3049,8 +2987,7 @@ class BaseImageTask(OzImageTask): + output should be included in the task results. + + Some image formats require others to be processed first, which is why +- we have to do this. rhevm-ova requires rhevm, but if the user did not +- request it, we should not pass it back up. ++ we have to do this. raw files in particular may not be kept. + """ + supported = ('raw', 'vmdk', 'qcow', 'qcow2', 'vdi', 'rhevm-ova', 'vsphere-ova', 'docker') + for f in formats: +@@ -3071,7 +3008,7 @@ class BaseImageTask(OzImageTask): + self.logger.debug('Image delivery plan: %s' % f_dict) + return f_dict + +- def do_images(self, template, inst_tree): ++ def do_images(self, ks, template, inst_tree): + """ + Call out to ImageFactory to build the image(s) we want. Returns a dict + of details for each image type we had to ask ImageFactory to build +@@ -3083,7 +3020,7 @@ class BaseImageTask(OzImageTask): + 'qcow2': self._buildConvert, + 'rhevm-ova': self._buildOVA, + 'vsphere-ova': self._buildOVA, +- 'docker': self._buildIndirect ++ 'docker': self._buildDocker + } + # add a handler to the logger so that we capture ImageFactory's logging + self.fhandler = logging.FileHandler(self.ozlog) +@@ -3093,13 +3030,7 @@ class BaseImageTask(OzImageTask): + self.tlog.addHandler(self.fhandler) + images = {} + random.seed() # necessary to ensure a unique mac address +- # if we need a utility image for the indirection plugin we create it +- # we do not join() this until later +- # Future: 'livecd' will be in here +- if 'docker' in self.formats: +- self.session.host.setTaskWeight(self.id, 3.0) +- self.util_img = self._buildDockerUtility(inst_tree) +- params = {'install_script': str(self.ks.handler), ++ params = {'install_script': str(ks.handler), + 'offline_icicle': True} + # build the base (raw) image + self.base_img = self._buildBase(template, params) +@@ -3166,26 +3097,6 @@ class BaseImageTask(OzImageTask): + raise koji.ApplianceError('Image status is %s: %s' % + (status, details)) + +- def _buildDockerUtility(self, inst_tree): +- """ +- Build a utility image used for the indirection plugin later. Docker +- and eventually liveCDs will use this. The utility image provides an +- environment where we will run post-build commands on the base image +- we generated. +- +- @args: +- inst_tree - a string URL to an install tree (a compose) +- @returns: +- a dict with some details about the image +- """ +- #dtemp = self.makeDockerUtilTemplate(inst_tree, 'docker-build') +- dtemp = self.makeDockerUtilTemplate('http://download.lab.bos.redhat.com/rel-eng/RHEL-7.0-20140507.0/compose/Server/x86_64/os', 'docker-build') +- # TODO: enable this and store it properly +- # pkgs = [x['packagelist'] for x in brew.getTagGroups('rhel-7.0-build') if x['name'] == 'livecd-build'][0] +- # print '\n'.join([p['package'] for p in pkgs]) +- dparams = {'generate_icicle': False} +- utilname = 'koji-%s-util' % self.id +- return self._buildBase(dtemp, dparams, wait=False) + + def _buildBase(self, template, params, wait=True): + """ +@@ -3217,7 +3128,7 @@ class BaseImageTask(OzImageTask): + + @args: + format - a string representing the image format, "rhevm-ova" +- @returns ++ @returns: + a dict with some metadata about the image + """ + img_opts = {} +@@ -3229,6 +3140,21 @@ class BaseImageTask(OzImageTask): + img_opts=img_opts) + return {'image': targ2.target_image.data} + ++ def _buildDocker(self, format): ++ """ ++ Build a base docker image. This image will be tagged with the NVR.A ++ automatically because we name it that way in the ImageFactory TDL. ++ ++ @args: ++ format - the string "docker" ++ @returns: ++ a dict with some metadata about the image ++ """ ++ img_opts = {'compress': 'gzip'} ++ targ = self._do_target_image(self.base_img.base_image.identifier, ++ 'docker', img_opts=img_opts) ++ return {'image': targ.target_image.data} ++ + def _do_target_image(self, base_id, image_type, img_opts={}): + """ + A generic method for building what ImageFactory calls "target images". +@@ -3277,54 +3203,26 @@ class BaseImageTask(OzImageTask): + self.getUploadDir(), logerror=1) + return {'image': newimg} + +- def _buildIndirect(self, format): +- """ +- "Indirect" images are target images that use the indirection plugin. +- This plugin makes use of an additional guest, and launches that to run +- commands on the base image we built earlier. The image that backs the +- modifying guest is called the "utility" image, which we built before. +- +- @args: +- format - a string representing the image format, "qcow2" +- @returns +- a dict with some metadata about the image +- """ +- # Future: livecd will be introduced here too +- # we should have waited for and checked the base image already +- # TODO: log the shit out of this new stuff +- # TODO: test the failure case where IF itself throws an exception +- # ungracefully (missing a plugin for example) +- # may need to still upload ozlog and remove the log handler +- self.util_img.base_thread.join() +- self._checkImageState(self.util_img) +- params = { +- 'compress': 'gzip', +- 'utility_image': self.util_img.base_image.identifier, +- 'utility_customizations': self._makeDockerCmds([self.imgname]) +- } +- targ = self.bd.builder_for_target_image('indirection', +- image_id=self.base.base_image.identifier, +- template=None, parameters=params) +- targ.target_thread.join() +- self._checkImageState(targ) +- return {'image': targ.target_image.data} +- + def handler(self, name, version, release, arch, target_info, build_tag, repo_info, inst_tree, opts=None): + + if opts == None: + opts = {} + self.arch = arch ++ self.target_info = target_info ++ self.repo_info = repo_info + self.opts = opts + self.formats = self._format_deps(opts.get('format')) + + # First, prepare the kickstart to use the repos we tell it + kspath = self.fetchKickstart() +- self.readKickstart(kspath) +- kskoji = self.prepareKickstart(repo_info, target_info) ++ ks = self.prepareKickstart(kspath) ++ kskoji = self.writeKickstart(ks, ++ os.path.join(self.workdir, 'koji-%s-%i-base.ks' % ++ (self.target_info['build_tag_name'], self.id))) + + # auto-generate a TDL file and config dict for ImageFactory + self.imgname = '%s-%s-%s.%s' % (name, version, release, self.arch) +- template = self.makeTemplate(inst_tree) ++ template = self.makeTemplate(self.imgname, inst_tree) + self.logger.debug('oz template: %s' % template) + config = self.makeConfig() + self.logger.debug('IF config object: %s' % config) +@@ -3346,7 +3244,7 @@ class BaseImageTask(OzImageTask): + self.ozlog = os.path.join(self.workdir, ozlogname) + + # invoke the image builds +- images = self.do_images(template, inst_tree) ++ images = self.do_images(ks, template, inst_tree) + images['raw']['tdl'] = os.path.basename(tdl_path), + + # structure the results to pass back to the hub: +@@ -3389,7 +3287,7 @@ class BaseImageTask(OzImageTask): + imgdata['rpmlist'].append(rpm) + # TODO: hack to make this work for now, need to refactor + br = BuildRoot(self.session, self.options, build_tag, self.arch, +- self.id, repo_id=repo_info['id']) ++ self.id, repo_id=self.repo_info['id']) + br.markExternalRPMs(imgdata['rpmlist']) + + # upload the results +-- +2.0.4 + diff --git a/0003-add-raw-xz-option.patch b/0003-add-raw-xz-option.patch new file mode 100644 index 0000000..62443fc --- /dev/null +++ b/0003-add-raw-xz-option.patch @@ -0,0 +1,148 @@ +From c1b42e0c67b1050c00c922f882cc192dbc571edc Mon Sep 17 00:00:00 2001 +From: Jay Greguske +Date: Tue, 29 Jul 2014 21:33:49 -0400 +Subject: [PATCH 3/3] add raw-xz option + +--- + builder/kojid | 50 +++++++++++++++++++++++++++++++++++++++++++------- + cli/koji | 2 +- + 2 files changed, 44 insertions(+), 8 deletions(-) + +diff --git a/builder/kojid b/builder/kojid +index d3446b9..b23e9ce 100755 +--- a/builder/kojid ++++ b/builder/kojid +@@ -2096,6 +2096,18 @@ class BuildBaseImageTask(BuildImageTask): + spec_url, subtask, target_info, bld_info, + repo_info['id']) + ++ # make sure we only import the user-submitted kickstart file one ++ # time, otherwise we will have collisions. Remove it from exactly ++ # 1 results hash from the subtasks ++ if opts.has_key('kickstart'): ++ saw_ks = False ++ for arch in results.keys(): ++ ks = os.path.basename(opts.get('kickstart')) ++ if ks in results[arch]['files']: ++ if saw_ks: ++ results[arch]['files'].remove(ks) ++ saw_ks = True ++ + self.logger.debug('Image Results for hub: %s' % results) + if opts.get('scratch'): + self.session.host.moveImageBuildToScratch(self.id, results) +@@ -2728,7 +2740,7 @@ class OzImageTask(BaseTaskHandler): + if self.opts.get('ksurl'): + scm = SCM(self.opts['ksurl']) + scm.assert_allowed(self.options.allowed_scms) +- logfile = os.path.join(self.workdir, 'checkout.log') ++ logfile = os.path.join(self.workdir, 'checkout-%s.log' % self.arch) + scmsrcdir = scm.checkout(self.workdir, self.session, + self.getUploadDir(), logfile) + kspath = os.path.join(scmsrcdir, os.path.basename(ksfile)) +@@ -2989,7 +3001,7 @@ class BaseImageTask(OzImageTask): + Some image formats require others to be processed first, which is why + we have to do this. raw files in particular may not be kept. + """ +- supported = ('raw', 'vmdk', 'qcow', 'qcow2', 'vdi', 'rhevm-ova', 'vsphere-ova', 'docker') ++ supported = ('raw', 'raw-xz', 'vmdk', 'qcow', 'qcow2', 'vdi', 'rhevm-ova', 'vsphere-ova', 'docker') + for f in formats: + if f not in supported: + raise koji.ApplianceError('Invalid format: %s' % f) +@@ -3014,6 +3026,7 @@ class BaseImageTask(OzImageTask): + of details for each image type we had to ask ImageFactory to build + """ + fcalls = {'raw': self._buildBase, ++ 'raw-xz': self._buildXZ, + 'vmdk': self._buildConvert, + 'vdi': self._buildConvert, + 'qcow': self._buildConvert, +@@ -3079,10 +3092,10 @@ class BaseImageTask(OzImageTask): + """ + if image.target_image: + status = image.target_image.status +- details = image.target_image.status_detail ++ details = image.target_image.status_detail['error'] + else: + status = image.base_image.status +- details = image.base_image.status_detail ++ details = image.base_image.status_detail['error'] + self.logger.debug('check image results: %s' % status) + if status == 'FAILED': + scrnshot = self.getScreenshot() +@@ -3094,10 +3107,11 @@ class BaseImageTask(OzImageTask): + if not self.session.checkUpload('', os.path.basename(self.ozlog)): + self.tlog.removeHandler(self.fhandler) + self.uploadFile(self.ozlog) ++ if 'No disk activity' in details: ++ details = 'Automated install failed or prompted for input. See the screenshot in the task results for more information.' + raise koji.ApplianceError('Image status is %s: %s' % + (status, details)) + +- + def _buildBase(self, template, params, wait=True): + """ + Build a base image using ImageFactory. This is a "raw" image. +@@ -3121,6 +3135,29 @@ class BaseImageTask(OzImageTask): + self._checkImageState(base) + return base + ++ def _buildXZ(self, format): ++ """ ++ Use xz to compress a raw disk image. This is very straightforward. ++ ++ @args: ++ format - a string representing the image format, "raw-xz" ++ @returns: ++ a dict with some metadata about the image ++ """ ++ newimg = os.path.join(self.workdir, self.imgname + '.raw.xz') ++ rawimg = os.path.join(self.workdir, self.imgname + '.raw') ++ cmd = ['/bin/cp', self.base_img.base_image.data, rawimg] ++ conlog = os.path.join(self.workdir, ++ 'xz-cp-%s-%s.log' % (format, self.arch)) ++ log_output(self.session, cmd[0], cmd, conlog, self.getUploadDir(), ++ logerror=1) ++ cmd = ['/usr/bin/xz', '-z', rawimg] ++ conlog = os.path.join(self.workdir, ++ 'xz-%s-%s.log' % (format, self.arch)) ++ log_output(self.session, cmd[0], cmd, conlog, self.getUploadDir(), ++ logerror=1) ++ return {'image': newimg} ++ + def _buildOVA(self, format): + """ + Build an OVA target image. This is a format supported by RHEV and +@@ -3191,7 +3228,6 @@ class BaseImageTask(OzImageTask): + @returns + a dict with some metadata about the image + """ +- + newimg = os.path.join(self.workdir, self.imgname + '.%s' % format) + cmd = ['/usr/bin/qemu-img', 'convert', '-f', 'raw', '-O', + format, self.base_img.base_image.data, newimg] +@@ -3293,7 +3329,7 @@ class BaseImageTask(OzImageTask): + # upload the results + for format in (f for f in self.formats.keys() if self.formats[f]): + newimg = images[format]['image'] +- if 'ova' in format: ++ if 'ova' in format or format == 'raw-xz': + newname = self.imgname + '.' + format.replace('-', '.') + elif format == 'docker': + newname = self.imgname + '.' + 'tar.gz' +diff --git a/cli/koji b/cli/koji +index 8722f01..9451963 100755 +--- a/cli/koji ++++ b/cli/koji +@@ -5130,7 +5130,7 @@ def handle_spin_appliance(options, session, args): + def handle_image_build(options, session, args): + """Create a disk image given an install tree""" + formats = ('vmdk', 'qcow', 'qcow2', 'vdi', 'rhevm-ova', 'vsphere-ova', +- 'docker') ++ 'docker', 'raw-xz') + usage = _("usage: %prog image-build [options] " + + " [...]") + usage += _("\n %prog image-build --config FILE") +-- +2.0.4 + diff --git a/koji.spec b/koji.spec index 4ab5fee..f3ebd45 100644 --- a/koji.spec +++ b/koji.spec @@ -2,7 +2,7 @@ Name: koji Version: 1.9.0 -Release: 4%{?dist} +Release: 5%{?dist} License: LGPLv2 and GPLv2+ # koji.ssl libs (from plague) are GPLv2+ Summary: Build system tools @@ -10,8 +10,11 @@ Group: Applications/System URL: http://fedorahosted.org/koji Patch0: fedora-config.patch Patch1: 0001-move-workdir-from-tmp-koji-to-var-tmp-koji.patch -Patch2: 0002-image-support-xz-compressed-raw-files.patch -Patch3: compress-docker.patch +Patch2: compress-docker.patch +Patch3: 0001-refactor-image-build-handlers-in-kojid.patch +Patch4: 0002-refactor-do_images.patch +Patch5: 0003-add-raw-xz-option.patch + Source: https://fedorahosted.org/released/koji/koji-%{version}.tar.bz2 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) @@ -131,6 +134,8 @@ koji-web is a web UI to the Koji system. %patch1 -p1 %patch2 -p1 %patch3 -p1 +%patch4 -p1 +%patch5 -p1 %build @@ -234,6 +239,9 @@ if [ $1 = 0 ]; then fi %changelog +* Mon Aug 04 2014 Dennis Gilmore - 1.9.0-5 +- add upstream patches for better docker support + * Tue Jul 29 2014 Dennis Gilmore - 1.9.0-4 - add upstream patch to compress docker images