#!/usr/bin/env python2.4 # # SchoolTool - common information systems platform for school administration # Copyright (c) 2005 Shuttleworth Foundation, # Brian Sutherland # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # """Download, work on and move things. The objective of this program is to download something, perform some standard build commands to it ('clean', 'build', 'test'), then extract some files from the directory tree and put them somewhere. It is probably most usefull for automatically building software releases. Definition: A collection is a group of releases built at the same time. Some options can be specified at the collection level or at the release level. The collection level takes precidence. TODO: usefull: * allow spcification of the subversion revision somehow. * more options for logging: * send mail containing log records * log to file. * deal with more than just subversion url's (a la zpkg) * ??should we clean after build as well as before?? * A --clean option which removes all working directories. * Document config file. no use case: * Command line defined substitutions (doesn't really make sense ) Patches accepted;) """ import os import sys import logging import subprocess import tempfile import optparse import ZConfig class Options: conffile = os.path.join(os.path.dirname(__file__), 'config.conf') schema = os.path.join(os.path.dirname(__file__), 'configschema.xml') # global optionsd substitutions = {} def loadConfig(self): schema = ZConfig.loadSchema(self.schema) self.config, _ = ZConfig.loadConfig(schema, self.conffile) def _mangle(self, s): if s is None: return s if isinstance(s, bool): return s if not isinstance(s, str): return [self._mangle(st) for st in s] for i in self.substitutions: s = s.replace(i, self.substitutions[i]) return s def getReleases(self): """returns a dictionary of dictionaries, each one representing a release Every key in the release directory represents a build option. This function takes care of all aspects of option inheritance. """ releases = {} for collection in self.config.collections: if collection.name == self.collection_to_build: break else: raise ValueError("No collection %s found in config file" % repr(self.collection_to_build)) for name in collection.release: for r in self.config.releases: if r.name == name: break else: raise ValueError("No release %s found in config file" % repr(name)) release = {} # set some global defaults release['overwrite'] = False release['test_command'] = [] release['build_command'] = [] release['clean_command'] = [] # get options from release and then collection for option in ['url', 'targetdir', 'overwrite', 'build_command', 'test_command', 'clean_command']: if getattr(r, option): release[option] = self._mangle(getattr(r, option)) if getattr(collection, option, None): release[option] = self._mangle(getattr(collection, option)) products = [] for p in r.products: products.append(self._mangle([p.source, p.dest])) release['products'] = products releases[name] = release return releases class CommandException(Exception): pass class CommandRunner: """Runs commands in the shell trying to be good about logging.""" def run(self, cmd): """Run `cmd` on the shell logging the output.""" logging.info('========= running: %s =========' % cmd) stdout = tempfile.TemporaryFile() stderr = tempfile.TemporaryFile() retval = subprocess.call(cmd, shell=True, stderr=stderr, stdout=stdout) # XXX stdout and std error should be intermixed stdout.seek(0) while 1: line = stdout.readline() if line == '': break if retval: # If the command failed, put the stdout as an error logging.error(line.strip()) else: logging.debug(line.strip()) stdout.close() stderr.seek(0) while 1: line = stderr.readline() if line == '': break logging.error(line.strip()) stderr.close() if retval: raise CommandException('%s failed!' % cmd) return retval def runMany(self, cmds): for cmd in cmds: self.run(cmd) class CollectionBuilder: """Builds the collection of releases and extracts the products from them.""" def __init__(self, options): self.releases = options.getReleases() self.runner = CommandRunner() self.old_pwd = os.getcwd() def run(self): self.buildReleases() def buildReleases(self): """Iterate through and build all releases in collection.""" for name in self.releases: self.release = self.releases[name] self.name = name self.prepareRelease() self.buildRelease() self.extractProducts() self.finishRelease() def prepareRelease(self): """This prepares the release directory. The directory may already exist, so care must be taken to be robust. """ # XXX could be extended to handle non-subversion url's assert self.release['url'][:4] == 'svn:' url = self.release['url'][4:] try: self.runner.run("[ `svn info %s | grep -c '%s'` != 0 ]" % (self.name, url)) except CommandException: logging.info('Removing old %s as not for the correct repository' % self.name) self.runner.run('rm -rf %s' % self.name) self.runner.run('svn co %s %s' % (url, self.name)) os.chdir(self.name) def extractProducts(self): """Move the products to where they are supposed to go.""" # XXX should probably move the products temporarily somewhere instead # of directly publishing them targetdir = self.release['targetdir'] if not os.path.isabs(targetdir): targetdir = os.path.join(self.old_pwd, targetdir) for p in self.release['products']: if p[1]: target = os.path.join(targetdir, p[1]) if not self.release['overwrite'] and os.path.exists(target): raise IOError('Cowardly refusing to overwrite file %s' % target) else: target = targetdir assert os.path.exists(target) base = os.path.dirname(target) if not os.path.exists(base): os.makedirs(base) assert os.path.isdir(base) self.runner.run('mv %s %s' % (p[0], target)) def finishRelease(self): """Any program state cleanup required.""" os.chdir(self.old_pwd) def buildRelease(self): self.runner.runMany(self.release['clean_command']) self.runner.runMany(self.release['build_command']) self.runner.runMany(self.release['test_command']) def parse_args(argv): parser = optparse.OptionParser( usage="usage: %prog [options] collection version") parser.add_option("-l", "--loglevel", dest="loglevel", help="set the loglevel, see the logging module for " "possible values") options, args = parser.parse_args(argv) if len(args) != 3: parser.error("Must have 2 arguments, you want --help") options.collection = args[1] options.version = args[2] return options def main(argv): cloptions = parse_args(argv) options = Options() options.loadConfig() # override some options with command line values options.collection_to_build = cloptions.collection options.substitutions['@VERSION@'] = cloptions.version if cloptions.loglevel: options.loglevel = cloptions.loglevel else: options.loglevel = options.config.loglevel # set up the logger logging.basicConfig(level=getattr(logging, options.loglevel), format='%(levelname)s: %(message)s') builder = CollectionBuilder(options) builder.run() if __name__ == '__main__': main(sys.argv)