X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=dak%2Fgenerate_packages_sources.py;h=3e01da2347d8dbe26043e0a3000a869f53d1a651;hb=013bd28f8a38603851e2bd01841fb9ee8af28d8b;hp=79db33ba57d0cc88d70d02d195514fd6fb2b17eb;hpb=f6a7dfc1e5cac520c07a0ca8bf498695d0df2356;p=dak.git diff --git a/dak/generate_packages_sources.py b/dak/generate_packages_sources.py index 79db33ba..3e01da23 100755 --- a/dak/generate_packages_sources.py +++ b/dak/generate_packages_sources.py @@ -6,6 +6,7 @@ @copyright: 2000, 2001, 2002, 2006 James Troup @copyright: 2009 Mark Hymers @copyright: 2010 Joerg Jaspert +@license: GNU General Public License version 2 or later """ @@ -27,20 +28,21 @@ import os import os.path -import stat import sys -from datetime import datetime import apt_pkg +from tempfile import mkstemp, mkdtemp +import commands +from multiprocessing import Pool, TimeoutError from daklib import daklog from daklib.dbconn import * from daklib.config import Config -from daklib.threadpool import ThreadPool ################################################################################ -Options = None -Logger = None +Options = None #: Commandline arguments parsed into this +Logger = None #: Our logging object +results = [] #: Results of the subprocesses ################################################################################ @@ -62,8 +64,146 @@ SUITE can be a space seperated list, e.g. ################################################################################ +def generate_packages_sources(arch, suite, tmppath): + """ + Generate Packages/Sources files with apt-ftparchive for the given suite/arch + + @type suite: string + @param suite: Suite name + + @type arch: string + @param arch: Architecture name + + @type tmppath: string + @param tmppath: The temporary path to work ing + """ + + DAILY_APT_CONF=""" +Dir +{ + ArchiveDir "/srv/ftp-master.debian.org/ftp/"; + OverrideDir "/srv/ftp-master.debian.org/scripts/override/"; + CacheDir "/srv/ftp-master.debian.org/database/"; +}; + +Default +{ + Packages::Compress "bzip2 gzip"; + Sources::Compress "bzip2 gzip"; + Contents::Compress "gzip"; + DeLinkLimit 0; + MaxContentsChange 25000; + FileMode 0664; +} + +TreeDefault +{ + Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top"; +}; + +""" + + apt_trees={} + apt_trees["di"]={} + + apt_trees["oldstable"]=""" +tree "dists/oldstable" +{ + FileList "/srv/ftp-master.debian.org/database/dists/oldstable_$(SECTION)_binary-$(ARCH).list"; + SourceFileList "/srv/ftp-master.debian.org/database/dists/oldstable_$(SECTION)_source.list"; + Sections "main contrib non-free"; + Architectures "%(arch)s"; + BinOverride "override.squeeze.$(SECTION)"; + ExtraOverride "override.squeeze.extra.$(SECTION)"; + SrcOverride "override.squeeze.$(SECTION).src"; +}; +""" + + apt_trees["di"]["oldstable"]=""" +tree "dists/oldstable/main" +{ + FileList "/srv/ftp-master.debian.org/database/dists/oldstable_main_$(SECTION)_binary-$(ARCH).list"; + Sections "debian-installer"; + Architectures "%(arch)s"; + BinOverride "override.squeeze.main.$(SECTION)"; + SrcOverride "override.squeeze.main.src"; + BinCacheDB "packages-debian-installer-$(ARCH).db"; + Packages::Extensions ".udeb"; + %(contentsline)s +}; + +tree "dists/oldstable/non-free" +{ + FileList "/srv/ftp-master.debian.org/database/dists/oldstable_non-free_$(SECTION)_binary-$(ARCH).list"; + Sections "debian-installer"; + Architectures "%(arch)s"; + BinOverride "override.squeeze.main.$(SECTION)"; + SrcOverride "override.squeeze.main.src"; + BinCacheDB "packages-debian-installer-$(ARCH).db"; + Packages::Extensions ".udeb"; + %(contentsline)s +}; +""" + + + cnf = Config() + try: + # Write apt.conf + (ac_fd, ac_name) = mkstemp(dir=tmppath, suffix=suite, prefix=arch) + os.write(ac_fd, DAILY_APT_CONF) + # here we want to generate the tree entries + os.write(ac_fd, apt_trees[suite] % {'arch': arch}) + # this special casing needs to go away, but this whole thing may just want an + # aptconfig class anyways + if arch != 'source': + if arch == 'hurd-i386' and suite == 'experimental': + pass + elif apt_trees["di"].has_key(suite): + if arch == "amd64": + os.write(ac_fd, apt_trees["di"][suite] % + {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'}) + else: + os.write(ac_fd, apt_trees["di"][suite] % {'arch': arch, 'contentsline': ''}) + os.close(ac_fd) + + print "Going to run apt-ftparchive for %s/%s" % (arch, suite) + # Run apt-ftparchive generate + # We dont want to add a -q or -qq here, this output should go into our logs, sometimes + # it has errormessages we like to see + os.environ['GZIP'] = '--rsyncable' + os.chdir(tmppath) + (result, output) = commands.getstatusoutput('apt-ftparchive -o APT::FTPArchive::Contents=off -o APT::FTPArchive::SHA512=off generate %s' % os.path.basename(ac_name)) + sn="a-f %s,%s: " % (suite, arch) + print sn + output.replace('\n', '\n%s' % (sn)) + return result + + # Clean up any left behind files + finally: + if ac_fd: + try: + os.close(ac_fd) + except OSError: + pass + + if ac_name: + try: + os.unlink(ac_name) + except OSError: + pass + +def sname(arch): + return arch.arch_string + +def get_result(arg): + global results + if arg: + results.append(arg) + +######################################################################## +######################################################################## + def main (): - global Options, Logger + global Options, Logger, results cnf = Config() @@ -75,13 +215,13 @@ def main (): ('s',"suite","Generate-Packages-Sources::Options::Suite"), ('f',"force","Generate-Packages-Sources::Options::Force")] - suite_names = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv) - Options = cnf.SubTree("Generate-Packages-Sources::Options") + suite_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) + Options = cnf.subtree("Generate-Packages-Sources::Options") if Options["Help"]: usage() - Logger = daklog.Logger(cnf, 'generate-packages-sources') + Logger = daklog.Logger('generate-packages-sources') session = DBConn().session() @@ -98,16 +238,30 @@ def main (): else: suites=session.query(Suite).filter(Suite.untouchable == False).all() - threadpool = ThreadPool() + startdir = os.getcwd() + os.chdir(cnf["Dir::TempPath"]) + + broken=[] # For each given suite, each architecture, run one apt-ftparchive for s in suites: + results=[] + # Setup a multiprocessing Pool. As many workers as we have CPU cores. + pool = Pool() arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=False, session=session) + Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))]) for a in arch_list: - Logger.log(['generating output for Suite %s, Architecture %s' % (s.suite_name, a.arch_string)]) - print 'generating output for Suite %s, Architecture %s' % (s.suite_name, a.arch_string) - threadpool.queueTask(s.generate_packages_sources, (s.suite_name, a.arch_string)) + pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]), callback=get_result) + + # No more work will be added to our pool, close it and then wait for all to finish + pool.close() + pool.join() + + if len(results) > 0: + Logger.log(['Trouble, something with a-f broke, resultcodes: %s' % (results)]) + print "Trouble, something with a-f broke, resultcodes: %s" % (results) + sys.exit(1) - threadpool.joinAll() + os.chdir(startdir) # this script doesn't change the database session.close() Logger.close()