@copyright: 2000, 2001, 2002, 2006 James Troup <james@nocrew.org>
@copyright: 2009 Mark Hymers <mhy@debian.org>
@copyright: 2010 Joerg Jaspert <joerg@debian.org>
+@license: GNU General Public License version 2 or later
"""
import os
import os.path
-import stat
import sys
-from datetime import datetime
import apt_pkg
+from tempfile import mkstemp, mkdtemp
+import commands
+from multiprocessing import Pool, TimeoutError
from daklib import daklog
from daklib.dbconn import *
################################################################################
-Options = None
-Logger = None
+Options = None #: Commandline arguments parsed into this
+Logger = None #: Our logging object
+results = [] #: Results of the subprocesses
################################################################################
################################################################################
+def generate_packages_sources(arch, suite, tmppath):
+ """
+ Generate Packages/Sources files with apt-ftparchive for the given suite/arch
+
+ @type suite: string
+ @param suite: Suite name
+
+ @type arch: string
+ @param arch: Architecture name
+
+ @type tmppath: string
+ @param tmppath: The temporary path to work ing
+ """
+
+ DAILY_APT_CONF="""
+Dir
+{
+ ArchiveDir "/srv/ftp-master.debian.org/ftp/";
+ OverrideDir "/srv/ftp-master.debian.org/scripts/override/";
+ CacheDir "/srv/ftp-master.debian.org/database/";
+};
+
+Default
+{
+ Packages::Compress "bzip2 gzip";
+ Sources::Compress "bzip2 gzip";
+ Contents::Compress "gzip";
+ DeLinkLimit 0;
+ MaxContentsChange 25000;
+ FileMode 0664;
+}
+
+TreeDefault
+{
+ Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top";
+};
+
+"""
+
+ apt_trees={}
+ apt_trees["di"]={}
+
+ apt_trees["oldstable"]="""
+tree "dists/oldstable"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/oldstable_$(SECTION)_binary-$(ARCH).list";
+ SourceFileList "/srv/ftp-master.debian.org/database/dists/oldstable_$(SECTION)_source.list";
+ Sections "main contrib non-free";
+ Architectures "%(arch)s";
+ BinOverride "override.squeeze.$(SECTION)";
+ ExtraOverride "override.squeeze.extra.$(SECTION)";
+ SrcOverride "override.squeeze.$(SECTION).src";
+};
+"""
+
+ apt_trees["di"]["oldstable"]="""
+tree "dists/oldstable/main"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/oldstable_main_$(SECTION)_binary-$(ARCH).list";
+ Sections "debian-installer";
+ Architectures "%(arch)s";
+ BinOverride "override.squeeze.main.$(SECTION)";
+ SrcOverride "override.squeeze.main.src";
+ BinCacheDB "packages-debian-installer-$(ARCH).db";
+ Packages::Extensions ".udeb";
+ %(contentsline)s
+};
+
+tree "dists/oldstable/non-free"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/oldstable_non-free_$(SECTION)_binary-$(ARCH).list";
+ Sections "debian-installer";
+ Architectures "%(arch)s";
+ BinOverride "override.squeeze.main.$(SECTION)";
+ SrcOverride "override.squeeze.main.src";
+ BinCacheDB "packages-debian-installer-$(ARCH).db";
+ Packages::Extensions ".udeb";
+ %(contentsline)s
+};
+"""
+
+
+ cnf = Config()
+ try:
+ # Write apt.conf
+ (ac_fd, ac_name) = mkstemp(dir=tmppath, suffix=suite, prefix=arch)
+ os.write(ac_fd, DAILY_APT_CONF)
+ # here we want to generate the tree entries
+ os.write(ac_fd, apt_trees[suite] % {'arch': arch})
+ # this special casing needs to go away, but this whole thing may just want an
+ # aptconfig class anyways
+ if arch != 'source':
+ if arch == 'hurd-i386' and suite == 'experimental':
+ pass
+ elif apt_trees["di"].has_key(suite):
+ if arch == "amd64":
+ os.write(ac_fd, apt_trees["di"][suite] %
+ {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'})
+ else:
+ os.write(ac_fd, apt_trees["di"][suite] % {'arch': arch, 'contentsline': ''})
+ os.close(ac_fd)
+
+ print "Going to run apt-ftparchive for %s/%s" % (arch, suite)
+ # Run apt-ftparchive generate
+ # We dont want to add a -q or -qq here, this output should go into our logs, sometimes
+ # it has errormessages we like to see
+ os.environ['GZIP'] = '--rsyncable'
+ os.chdir(tmppath)
+ (result, output) = commands.getstatusoutput('apt-ftparchive -o APT::FTPArchive::Contents=off -o APT::FTPArchive::SHA512=off generate %s' % os.path.basename(ac_name))
+ sn="a-f %s,%s: " % (suite, arch)
+ print sn + output.replace('\n', '\n%s' % (sn))
+ return result
+
+ # Clean up any left behind files
+ finally:
+ if ac_fd:
+ try:
+ os.close(ac_fd)
+ except OSError:
+ pass
+
+ if ac_name:
+ try:
+ os.unlink(ac_name)
+ except OSError:
+ pass
+
+def sname(arch):
+ return arch.arch_string
+
+def get_result(arg):
+ global results
+ if arg:
+ results.append(arg)
+
+########################################################################
+########################################################################
+
def main ():
- global Options, Logger
+ global Options, Logger, results
cnf = Config()
('s',"suite","Generate-Packages-Sources::Options::Suite"),
('f',"force","Generate-Packages-Sources::Options::Force")]
- suite_names = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
- Options = cnf.SubTree("Generate-Packages-Sources::Options")
+ suite_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.subtree("Generate-Packages-Sources::Options")
if Options["Help"]:
usage()
-# Logger = daklog.Logger(cnf, 'generate-packages-sources')
+ Logger = daklog.Logger('generate-packages-sources')
session = DBConn().session()
suites.append(suite)
else:
print "cannot find suite %s" % s
-# Logger.log(['cannot find suite %s' % s])
+ Logger.log(['cannot find suite %s' % s])
else:
- suites=session.query(Suite).filter(Suite.untouchable == 'false').all()
+ suites=session.query(Suite).filter(Suite.untouchable == False).all()
+ startdir = os.getcwd()
+ os.chdir(cnf["Dir::TempPath"])
+
+ broken=[]
+ # For each given suite, each architecture, run one apt-ftparchive
for s in suites:
- print "Working on: %s" % (s.suite_name)
- sys.exit(0)
- # For each given suite, look up object and call generate-filelist
- for s in suites:
- Logger.log(['generating filelist for %s' % (s.suite_name)])
- s.generate_filelist(Logger)
+ results=[]
+ # Setup a multiprocessing Pool. As many workers as we have CPU cores.
+ pool = Pool()
+ arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=False, session=session)
+ Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))])
+ for a in arch_list:
+ pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]), callback=get_result)
+
+ # No more work will be added to our pool, close it and then wait for all to finish
+ pool.close()
+ pool.join()
+
+ if len(results) > 0:
+ Logger.log(['Trouble, something with a-f broke, resultcodes: %s' % (results)])
+ print "Trouble, something with a-f broke, resultcodes: %s" % (results)
+ sys.exit(1)
-# Logger.close()
+ os.chdir(startdir)
+ # this script doesn't change the database
+ session.close()
+ Logger.close()
#######################################################################################
projects / dak.git / blobdiff