dak/generate_packages_sources.py

   1 #!/usr/bin/env python
   2
   3 """ Generate Packages/Sources files
   4
   5 @contact: Debian FTPMaster <ftpmaster@debian.org>
   6 @copyright: 2000, 2001, 2002, 2006  James Troup <james@nocrew.org>
   7 @copyright: 2009  Mark Hymers <mhy@debian.org>
   8 @copyright: 2010  Joerg Jaspert <joerg@debian.org>
   9 @license: GNU General Public License version 2 or later
  10
  11 """
  12
  13 # This program is free software; you can redistribute it and/or modify
  14 # it under the terms of the GNU General Public License as published by
  15 # the Free Software Foundation; either version 2 of the License, or
  16 # (at your option) any later version.
  17
  18 # This program is distributed in the hope that it will be useful,
  19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 # GNU General Public License for more details.
  22
  23 # You should have received a copy of the GNU General Public License
  24 # along with this program; if not, write to the Free Software
  25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26
  27 ################################################################################
  28
  29 import os
  30 import os.path
  31 import sys
  32 import apt_pkg
  33 from tempfile import mkstemp, mkdtemp
  34 import commands
  35 from multiprocessing import Pool, TimeoutError
  36
  37 from daklib import daklog
  38 from daklib.dbconn import *
  39 from daklib.config import Config
  40
  41 ################################################################################
  42
  43 Options = None                 #: Commandline arguments parsed into this
  44 Logger = None                  #: Our logging object
  45 results = []                   #: Results of the subprocesses
  46
  47 ################################################################################
  48
  49 def usage (exit_code=0):
  50     print """Usage: dak generate-packages-sources [OPTIONS]
  51 Generate the Packages/Sources files
  52
  53   -s, --suite=SUITE(s)       process this suite
  54                              Default: All suites not marked 'untouchable'
  55   -f, --force                Allow processing of untouchable suites
  56                              CAREFUL: Only to be used at point release time!
  57   -h, --help                 show this help and exit
  58
  59 SUITE can be a space seperated list, e.g.
  60    --suite=unstable testing
  61   """
  62
  63     sys.exit(exit_code)
  64
  65 ################################################################################
  66
  67 def generate_packages_sources(arch, suite, tmppath):
  68     """
  69     Generate Packages/Sources files with apt-ftparchive for the given suite/arch
  70
  71     @type suite: string
  72     @param suite: Suite name
  73
  74     @type arch: string
  75     @param arch: Architecture name
  76
  77     @type tmppath: string
  78     @param tmppath: The temporary path to work ing
  79     """
  80
  81     DAILY_APT_CONF="""
  82 Dir
  83 {
  84    ArchiveDir "/srv/ftp-master.debian.org/ftp/";
  85    OverrideDir "/srv/ftp-master.debian.org/scripts/override/";
  86    CacheDir "/srv/ftp-master.debian.org/database/";
  87 };
  88
  89 Default
  90 {
  91    Packages::Compress "bzip2 gzip";
  92    Sources::Compress "bzip2 gzip";
  93    Contents::Compress "gzip";
  94    DeLinkLimit 0;
  95    MaxContentsChange 25000;
  96    FileMode 0664;
  97 }
  98
  99 TreeDefault
 100 {
 101    Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top";
 102 };
 103
 104 """
 105
 106     apt_trees={}
 107     apt_trees["di"]={}
 108     apt_trees["testing"]="""
 109 tree "dists/testing"
 110 {
 111    FakeDI "dists/unstable";
 112    FileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list";
 113    SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list";
 114    Sections "main contrib non-free";
 115    Architectures "%(arch)s";
 116    BinOverride "override.wheezy.$(SECTION)";
 117    ExtraOverride "override.wheezy.extra.$(SECTION)";
 118    SrcOverride "override.wheezy.$(SECTION).src";
 119 };
 120 """
 121
 122     apt_trees["squeeze-updates"]="""
 123 tree "dists/squeeze-updates"
 124 {
 125    FileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_binary-$(ARCH).list";
 126    SourceFileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_source.list";
 127    Sections "main contrib non-free";
 128    Architectures "%(arch)s";
 129    BinOverride "override.squeeze.$(SECTION)";
 130    ExtraOverride "override.squeeze.extra.$(SECTION)";
 131    SrcOverride "override.squeeze.$(SECTION).src";
 132    Contents " ";
 133 };
 134 """
 135
 136     apt_trees["di"]["testing"]="""
 137 tree "dists/testing/main"
 138 {
 139    FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list";
 140    Sections "debian-installer";
 141    Architectures "%(arch)s";
 142    BinOverride "override.wheezy.main.$(SECTION)";
 143    SrcOverride "override.wheezy.main.src";
 144    BinCacheDB "packages-debian-installer-$(ARCH).db";
 145    Packages::Extensions ".udeb";
 146    %(contentsline)s
 147 };
 148
 149 tree "dists/testing/non-free"
 150 {
 151    FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list";
 152    Sections "debian-installer";
 153    Architectures "%(arch)s";
 154    BinOverride "override.wheezy.main.$(SECTION)";
 155    SrcOverride "override.wheezy.main.src";
 156    BinCacheDB "packages-debian-installer-$(ARCH).db";
 157    Packages::Extensions ".udeb";
 158    %(contentsline)s
 159 };
 160 """
 161
 162     apt_trees["unstable"]="""
 163 tree "dists/unstable"
 164 {
 165    FileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_binary-$(ARCH).list";
 166    SourceFileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_source.list";
 167    Sections "main contrib non-free";
 168    Architectures "%(arch)s";
 169    BinOverride "override.sid.$(SECTION)";
 170    ExtraOverride "override.sid.extra.$(SECTION)";
 171    SrcOverride "override.sid.$(SECTION).src";
 172 };
 173 """
 174     apt_trees["di"]["unstable"]="""
 175 tree "dists/unstable/main"
 176 {
 177    FileList "/srv/ftp-master.debian.org/database/dists/unstable_main_$(SECTION)_binary-$(ARCH).list";
 178    Sections "debian-installer";
 179    Architectures "%(arch)s";
 180    BinOverride "override.sid.main.$(SECTION)";
 181    SrcOverride "override.sid.main.src";
 182    BinCacheDB "packages-debian-installer-$(ARCH).db";
 183    Packages::Extensions ".udeb";
 184    %(contentsline)s
 185 };
 186
 187 tree "dists/unstable/non-free"
 188 {
 189    FileList "/srv/ftp-master.debian.org/database/dists/unstable_non-free_$(SECTION)_binary-$(ARCH).list";
 190    Sections "debian-installer";
 191    Architectures "%(arch)s";
 192    BinOverride "override.sid.main.$(SECTION)";
 193    SrcOverride "override.sid.main.src";
 194    BinCacheDB "packages-debian-installer-$(ARCH).db";
 195    Packages::Extensions ".udeb";
 196    %(contentsline)s
 197 };
 198 """
 199
 200     apt_trees["experimental"]="""
 201 tree "dists/experimental"
 202 {
 203    FileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_binary-$(ARCH).list";
 204    SourceFileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_source.list";
 205    Sections "main contrib non-free";
 206    Architectures "%(arch)s";
 207    BinOverride "override.sid.$(SECTION)";
 208    SrcOverride "override.sid.$(SECTION).src";
 209 };
 210 """
 211     apt_trees["di"]["experimental"]="""
 212 tree "dists/experimental/main"
 213 {
 214    FileList "/srv/ftp-master.debian.org/database/dists/experimental_main_$(SECTION)_binary-$(ARCH).list";
 215    Sections "debian-installer";
 216    Architectures "%(arch)s";
 217    BinOverride "override.sid.main.$(SECTION)";
 218    SrcOverride "override.sid.main.src";
 219    BinCacheDB "packages-debian-installer-$(ARCH).db";
 220    Packages::Extensions ".udeb";
 221    %(contentsline)s
 222 };
 223
 224 tree "dists/experimental/non-free"
 225 {
 226    FileList "/srv/ftp-master.debian.org/database/dists/experimental_non-free_$(SECTION)_binary-$(ARCH).list";
 227    Sections "debian-installer";
 228    Architectures "%(arch)s";
 229    BinOverride "override.sid.main.$(SECTION)";
 230    SrcOverride "override.sid.main.src";
 231    BinCacheDB "packages-debian-installer-$(ARCH).db";
 232    Packages::Extensions ".udeb";
 233    %(contentsline)s
 234 };
 235 """
 236
 237     apt_trees["testing-proposed-updates"]="""
 238 tree "dists/testing-proposed-updates"
 239 {
 240    FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list";
 241    SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list";
 242    Sections "main contrib non-free";
 243    Architectures "%(arch)s";
 244    BinOverride "override.wheezy.$(SECTION)";
 245    ExtraOverride "override.wheezy.extra.$(SECTION)";
 246    SrcOverride "override.wheezy.$(SECTION).src";
 247    Contents " ";
 248 };
 249 """
 250     apt_trees["di"]["testing-proposed-updates"]="""
 251 tree "dists/testing-proposed-updates/main"
 252 {
 253    FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
 254    Sections "debian-installer";
 255    Architectures "%(arch)s";
 256    BinOverride "override.wheezy.main.$(SECTION)";
 257    SrcOverride "override.wheezy.main.src";
 258    BinCacheDB "packages-debian-installer-$(ARCH).db";
 259    Packages::Extensions ".udeb";
 260    Contents " ";
 261 };
 262 """
 263
 264     apt_trees["proposed-updates"]="""
 265 tree "dists/proposed-updates"
 266 {
 267    FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list";
 268    SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list";
 269    Sections "main contrib non-free";
 270    Architectures "%(arch)s";
 271    BinOverride "override.squeeze.$(SECTION)";
 272    ExtraOverride "override.squeeze.extra.$(SECTION)";
 273    SrcOverride "override.squeeze.$(SECTION).src";
 274    Contents " ";
 275 };
 276 """
 277     apt_trees["di"]["proposed-updates"]="""
 278 tree "dists/proposed-updates/main"
 279 {
 280    FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
 281    Sections "debian-installer";
 282    Architectures "%(arch)s";
 283    BinOverride "override.squeeze.main.$(SECTION)";
 284    SrcOverride "override.squeeze.main.src";
 285    BinCacheDB "packages-debian-installer-$(ARCH).db";
 286    Packages::Extensions ".udeb";
 287    Contents " ";
 288 };
 289 """
 290     apt_trees["oldstable-proposed-updates"]="""
 291 tree "dists/oldstable-proposed-updates"
 292 {
 293    FileList "/srv/ftp-master.debian.org/database/dists/oldstable-proposed-updates_$(SECTION)_binary-$(ARCH).list";
 294    SourceFileList "/srv/ftp-master.debian.org/database/dists/oldstable-proposed-updates_$(SECTION)_source.list";
 295    Sections "main contrib non-free";
 296    Architectures "%(arch)s";
 297    BinOverride "override.lenny.$(SECTION)";
 298    ExtraOverride "override.lenny.extra.$(SECTION)";
 299    SrcOverride "override.lenny.$(SECTION).src";
 300    Contents " ";
 301 };
 302 """
 303     apt_trees["di"]["oldstable-proposed-updates"]="""
 304 tree "dists/oldstable-proposed-updates/main"
 305 {
 306    FileList "/srv/ftp-master.debian.org/database/dists/oldstable-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
 307    Sections "debian-installer";
 308    Architectures "%(arch)s";
 309    BinOverride "override.lenny.main.$(SECTION)";
 310    SrcOverride "override.lenny.main.src";
 311    BinCacheDB "packages-debian-installer-$(ARCH).db";
 312    Packages::Extensions ".udeb";
 313    Contents " ";
 314 };
 315 """
 316
 317     cnf = Config()
 318     try:
 319         # Write apt.conf
 320         (ac_fd, ac_name) = mkstemp(dir=tmppath, suffix=suite, prefix=arch)
 321         os.write(ac_fd, DAILY_APT_CONF)
 322         # here we want to generate the tree entries
 323         os.write(ac_fd, apt_trees[suite] % {'arch': arch})
 324         # this special casing needs to go away, but this whole thing may just want an
 325         # aptconfig class anyways
 326         if arch != 'source':
 327             if arch == 'hurd-i386' and suite == 'experimental':
 328                 pass
 329             elif apt_trees["di"].has_key(suite):
 330                 if arch == "amd64":
 331                     os.write(ac_fd, apt_trees["di"][suite] %
 332                              {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'})
 333                 else:
 334                     os.write(ac_fd, apt_trees["di"][suite] % {'arch': arch, 'contentsline': ''})
 335         os.close(ac_fd)
 336
 337         print "Going to run apt-ftparchive for %s/%s" % (arch, suite)
 338         # Run apt-ftparchive generate
 339         # We dont want to add a -q or -qq here, this output should go into our logs, sometimes
 340         # it has errormessages we like to see
 341         os.environ['GZIP'] = '--rsyncable'
 342         os.chdir(tmppath)
 343         (result, output) = commands.getstatusoutput('apt-ftparchive -o APT::FTPArchive::Contents=off generate %s' % os.path.basename(ac_name))
 344         sn="a-f %s,%s: " % (suite, arch)
 345         print sn + output.replace('\n', '\n%s' % (sn))
 346         return result
 347
 348     # Clean up any left behind files
 349     finally:
 350         if ac_fd:
 351             try:
 352                 os.close(ac_fd)
 353             except OSError:
 354                 pass
 355
 356         if ac_name:
 357             try:
 358                 os.unlink(ac_name)
 359             except OSError:
 360                 pass
 361
 362 def sname(arch):
 363     return arch.arch_string
 364
 365 def get_result(arg):
 366     global results
 367     if arg:
 368         results.append(arg)
 369
 370 ########################################################################
 371 ########################################################################
 372
 373 def main ():
 374     global Options, Logger, results
 375
 376     cnf = Config()
 377
 378     for i in ["Help", "Suite", "Force"]:
 379         if not cnf.has_key("Generate-Packages-Sources::Options::%s" % (i)):
 380             cnf["Generate-Packages-Sources::Options::%s" % (i)] = ""
 381
 382     Arguments = [('h',"help","Generate-Packages-Sources::Options::Help"),
 383                  ('s',"suite","Generate-Packages-Sources::Options::Suite"),
 384                  ('f',"force","Generate-Packages-Sources::Options::Force")]
 385
 386     suite_names = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
 387     Options = cnf.SubTree("Generate-Packages-Sources::Options")
 388
 389     if Options["Help"]:
 390         usage()
 391
 392     Logger = daklog.Logger(cnf, 'generate-packages-sources')
 393
 394     session = DBConn().session()
 395
 396     if Options["Suite"]:
 397         # Something here
 398         suites = []
 399         for s in suite_names:
 400             suite = get_suite(s.lower(), session)
 401             if suite:
 402                 suites.append(suite)
 403             else:
 404                 print "cannot find suite %s" % s
 405                 Logger.log(['cannot find suite %s' % s])
 406     else:
 407         suites=session.query(Suite).filter(Suite.untouchable == False).all()
 408
 409     startdir = os.getcwd()
 410     os.chdir(cnf["Dir::TempPath"])
 411
 412     broken=[]
 413     # For each given suite, each architecture, run one apt-ftparchive
 414     for s in suites:
 415         results=[]
 416         # Setup a multiprocessing Pool. As many workers as we have CPU cores.
 417         pool = Pool()
 418         arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=True, session=session)
 419         Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))])
 420         for a in arch_list:
 421             pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]), callback=get_result)
 422
 423         # No more work will be added to our pool, close it and then wait for all to finish
 424         pool.close()
 425         pool.join()
 426
 427     if len(results) > 0:
 428         Logger.log(['Trouble, something with a-f broke, resultcodes: %s' % (results)])
 429         print "Trouble, something with a-f broke, resultcodes: %s" % (results)
 430         sys.exit(1)
 431
 432     os.chdir(startdir)
 433     # this script doesn't change the database
 434     session.close()
 435     Logger.close()
 436
 437 #######################################################################################
 438
 439 if __name__ == '__main__':
 440     main()