daklib/archive.py

   1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
   2 #
   3 # This program is free software; you can redistribute it and/or modify
   4 # it under the terms of the GNU General Public License as published by
   5 # the Free Software Foundation; either version 2 of the License, or
   6 # (at your option) any later version.
   7 #
   8 # This program is distributed in the hope that it will be useful,
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 # GNU General Public License for more details.
  12 #
  13 # You should have received a copy of the GNU General Public License along
  14 # with this program; if not, write to the Free Software Foundation, Inc.,
  15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16
  17 """module to manipulate the archive
  18
  19 This module provides classes to manipulate the archive.
  20 """
  21
  22 from daklib.dbconn import *
  23 import daklib.checks as checks
  24 from daklib.config import Config
  25 import daklib.upload as upload
  26 import daklib.utils as utils
  27 from daklib.fstransactions import FilesystemTransaction
  28 from daklib.regexes import re_changelog_versions, re_bin_only_nmu
  29 import daklib.daksubprocess
  30
  31 import apt_pkg
  32 from datetime import datetime
  33 import os
  34 import shutil
  35 from sqlalchemy.orm.exc import NoResultFound
  36 from sqlalchemy.orm import object_session
  37 import sqlalchemy.exc
  38 import tempfile
  39 import traceback
  40
  41 class ArchiveException(Exception):
  42     pass
  43
  44 class HashMismatchException(ArchiveException):
  45     pass
  46
  47 class ArchiveTransaction(object):
  48     """manipulate the archive in a transaction
  49     """
  50     def __init__(self):
  51         self.fs = FilesystemTransaction()
  52         self.session = DBConn().session()
  53
  54     def get_file(self, hashed_file, source_name, check_hashes=True):
  55         """Look for file C{hashed_file} in database
  56
  57         @type  hashed_file: L{daklib.upload.HashedFile}
  58         @param hashed_file: file to look for in the database
  59
  60         @type  source_name: str
  61         @param source_name: source package name
  62
  63         @type  check_hashes: bool
  64         @param check_hashes: check size and hashes match
  65
  66         @raise KeyError: file was not found in the database
  67         @raise HashMismatchException: hash mismatch
  68
  69         @rtype:  L{daklib.dbconn.PoolFile}
  70         @return: database entry for the file
  71         """
  72         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
  73         try:
  74             poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
  75             if check_hashes and (poolfile.filesize != hashed_file.size
  76                                  or poolfile.md5sum != hashed_file.md5sum
  77                                  or poolfile.sha1sum != hashed_file.sha1sum
  78                                  or poolfile.sha256sum != hashed_file.sha256sum):
  79                 raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
  80             return poolfile
  81         except NoResultFound:
  82             raise KeyError('{0} not found in database.'.format(poolname))
  83
  84     def _install_file(self, directory, hashed_file, archive, component, source_name):
  85         """Install a file
  86
  87         Will not give an error when the file is already present.
  88
  89         @rtype:  L{daklib.dbconn.PoolFile}
  90         @return: database object for the new file
  91         """
  92         session = self.session
  93
  94         poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
  95         try:
  96             poolfile = self.get_file(hashed_file, source_name)
  97         except KeyError:
  98             poolfile = PoolFile(filename=poolname, filesize=hashed_file.size)
  99             poolfile.md5sum = hashed_file.md5sum
 100             poolfile.sha1sum = hashed_file.sha1sum
 101             poolfile.sha256sum = hashed_file.sha256sum
 102             session.add(poolfile)
 103             session.flush()
 104
 105         try:
 106             session.query(ArchiveFile).filter_by(archive=archive, component=component, file=poolfile).one()
 107         except NoResultFound:
 108             archive_file = ArchiveFile(archive, component, poolfile)
 109             session.add(archive_file)
 110             session.flush()
 111
 112             path = os.path.join(archive.path, 'pool', component.component_name, poolname)
 113             hashed_file_path = os.path.join(directory, hashed_file.filename)
 114             self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
 115
 116         return poolfile
 117
 118     def install_binary(self, directory, binary, suite, component, allow_tainted=False, fingerprint=None, source_suites=None, extra_source_archives=None):
 119         """Install a binary package
 120
 121         @type  directory: str
 122         @param directory: directory the binary package is located in
 123
 124         @type  binary: L{daklib.upload.Binary}
 125         @param binary: binary package to install
 126
 127         @type  suite: L{daklib.dbconn.Suite}
 128         @param suite: target suite
 129
 130         @type  component: L{daklib.dbconn.Component}
 131         @param component: target component
 132
 133         @type  allow_tainted: bool
 134         @param allow_tainted: allow to copy additional files from tainted archives
 135
 136         @type  fingerprint: L{daklib.dbconn.Fingerprint}
 137         @param fingerprint: optional fingerprint
 138
 139         @type  source_suites: SQLAlchemy subquery for C{daklib.dbconn.Suite} or C{True}
 140         @param source_suites: suites to copy the source from if they are not
 141                               in C{suite} or C{True} to allow copying from any
 142                               suite.
 143
 144         @type  extra_source_archives: list of L{daklib.dbconn.Archive}
 145         @param extra_source_archives: extra archives to copy Built-Using sources from
 146
 147         @rtype:  L{daklib.dbconn.DBBinary}
 148         @return: databse object for the new package
 149         """
 150         session = self.session
 151         control = binary.control
 152         maintainer = get_or_set_maintainer(control['Maintainer'], session)
 153         architecture = get_architecture(control['Architecture'], session)
 154
 155         (source_name, source_version) = binary.source
 156         source_query = session.query(DBSource).filter_by(source=source_name, version=source_version)
 157         source = source_query.filter(DBSource.suites.contains(suite)).first()
 158         if source is None:
 159             if source_suites != True:
 160                 source_query = source_query.join(DBSource.suites) \
 161                     .filter(Suite.suite_id == source_suites.c.id)
 162             source = source_query.first()
 163             if source is None:
 164                 raise ArchiveException('{0}: trying to install to {1}, but could not find source'.format(binary.hashed_file.filename, suite.suite_name))
 165             self.copy_source(source, suite, component)
 166
 167         db_file = self._install_file(directory, binary.hashed_file, suite.archive, component, source_name)
 168
 169         unique = dict(
 170             package=control['Package'],
 171             version=control['Version'],
 172             architecture=architecture,
 173             )
 174         rest = dict(
 175             source=source,
 176             maintainer=maintainer,
 177             poolfile=db_file,
 178             binarytype=binary.type,
 179             fingerprint=fingerprint,
 180             )
 181
 182         try:
 183             db_binary = session.query(DBBinary).filter_by(**unique).one()
 184             for key, value in rest.iteritems():
 185                 if getattr(db_binary, key) != value:
 186                     raise ArchiveException('{0}: Does not match binary in database.'.format(binary.hashed_file.filename))
 187         except NoResultFound:
 188             db_binary = DBBinary(**unique)
 189             for key, value in rest.iteritems():
 190                 setattr(db_binary, key, value)
 191             session.add(db_binary)
 192             session.flush()
 193             import_metadata_into_db(db_binary, session)
 194
 195             self._add_built_using(db_binary, binary.hashed_file.filename, control, suite, extra_archives=extra_source_archives)
 196
 197         if suite not in db_binary.suites:
 198             db_binary.suites.append(suite)
 199
 200         session.flush()
 201
 202         return db_binary
 203
 204     def _ensure_extra_source_exists(self, filename, source, archive, extra_archives=None):
 205         """ensure source exists in the given archive
 206
 207         This is intended to be used to check that Built-Using sources exist.
 208
 209         @type  filename: str
 210         @param filename: filename to use in error messages
 211
 212         @type  source: L{daklib.dbconn.DBSource}
 213         @param source: source to look for
 214
 215         @type  archive: L{daklib.dbconn.Archive}
 216         @param archive: archive to look in
 217
 218         @type  extra_archives: list of L{daklib.dbconn.Archive}
 219         @param extra_archives: list of archives to copy the source package from
 220                                if it is not yet present in C{archive}
 221         """
 222         session = self.session
 223         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile, archive=archive).first()
 224         if db_file is not None:
 225             return True
 226
 227         # Try to copy file from one extra archive
 228         if extra_archives is None:
 229             extra_archives = []
 230         db_file = session.query(ArchiveFile).filter_by(file=source.poolfile).filter(ArchiveFile.archive_id.in_([ a.archive_id for a in extra_archives])).first()
 231         if db_file is None:
 232             raise ArchiveException('{0}: Built-Using refers to package {1} (= {2}) not in target archive {3}.'.format(filename, source.source, source.version, archive.archive_name))
 233
 234         source_archive = db_file.archive
 235         for dsc_file in source.srcfiles:
 236             af = session.query(ArchiveFile).filter_by(file=dsc_file.poolfile, archive=source_archive, component=db_file.component).one()
 237             # We were given an explicit list of archives so it is okay to copy from tainted archives.
 238             self._copy_file(af.file, archive, db_file.component, allow_tainted=True)
 239
 240     def _add_built_using(self, db_binary, filename, control, suite, extra_archives=None):
 241         """Add Built-Using sources to C{db_binary.extra_sources}
 242         """
 243         session = self.session
 244         built_using = control.get('Built-Using', None)
 245
 246         if built_using is not None:
 247             for dep in apt_pkg.parse_depends(built_using):
 248                 assert len(dep) == 1, 'Alternatives are not allowed in Built-Using field'
 249                 bu_source_name, bu_source_version, comp = dep[0]
 250                 assert comp == '=', 'Built-Using must contain strict dependencies'
 251
 252                 bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first()
 253                 if bu_source is None:
 254                     raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version))
 255
 256                 self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives)
 257
 258                 db_binary.extra_sources.append(bu_source)
 259
 260     def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None):
 261         """Install a source package
 262
 263         @type  directory: str
 264         @param directory: directory the source package is located in
 265
 266         @type  source: L{daklib.upload.Source}
 267         @param source: source package to install
 268
 269         @type  suite: L{daklib.dbconn.Suite}
 270         @param suite: target suite
 271
 272         @type  component: L{daklib.dbconn.Component}
 273         @param component: target component
 274
 275         @type  changed_by: L{daklib.dbconn.Maintainer}
 276         @param changed_by: person who prepared this version of the package
 277
 278         @type  allow_tainted: bool
 279         @param allow_tainted: allow to copy additional files from tainted archives
 280
 281         @type  fingerprint: L{daklib.dbconn.Fingerprint}
 282         @param fingerprint: optional fingerprint
 283
 284         @rtype:  L{daklib.dbconn.DBSource}
 285         @return: database object for the new source
 286         """
 287         session = self.session
 288         archive = suite.archive
 289         control = source.dsc
 290         maintainer = get_or_set_maintainer(control['Maintainer'], session)
 291         source_name = control['Source']
 292
 293         ### Add source package to database
 294
 295         # We need to install the .dsc first as the DBSource object refers to it.
 296         db_file_dsc = self._install_file(directory, source._dsc_file, archive, component, source_name)
 297
 298         unique = dict(
 299             source=source_name,
 300             version=control['Version'],
 301             )
 302         rest = dict(
 303             maintainer=maintainer,
 304             changedby=changed_by,
 305             #install_date=datetime.now().date(),
 306             poolfile=db_file_dsc,
 307             fingerprint=fingerprint,
 308             dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'),
 309             )
 310
 311         created = False
 312         try:
 313             db_source = session.query(DBSource).filter_by(**unique).one()
 314             for key, value in rest.iteritems():
 315                 if getattr(db_source, key) != value:
 316                     raise ArchiveException('{0}: Does not match source in database.'.format(source._dsc_file.filename))
 317         except NoResultFound:
 318             created = True
 319             db_source = DBSource(**unique)
 320             for key, value in rest.iteritems():
 321                 setattr(db_source, key, value)
 322             # XXX: set as default in postgres?
 323             db_source.install_date = datetime.now().date()
 324             session.add(db_source)
 325             session.flush()
 326
 327             # Add .dsc file. Other files will be added later.
 328             db_dsc_file = DSCFile()
 329             db_dsc_file.source = db_source
 330             db_dsc_file.poolfile = db_file_dsc
 331             session.add(db_dsc_file)
 332             session.flush()
 333
 334         if suite in db_source.suites:
 335             return db_source
 336
 337         db_source.suites.append(suite)
 338
 339         if not created:
 340             for f in db_source.srcfiles:
 341                 self._copy_file(f.poolfile, archive, component, allow_tainted=allow_tainted)
 342             return db_source
 343
 344         ### Now add remaining files and copy them to the archive.
 345
 346         for hashed_file in source.files.itervalues():
 347             hashed_file_path = os.path.join(directory, hashed_file.filename)
 348             if os.path.exists(hashed_file_path):
 349                 db_file = self._install_file(directory, hashed_file, archive, component, source_name)
 350                 session.add(db_file)
 351             else:
 352                 db_file = self.get_file(hashed_file, source_name)
 353                 self._copy_file(db_file, archive, component, allow_tainted=allow_tainted)
 354
 355             db_dsc_file = DSCFile()
 356             db_dsc_file.source = db_source
 357             db_dsc_file.poolfile = db_file
 358             session.add(db_dsc_file)
 359
 360         session.flush()
 361
 362         # Importing is safe as we only arrive here when we did not find the source already installed earlier.
 363         import_metadata_into_db(db_source, session)
 364
 365         # Uploaders are the maintainer and co-maintainers from the Uploaders field
 366         db_source.uploaders.append(maintainer)
 367         if 'Uploaders' in control:
 368             from daklib.textutils import split_uploaders
 369             for u in split_uploaders(control['Uploaders']):
 370                 db_source.uploaders.append(get_or_set_maintainer(u, session))
 371         session.flush()
 372
 373         return db_source
 374
 375     def _copy_file(self, db_file, archive, component, allow_tainted=False):
 376         """Copy a file to the given archive and component
 377
 378         @type  db_file: L{daklib.dbconn.PoolFile}
 379         @param db_file: file to copy
 380
 381         @type  archive: L{daklib.dbconn.Archive}
 382         @param archive: target archive
 383
 384         @type  component: L{daklib.dbconn.Archive}
 385         @param component: target component
 386
 387         @type  allow_tainted: bool
 388         @param allow_tainted: allow to copy from tainted archives (such as NEW)
 389         """
 390         session = self.session
 391
 392         if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None:
 393             query = session.query(ArchiveFile).filter_by(file=db_file)
 394             if not allow_tainted:
 395                 query = query.join(Archive).filter(Archive.tainted == False)
 396
 397             source_af = query.first()
 398             if source_af is None:
 399                 raise ArchiveException('cp: Could not find {0} in any archive.'.format(db_file.filename))
 400             target_af = ArchiveFile(archive, component, db_file)
 401             session.add(target_af)
 402             session.flush()
 403             self.fs.copy(source_af.path, target_af.path, link=False, mode=archive.mode)
 404
 405     def copy_binary(self, db_binary, suite, component, allow_tainted=False, extra_archives=None):
 406         """Copy a binary package to the given suite and component
 407
 408         @type  db_binary: L{daklib.dbconn.DBBinary}
 409         @param db_binary: binary to copy
 410
 411         @type  suite: L{daklib.dbconn.Suite}
 412         @param suite: target suite
 413
 414         @type  component: L{daklib.dbconn.Component}
 415         @param component: target component
 416
 417         @type  allow_tainted: bool
 418         @param allow_tainted: allow to copy from tainted archives (such as NEW)
 419
 420         @type  extra_archives: list of L{daklib.dbconn.Archive}
 421         @param extra_archives: extra archives to copy Built-Using sources from
 422         """
 423         session = self.session
 424         archive = suite.archive
 425         if archive.tainted:
 426             allow_tainted = True
 427
 428         filename = db_binary.poolfile.filename
 429
 430         # make sure source is present in target archive
 431         db_source = db_binary.source
 432         if session.query(ArchiveFile).filter_by(archive=archive, file=db_source.poolfile).first() is None:
 433             raise ArchiveException('{0}: cannot copy to {1}: source is not present in target archive'.format(filename, suite.suite_name))
 434
 435         # make sure built-using packages are present in target archive
 436         for db_source in db_binary.extra_sources:
 437             self._ensure_extra_source_exists(filename, db_source, archive, extra_archives=extra_archives)
 438
 439         # copy binary
 440         db_file = db_binary.poolfile
 441         self._copy_file(db_file, suite.archive, component, allow_tainted=allow_tainted)
 442         if suite not in db_binary.suites:
 443             db_binary.suites.append(suite)
 444         self.session.flush()
 445
 446     def copy_source(self, db_source, suite, component, allow_tainted=False):
 447         """Copy a source package to the given suite and component
 448
 449         @type  db_source: L{daklib.dbconn.DBSource}
 450         @param db_source: source to copy
 451
 452         @type  suite: L{daklib.dbconn.Suite}
 453         @param suite: target suite
 454
 455         @type  component: L{daklib.dbconn.Component}
 456         @param component: target component
 457
 458         @type  allow_tainted: bool
 459         @param allow_tainted: allow to copy from tainted archives (such as NEW)
 460         """
 461         archive = suite.archive
 462         if archive.tainted:
 463             allow_tainted = True
 464         for db_dsc_file in db_source.srcfiles:
 465             self._copy_file(db_dsc_file.poolfile, archive, component, allow_tainted=allow_tainted)
 466         if suite not in db_source.suites:
 467             db_source.suites.append(suite)
 468         self.session.flush()
 469
 470     def remove_file(self, db_file, archive, component):
 471         """Remove a file from a given archive and component
 472
 473         @type  db_file: L{daklib.dbconn.PoolFile}
 474         @param db_file: file to remove
 475
 476         @type  archive: L{daklib.dbconn.Archive}
 477         @param archive: archive to remove the file from
 478
 479         @type  component: L{daklib.dbconn.Component}
 480         @param component: component to remove the file from
 481         """
 482         af = self.session.query(ArchiveFile).filter_by(file=db_file, archive=archive, component=component)
 483         self.fs.unlink(af.path)
 484         self.session.delete(af)
 485
 486     def remove_binary(self, binary, suite):
 487         """Remove a binary from a given suite and component
 488
 489         @type  binary: L{daklib.dbconn.DBBinary}
 490         @param binary: binary to remove
 491
 492         @type  suite: L{daklib.dbconn.Suite}
 493         @param suite: suite to remove the package from
 494         """
 495         binary.suites.remove(suite)
 496         self.session.flush()
 497
 498     def remove_source(self, source, suite):
 499         """Remove a source from a given suite and component
 500
 501         @type  source: L{daklib.dbconn.DBSource}
 502         @param source: source to remove
 503
 504         @type  suite: L{daklib.dbconn.Suite}
 505         @param suite: suite to remove the package from
 506
 507         @raise ArchiveException: source package is still referenced by other
 508                                  binaries in the suite
 509         """
 510         session = self.session
 511
 512         query = session.query(DBBinary).filter_by(source=source) \
 513             .filter(DBBinary.suites.contains(suite))
 514         if query.first() is not None:
 515             raise ArchiveException('src:{0} is still used by binaries in suite {1}'.format(source.source, suite.suite_name))
 516
 517         source.suites.remove(suite)
 518         session.flush()
 519
 520     def commit(self):
 521         """commit changes"""
 522         try:
 523             self.session.commit()
 524             self.fs.commit()
 525         finally:
 526             self.session.rollback()
 527             self.fs.rollback()
 528
 529     def rollback(self):
 530         """rollback changes"""
 531         self.session.rollback()
 532         self.fs.rollback()
 533
 534     def __enter__(self):
 535         return self
 536
 537     def __exit__(self, type, value, traceback):
 538         if type is None:
 539             self.commit()
 540         else:
 541             self.rollback()
 542         return None
 543
 544 def source_component_from_package_list(package_list, suite):
 545     """Get component for a source package
 546
 547     This function will look at the Package-List field to determine the
 548     component the source package belongs to. This is the first component
 549     the source package provides binaries for (first with respect to the
 550     ordering of components).
 551
 552     It the source package has no Package-List field, None is returned.
 553
 554     @type  package_list: L{daklib.packagelist.PackageList}
 555     @param package_list: package list of the source to get the override for
 556
 557     @type  suite: L{daklib.dbconn.Suite}
 558     @param suite: suite to consider for binaries produced
 559
 560     @rtype:  L{daklib.dbconn.Component} or C{None}
 561     @return: component for the given source or C{None}
 562     """
 563     if package_list.fallback:
 564         return None
 565     session = object_session(suite)
 566     packages = package_list.packages_for_suite(suite)
 567     components = set(p.component for p in packages)
 568     query = session.query(Component).order_by(Component.ordering) \
 569             .filter(Component.component_name.in_(components))
 570     return query.first()
 571
 572 class ArchiveUpload(object):
 573     """handle an upload
 574
 575     This class can be used in a with-statement::
 576
 577        with ArchiveUpload(...) as upload:
 578           ...
 579
 580     Doing so will automatically run any required cleanup and also rollback the
 581     transaction if it was not committed.
 582     """
 583     def __init__(self, directory, changes, keyrings):
 584         self.transaction = ArchiveTransaction()
 585         """transaction used to handle the upload
 586         @type: L{daklib.archive.ArchiveTransaction}
 587         """
 588
 589         self.session = self.transaction.session
 590         """database session"""
 591
 592         self.original_directory = directory
 593         self.original_changes = changes
 594
 595         self.changes = None
 596         """upload to process
 597         @type: L{daklib.upload.Changes}
 598         """
 599
 600         self.directory = None
 601         """directory with temporary copy of files. set by C{prepare}
 602         @type: str
 603         """
 604
 605         self.keyrings = keyrings
 606
 607         self.fingerprint = self.session.query(Fingerprint).filter_by(fingerprint=changes.primary_fingerprint).one()
 608         """fingerprint of the key used to sign the upload
 609         @type: L{daklib.dbconn.Fingerprint}
 610         """
 611
 612         self.reject_reasons = []
 613         """reasons why the upload cannot by accepted
 614         @type: list of str
 615         """
 616
 617         self.warnings = []
 618         """warnings
 619         @note: Not used yet.
 620         @type: list of str
 621         """
 622
 623         self.final_suites = None
 624
 625         self.new = False
 626         """upload is NEW. set by C{check}
 627         @type: bool
 628         """
 629
 630         self._checked = False
 631         """checks passes. set by C{check}
 632         @type: bool
 633         """
 634
 635         self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one()
 636         self._new = self._new_queue.suite
 637
 638     def warn(self, message):
 639         """add a warning message
 640
 641         Adds a warning message that can later be seen in C{self.warnings}
 642
 643         @type  message: string
 644         @param message: warning message
 645         """
 646         self.warnings.append(message)
 647
 648     def prepare(self):
 649         """prepare upload for further processing
 650
 651         This copies the files involved to a temporary directory.  If you use
 652         this method directly, you have to remove the directory given by the
 653         C{directory} attribute later on your own.
 654
 655         Instead of using the method directly, you can also use a with-statement::
 656
 657            with ArchiveUpload(...) as upload:
 658               ...
 659
 660         This will automatically handle any required cleanup.
 661         """
 662         assert self.directory is None
 663         assert self.original_changes.valid_signature
 664
 665         cnf = Config()
 666         session = self.transaction.session
 667
 668         group = cnf.get('Dinstall::UnprivGroup') or None
 669         self.directory = utils.temp_dirname(parent=cnf.get('Dir::TempPath'),
 670                                             mode=0o2750, group=group)
 671         with FilesystemTransaction() as fs:
 672             src = os.path.join(self.original_directory, self.original_changes.filename)
 673             dst = os.path.join(self.directory, self.original_changes.filename)
 674             fs.copy(src, dst, mode=0o640)
 675
 676             self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings)
 677
 678             for f in self.changes.files.itervalues():
 679                 src = os.path.join(self.original_directory, f.filename)
 680                 dst = os.path.join(self.directory, f.filename)
 681                 if not os.path.exists(src):
 682                     continue
 683                 fs.copy(src, dst, mode=0o640)
 684
 685             source = None
 686             try:
 687                 source = self.changes.source
 688             except Exception:
 689                 # Do not raise an exception here if the .dsc is invalid.
 690                 pass
 691
 692             if source is not None:
 693                 for f in source.files.itervalues():
 694                     src = os.path.join(self.original_directory, f.filename)
 695                     dst = os.path.join(self.directory, f.filename)
 696                     if not os.path.exists(dst):
 697                         try:
 698                             db_file = self.transaction.get_file(f, source.dsc['Source'], check_hashes=False)
 699                             db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first()
 700                             fs.copy(db_archive_file.path, dst, mode=0o640)
 701                         except KeyError:
 702                             # Ignore if get_file could not find it. Upload will
 703                             # probably be rejected later.
 704                             pass
 705
 706     def unpacked_source(self):
 707         """Path to unpacked source
 708
 709         Get path to the unpacked source. This method does unpack the source
 710         into a temporary directory under C{self.directory} if it has not
 711         been done so already.
 712
 713         @rtype:  str or C{None}
 714         @return: string giving the path to the unpacked source directory
 715                  or C{None} if no source was included in the upload.
 716         """
 717         assert self.directory is not None
 718
 719         source = self.changes.source
 720         if source is None:
 721             return None
 722         dsc_path = os.path.join(self.directory, source._dsc_file.filename)
 723
 724         sourcedir = os.path.join(self.directory, 'source')
 725         if not os.path.exists(sourcedir):
 726             devnull = open('/dev/null', 'w')
 727             daklib.daksubprocess.check_call(["dpkg-source", "--no-copy", "--no-check", "-x", dsc_path, sourcedir], shell=False, stdout=devnull)
 728         if not os.path.isdir(sourcedir):
 729             raise Exception("{0} is not a directory after extracting source package".format(sourcedir))
 730         return sourcedir
 731
 732     def _map_suite(self, suite_name):
 733         for rule in Config().value_list("SuiteMappings"):
 734             fields = rule.split()
 735             rtype = fields[0]
 736             if rtype == "map" or rtype == "silent-map":
 737                 (src, dst) = fields[1:3]
 738                 if src == suite_name:
 739                     suite_name = dst
 740                     if rtype != "silent-map":
 741                         self.warnings.append('Mapping {0} to {1}.'.format(src, dst))
 742             elif rtype == "ignore":
 743                 ignored = fields[1]
 744                 if suite_name == ignored:
 745                     self.warnings.append('Ignoring target suite {0}.'.format(ignored))
 746                     suite_name = None
 747             elif rtype == "reject":
 748                 rejected = fields[1]
 749                 if suite_name == rejected:
 750                     raise checks.Reject('Uploads to {0} are not accepted.'.format(rejected))
 751             ## XXX: propup-version and map-unreleased not yet implemented
 752         return suite_name
 753
 754     def _mapped_suites(self):
 755         """Get target suites after mappings
 756
 757         @rtype:  list of L{daklib.dbconn.Suite}
 758         @return: list giving the mapped target suites of this upload
 759         """
 760         session = self.session
 761
 762         suite_names = []
 763         for dist in self.changes.distributions:
 764             suite_name = self._map_suite(dist)
 765             if suite_name is not None:
 766                 suite_names.append(suite_name)
 767
 768         suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
 769         return suites
 770
 771     def _check_new_binary_overrides(self, suite):
 772         new = False
 773
 774         binaries = self.changes.binaries
 775         source = self.changes.source
 776         if source is not None and not source.package_list.fallback:
 777             packages = source.package_list.packages_for_suite(suite)
 778             binaries = [ entry for entry in packages ]
 779
 780         for b in binaries:
 781             override = self._binary_override(suite, b)
 782             if override is None:
 783                 self.warnings.append('binary:{0} is NEW.'.format(b.name))
 784                 new = True
 785
 786         return new
 787
 788     def _check_new(self, suite):
 789         """Check if upload is NEW
 790
 791         An upload is NEW if it has binary or source packages that do not have
 792         an override in C{suite} OR if it references files ONLY in a tainted
 793         archive (eg. when it references files in NEW).
 794
 795         @rtype:  bool
 796         @return: C{True} if the upload is NEW, C{False} otherwise
 797         """
 798         session = self.session
 799         new = False
 800
 801         # Check for missing overrides
 802         if self._check_new_binary_overrides(suite):
 803             new = True
 804         if self.changes.source is not None:
 805             override = self._source_override(suite, self.changes.source)
 806             if override is None:
 807                 self.warnings.append('source:{0} is NEW.'.format(self.changes.source.dsc['Source']))
 808                 new = True
 809
 810         # Check if we reference a file only in a tainted archive
 811         files = self.changes.files.values()
 812         if self.changes.source is not None:
 813             files.extend(self.changes.source.files.values())
 814         for f in files:
 815             query = session.query(ArchiveFile).join(PoolFile).filter(PoolFile.sha1sum == f.sha1sum)
 816             query_untainted = query.join(Archive).filter(Archive.tainted == False)
 817
 818             in_archive = (query.first() is not None)
 819             in_untainted_archive = (query_untainted.first() is not None)
 820
 821             if in_archive and not in_untainted_archive:
 822                 self.warnings.append('{0} is only available in NEW.'.format(f.filename))
 823                 new = True
 824
 825         return new
 826
 827     def _final_suites(self):
 828         session = self.session
 829
 830         mapped_suites = self._mapped_suites()
 831         final_suites = set()
 832
 833         for suite in mapped_suites:
 834             overridesuite = suite
 835             if suite.overridesuite is not None:
 836                 overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 837             if self._check_new(overridesuite):
 838                 self.new = True
 839             final_suites.add(suite)
 840
 841         return final_suites
 842
 843     def _binary_override(self, suite, binary):
 844         """Get override entry for a binary
 845
 846         @type  suite: L{daklib.dbconn.Suite}
 847         @param suite: suite to get override for
 848
 849         @type  binary: L{daklib.upload.Binary} or L{daklib.packagelist.PackageListEntry}
 850         @param binary: binary to get override for
 851
 852         @rtype:  L{daklib.dbconn.Override} or C{None}
 853         @return: override for the given binary or C{None}
 854         """
 855         if suite.overridesuite is not None:
 856             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 857
 858         mapped_component = get_mapped_component(binary.component)
 859         if mapped_component is None:
 860             return None
 861
 862         query = self.session.query(Override).filter_by(suite=suite, package=binary.name) \
 863                 .join(Component).filter(Component.component_name == mapped_component.component_name) \
 864                 .join(OverrideType).filter(OverrideType.overridetype == binary.type)
 865
 866         try:
 867             return query.one()
 868         except NoResultFound:
 869             return None
 870
 871     def _source_override(self, suite, source):
 872         """Get override entry for a source
 873
 874         @type  suite: L{daklib.dbconn.Suite}
 875         @param suite: suite to get override for
 876
 877         @type  source: L{daklib.upload.Source}
 878         @param source: source to get override for
 879
 880         @rtype:  L{daklib.dbconn.Override} or C{None}
 881         @return: override for the given source or C{None}
 882         """
 883         if suite.overridesuite is not None:
 884             suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
 885
 886         query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \
 887                 .join(OverrideType).filter(OverrideType.overridetype == 'dsc')
 888
 889         component = source_component_from_package_list(source.package_list, suite)
 890         if component is not None:
 891             query = query.filter(Override.component == component)
 892
 893         try:
 894             return query.one()
 895         except NoResultFound:
 896             return None
 897
 898     def _binary_component(self, suite, binary, only_overrides=True):
 899         """get component for a binary
 900
 901         By default this will only look at overrides to get the right component;
 902         if C{only_overrides} is C{False} this method will also look at the
 903         Section field.
 904
 905         @type  suite: L{daklib.dbconn.Suite}
 906
 907         @type  binary: L{daklib.upload.Binary}
 908
 909         @type  only_overrides: bool
 910         @param only_overrides: only use overrides to get the right component
 911
 912         @rtype: L{daklib.dbconn.Component} or C{None}
 913         """
 914         override = self._binary_override(suite, binary)
 915         if override is not None:
 916             return override.component
 917         if only_overrides:
 918             return None
 919         return get_mapped_component(binary.component, self.session)
 920
 921     def check(self, force=False):
 922         """run checks against the upload
 923
 924         @type  force: bool
 925         @param force: ignore failing forcable checks
 926
 927         @rtype:  bool
 928         @return: C{True} if all checks passed, C{False} otherwise
 929         """
 930         # XXX: needs to be better structured.
 931         assert self.changes.valid_signature
 932
 933         try:
 934             # Validate signatures and hashes before we do any real work:
 935             for chk in (
 936                     checks.SignatureAndHashesCheck,
 937                     checks.SignatureTimestampCheck,
 938                     checks.ChangesCheck,
 939                     checks.ExternalHashesCheck,
 940                     checks.SourceCheck,
 941                     checks.BinaryCheck,
 942                     checks.BinaryTimestampCheck,
 943                     checks.SingleDistributionCheck,
 944                     ):
 945                 chk().check(self)
 946
 947             final_suites = self._final_suites()
 948             if len(final_suites) == 0:
 949                 self.reject_reasons.append('No target suite found. Please check your target distribution and that you uploaded to the right archive.')
 950                 return False
 951
 952             self.final_suites = final_suites
 953
 954             for chk in (
 955                     checks.TransitionCheck,
 956                     checks.ACLCheck,
 957                     checks.NoSourceOnlyCheck,
 958                     checks.LintianCheck,
 959                     ):
 960                 chk().check(self)
 961
 962             for chk in (
 963                     checks.ACLCheck,
 964                     checks.SourceFormatCheck,
 965                     checks.SuiteArchitectureCheck,
 966                     checks.VersionCheck,
 967                     ):
 968                 for suite in final_suites:
 969                     chk().per_suite_check(self, suite)
 970
 971             if len(self.reject_reasons) != 0:
 972                 return False
 973
 974             self._checked = True
 975             return True
 976         except checks.Reject as e:
 977             self.reject_reasons.append(unicode(e))
 978         except Exception as e:
 979             self.reject_reasons.append("Processing raised an exception: {0}.\n{1}".format(e, traceback.format_exc()))
 980         return False
 981
 982     def _install_to_suite(self, suite, source_component_func, binary_component_func, source_suites=None, extra_source_archives=None):
 983         """Install upload to the given suite
 984
 985         @type  suite: L{daklib.dbconn.Suite}
 986         @param suite: suite to install the package into. This is the real suite,
 987                       ie. after any redirection to NEW or a policy queue
 988
 989         @param source_component_func: function to get the L{daklib.dbconn.Component}
 990                                       for a L{daklib.upload.Source} object
 991
 992         @param binary_component_func: function to get the L{daklib.dbconn.Component}
 993                                       for a L{daklib.upload.Binary} object
 994
 995         @param source_suites: see L{daklib.archive.ArchiveTransaction.install_binary}
 996
 997         @param extra_source_archives: see L{daklib.archive.ArchiveTransaction.install_binary}
 998
 999         @return: tuple with two elements. The first is a L{daklib.dbconn.DBSource}
1000                  object for the install source or C{None} if no source was
1001                  included. The second is a list of L{daklib.dbconn.DBBinary}
1002                  objects for the installed binary packages.
1003         """
1004         # XXX: move this function to ArchiveTransaction?
1005
1006         control = self.changes.changes
1007         changed_by = get_or_set_maintainer(control.get('Changed-By', control['Maintainer']), self.session)
1008
1009         if source_suites is None:
1010             source_suites = self.session.query(Suite).join((VersionCheck, VersionCheck.reference_id == Suite.suite_id)).filter(VersionCheck.check == 'Enhances').filter(VersionCheck.suite == suite).subquery()
1011
1012         source = self.changes.source
1013         if source is not None:
1014             component = source_component_func(source)
1015             db_source = self.transaction.install_source(self.directory, source, suite, component, changed_by, fingerprint=self.fingerprint)
1016         else:
1017             db_source = None
1018
1019         db_binaries = []
1020         for binary in self.changes.binaries:
1021             component = binary_component_func(binary)
1022             db_binary = self.transaction.install_binary(self.directory, binary, suite, component, fingerprint=self.fingerprint, source_suites=source_suites, extra_source_archives=extra_source_archives)
1023             db_binaries.append(db_binary)
1024
1025         if suite.copychanges:
1026             src = os.path.join(self.directory, self.changes.filename)
1027             dst = os.path.join(suite.archive.path, 'dists', suite.suite_name, self.changes.filename)
1028             self.transaction.fs.copy(src, dst, mode=suite.archive.mode)
1029
1030         return (db_source, db_binaries)
1031
1032     def _install_changes(self):
1033         assert self.changes.valid_signature
1034         control = self.changes.changes
1035         session = self.transaction.session
1036         config = Config()
1037
1038         changelog_id = None
1039         # Only add changelog for sourceful uploads and binNMUs
1040         if 'source' in self.changes.architectures or re_bin_only_nmu.search(control['Version']):
1041             query = 'INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id'
1042             changelog_id = session.execute(query, {'changelog': control['Changes']}).scalar()
1043             assert changelog_id is not None
1044
1045         db_changes = DBChange()
1046         db_changes.changesname = self.changes.filename
1047         db_changes.source = control['Source']
1048         db_changes.binaries = control.get('Binary', None)
1049         db_changes.architecture = control['Architecture']
1050         db_changes.version = control['Version']
1051         db_changes.distribution = control['Distribution']
1052         db_changes.urgency = control['Urgency']
1053         db_changes.maintainer = control['Maintainer']
1054         db_changes.changedby = control.get('Changed-By', control['Maintainer'])
1055         db_changes.date = control['Date']
1056         db_changes.fingerprint = self.fingerprint.fingerprint
1057         db_changes.changelog_id = changelog_id
1058         db_changes.closes = self.changes.closed_bugs
1059
1060         try:
1061             self.transaction.session.add(db_changes)
1062             self.transaction.session.flush()
1063         except sqlalchemy.exc.IntegrityError:
1064             raise ArchiveException('{0} is already known.'.format(self.changes.filename))
1065
1066         return db_changes
1067
1068     def _install_policy(self, policy_queue, target_suite, db_changes, db_source, db_binaries):
1069         u = PolicyQueueUpload()
1070         u.policy_queue = policy_queue
1071         u.target_suite = target_suite
1072         u.changes = db_changes
1073         u.source = db_source
1074         u.binaries = db_binaries
1075         self.transaction.session.add(u)
1076         self.transaction.session.flush()
1077
1078         dst = os.path.join(policy_queue.path, self.changes.filename)
1079         self.transaction.fs.copy(self.changes.path, dst, mode=policy_queue.change_perms)
1080
1081         return u
1082
1083     def try_autobyhand(self):
1084         """Try AUTOBYHAND
1085
1086         Try to handle byhand packages automatically.
1087
1088         @rtype:  list of L{daklib.upload.HashedFile}
1089         @return: list of remaining byhand files
1090         """
1091         assert len(self.reject_reasons) == 0
1092         assert self.changes.valid_signature
1093         assert self.final_suites is not None
1094         assert self._checked
1095
1096         byhand = self.changes.byhand_files
1097         if len(byhand) == 0:
1098             return True
1099
1100         suites = list(self.final_suites)
1101         assert len(suites) == 1, "BYHAND uploads must be to a single suite"
1102         suite = suites[0]
1103
1104         cnf = Config()
1105         control = self.changes.changes
1106         automatic_byhand_packages = cnf.subtree("AutomaticByHandPackages")
1107
1108         remaining = []
1109         for f in byhand:
1110             if '_' in f.filename:
1111                 parts = f.filename.split('_', 2)
1112                 if len(parts) != 3:
1113                     print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
1114                     remaining.append(f)
1115                     continue
1116
1117                 package, version, archext = parts
1118                 arch, ext = archext.split('.', 1)
1119             else:
1120                 parts = f.filename.split('.')
1121                 if len(parts) < 2:
1122                     print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
1123                     remaining.append(f)
1124                     continue
1125
1126                 package = parts[0]
1127                 version = '0'
1128                 arch = 'all'
1129                 ext = parts[-1]
1130
1131             try:
1132                 rule = automatic_byhand_packages.subtree(package)
1133             except KeyError:
1134                 remaining.append(f)
1135                 continue
1136
1137             if rule['Source'] != self.changes.source_name \
1138                     or rule['Section'] != f.section \
1139                     or ('Extension' in rule and rule['Extension'] != ext):
1140                 remaining.append(f)
1141                 continue
1142
1143             script = rule['Script']
1144             retcode = daklib.daksubprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False)
1145             if retcode != 0:
1146                 print "W: error processing {0}.".format(f.filename)
1147                 remaining.append(f)
1148
1149         return len(remaining) == 0
1150
1151     def _install_byhand(self, policy_queue_upload, hashed_file):
1152         """install byhand file
1153
1154         @type  policy_queue_upload: L{daklib.dbconn.PolicyQueueUpload}
1155
1156         @type  hashed_file: L{daklib.upload.HashedFile}
1157         """
1158         fs = self.transaction.fs
1159         session = self.transaction.session
1160         policy_queue = policy_queue_upload.policy_queue
1161
1162         byhand_file = PolicyQueueByhandFile()
1163         byhand_file.upload = policy_queue_upload
1164         byhand_file.filename = hashed_file.filename
1165         session.add(byhand_file)
1166         session.flush()
1167
1168         src = os.path.join(self.directory, hashed_file.filename)
1169         dst = os.path.join(policy_queue.path, hashed_file.filename)
1170         fs.copy(src, dst, mode=policy_queue.change_perms)
1171
1172         return byhand_file
1173
1174     def _do_bts_versiontracking(self):
1175         cnf = Config()
1176         fs = self.transaction.fs
1177
1178         btsdir = cnf.get('Dir::BTSVersionTrack')
1179         if btsdir is None or btsdir == '':
1180             return
1181
1182         base = os.path.join(btsdir, self.changes.filename[:-8])
1183
1184         # version history
1185         sourcedir = self.unpacked_source()
1186         if sourcedir is not None:
1187             fh = open(os.path.join(sourcedir, 'debian', 'changelog'), 'r')
1188             versions = fs.create("{0}.versions".format(base), mode=0o644)
1189             for line in fh.readlines():
1190                 if re_changelog_versions.match(line):
1191                     versions.write(line)
1192             fh.close()
1193             versions.close()
1194
1195         # binary -> source mapping
1196         debinfo = fs.create("{0}.debinfo".format(base), mode=0o644)
1197         for binary in self.changes.binaries:
1198             control = binary.control
1199             source_package, source_version = binary.source
1200             line = " ".join([control['Package'], control['Version'], control['Architecture'], source_package, source_version])
1201             print >>debinfo, line
1202         debinfo.close()
1203
1204     def _policy_queue(self, suite):
1205         if suite.policy_queue is not None:
1206             return suite.policy_queue
1207         return None
1208
1209     def install(self):
1210         """install upload
1211
1212         Install upload to a suite or policy queue.  This method does B{not}
1213         handle uploads to NEW.
1214
1215         You need to have called the C{check} method before calling this method.
1216         """
1217         assert len(self.reject_reasons) == 0
1218         assert self.changes.valid_signature
1219         assert self.final_suites is not None
1220         assert self._checked
1221         assert not self.new
1222
1223         db_changes = self._install_changes()
1224
1225         for suite in self.final_suites:
1226             overridesuite = suite
1227             if suite.overridesuite is not None:
1228                 overridesuite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
1229
1230             policy_queue = self._policy_queue(suite)
1231
1232             redirected_suite = suite
1233             if policy_queue is not None:
1234                 redirected_suite = policy_queue.suite
1235
1236             # source can be in the suite we install to or any suite we enhance
1237             source_suite_ids = set([suite.suite_id, redirected_suite.suite_id])
1238             for enhanced_suite_id, in self.session.query(VersionCheck.reference_id) \
1239                     .filter(VersionCheck.suite_id.in_(source_suite_ids)) \
1240                     .filter(VersionCheck.check == 'Enhances'):
1241                 source_suite_ids.add(enhanced_suite_id)
1242
1243             source_suites = self.session.query(Suite).filter(Suite.suite_id.in_(source_suite_ids)).subquery()
1244
1245             source_component_func = lambda source: self._source_override(overridesuite, source).component
1246             binary_component_func = lambda binary: self._binary_component(overridesuite, binary)
1247
1248             (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])
1249
1250             if policy_queue is not None:
1251                 self._install_policy(policy_queue, suite, db_changes, db_source, db_binaries)
1252
1253             # copy to build queues
1254             if policy_queue is None or policy_queue.send_to_build_queues:
1255                 for build_queue in suite.copy_queues:
1256                     self._install_to_suite(build_queue.suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])
1257
1258         self._do_bts_versiontracking()
1259
1260     def install_to_new(self):
1261         """install upload to NEW
1262
1263         Install upload to NEW.  This method does B{not} handle regular uploads
1264         to suites or policy queues.
1265
1266         You need to have called the C{check} method before calling this method.
1267         """
1268         # Uploads to NEW are special as we don't have overrides.
1269         assert len(self.reject_reasons) == 0
1270         assert self.changes.valid_signature
1271         assert self.final_suites is not None
1272
1273         source = self.changes.source
1274         binaries = self.changes.binaries
1275         byhand = self.changes.byhand_files
1276
1277         # we need a suite to guess components
1278         suites = list(self.final_suites)
1279         assert len(suites) == 1, "NEW uploads must be to a single suite"
1280         suite = suites[0]
1281
1282         # decide which NEW queue to use
1283         if suite.new_queue is None:
1284             new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one()
1285         else:
1286             new_queue = suite.new_queue
1287         if len(byhand) > 0:
1288             # There is only one global BYHAND queue
1289             new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one()
1290         new_suite = new_queue.suite
1291
1292
1293         def binary_component_func(binary):
1294             return self._binary_component(suite, binary, only_overrides=False)
1295
1296         # guess source component
1297         # XXX: should be moved into an extra method
1298         binary_component_names = set()
1299         for binary in binaries:
1300             component = binary_component_func(binary)
1301             binary_component_names.add(component.component_name)
1302         source_component_name = None
1303         for c in self.session.query(Component).order_by(Component.component_id):
1304             guess = c.component_name
1305             if guess in binary_component_names:
1306                 source_component_name = guess
1307                 break
1308         if source_component_name is None:
1309             source_component = self.session.query(Component).order_by(Component.component_id).first()
1310         else:
1311             source_component = self.session.query(Component).filter_by(component_name=source_component_name).one()
1312         source_component_func = lambda source: source_component
1313
1314         db_changes = self._install_changes()
1315         (db_source, db_binaries) = self._install_to_suite(new_suite, source_component_func, binary_component_func, source_suites=True, extra_source_archives=[suite.archive])
1316         policy_upload = self._install_policy(new_queue, suite, db_changes, db_source, db_binaries)
1317
1318         for f in byhand:
1319             self._install_byhand(policy_upload, f)
1320
1321         self._do_bts_versiontracking()
1322
1323     def commit(self):
1324         """commit changes"""
1325         self.transaction.commit()
1326
1327     def rollback(self):
1328         """rollback changes"""
1329         self.transaction.rollback()
1330
1331     def __enter__(self):
1332         self.prepare()
1333         return self
1334
1335     def __exit__(self, type, value, traceback):
1336         if self.directory is not None:
1337             shutil.rmtree(self.directory)
1338             self.directory = None
1339         self.changes = None
1340         self.transaction.rollback()
1341         return None