X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=daklib%2Farchive.py;h=04c55c0ae8f45ba0e7b4f63870b24a627fe191e5;hb=2b8e14db14a20525fc6bb4b38fee86d99e4d382c;hp=2badae7252693c6cc57ec025c695ba28e36ea51d;hpb=1d2a3e7dc88565f49d0f438c389f57c340800331;p=dak.git diff --git a/daklib/archive.py b/daklib/archive.py index 2badae72..04c55c0a 100644 --- a/daklib/archive.py +++ b/daklib/archive.py @@ -26,13 +26,15 @@ import daklib.upload as upload import daklib.utils as utils from daklib.fstransactions import FilesystemTransaction from daklib.regexes import re_changelog_versions, re_bin_only_nmu +import daklib.daksubprocess import apt_pkg from datetime import datetime import os import shutil -import subprocess from sqlalchemy.orm.exc import NoResultFound +from sqlalchemy.orm import object_session +import sqlalchemy.exc import tempfile import traceback @@ -85,7 +87,7 @@ class ArchiveTransaction(object): Will not give an error when the file is already present. @rtype: L{daklib.dbconn.PoolFile} - @return: batabase object for the new file + @return: database object for the new file """ session = self.session @@ -335,6 +337,8 @@ class ArchiveTransaction(object): db_source.suites.append(suite) if not created: + for f in db_source.srcfiles: + self._copy_file(f.poolfile, archive, component, allow_tainted=allow_tainted) return db_source ### Now add remaining files and copy them to the archive. @@ -361,11 +365,7 @@ class ArchiveTransaction(object): # Uploaders are the maintainer and co-maintainers from the Uploaders field db_source.uploaders.append(maintainer) if 'Uploaders' in control: - def split_uploaders(field): - import re - for u in re.sub(">[ ]*,", ">\t", field).split("\t"): - yield u.strip() - + from daklib.textutils import split_uploaders for u in split_uploaders(control['Uploaders']): db_source.uploaders.append(get_or_set_maintainer(u, session)) session.flush() @@ -390,13 +390,13 @@ class ArchiveTransaction(object): session = self.session if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None: - query = session.query(ArchiveFile).filter_by(file=db_file, component=component) + query = session.query(ArchiveFile).filter_by(file=db_file) if not allow_tainted: query = query.join(Archive).filter(Archive.tainted == False) source_af = query.first() if source_af is None: - raise ArchiveException('cp: Could not find {0} in component {1} in any archive.'.format(db_file.filename, component.component_name)) + raise ArchiveException('cp: Could not find {0} in any archive.'.format(db_file.filename)) target_af = ArchiveFile(archive, component, db_file) session.add(target_af) session.flush() @@ -541,6 +541,34 @@ class ArchiveTransaction(object): self.rollback() return None +def source_component_from_package_list(package_list, suite): + """Get component for a source package + + This function will look at the Package-List field to determine the + component the source package belongs to. This is the first component + the source package provides binaries for (first with respect to the + ordering of components). + + It the source package has no Package-List field, None is returned. + + @type package_list: L{daklib.packagelist.PackageList} + @param package_list: package list of the source to get the override for + + @type suite: L{daklib.dbconn.Suite} + @param suite: suite to consider for binaries produced + + @rtype: L{daklib.dbconn.Component} or C{None} + @return: component for the given source or C{None} + """ + if package_list.fallback: + return None + session = object_session(suite) + packages = package_list.packages_for_suite(suite) + components = set(p.component for p in packages) + query = session.query(Component).order_by(Component.ordering) \ + .filter(Component.component_name.in_(components)) + return query.first() + class ArchiveUpload(object): """handle an upload @@ -599,9 +627,24 @@ class ArchiveUpload(object): @type: bool """ + self._checked = False + """checks passes. set by C{check} + @type: bool + """ + self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one() self._new = self._new_queue.suite + def warn(self, message): + """add a warning message + + Adds a warning message that can later be seen in C{self.warnings} + + @type message: string + @param message: warning message + """ + self.warnings.append(message) + def prepare(self): """prepare upload for further processing @@ -622,11 +665,13 @@ class ArchiveUpload(object): cnf = Config() session = self.transaction.session - self.directory = tempfile.mkdtemp(dir=cnf.get('Dir::TempPath')) + group = cnf.get('Dinstall::UnprivGroup') or None + self.directory = utils.temp_dirname(parent=cnf.get('Dir::TempPath'), + mode=0o2750, group=group) with FilesystemTransaction() as fs: src = os.path.join(self.original_directory, self.original_changes.filename) dst = os.path.join(self.directory, self.original_changes.filename) - fs.copy(src, dst) + fs.copy(src, dst, mode=0o640) self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings) @@ -635,9 +680,15 @@ class ArchiveUpload(object): dst = os.path.join(self.directory, f.filename) if not os.path.exists(src): continue - fs.copy(src, dst) + fs.copy(src, dst, mode=0o640) + + source = None + try: + source = self.changes.source + except Exception: + # Do not raise an exception here if the .dsc is invalid. + pass - source = self.changes.source if source is not None: for f in source.files.itervalues(): src = os.path.join(self.original_directory, f.filename) @@ -646,7 +697,7 @@ class ArchiveUpload(object): try: db_file = self.transaction.get_file(f, source.dsc['Source'], check_hashes=False) db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first() - fs.copy(db_archive_file.path, dst, symlink=True) + fs.copy(db_archive_file.path, dst, mode=0o640) except KeyError: # Ignore if get_file could not find it. Upload will # probably be rejected later. @@ -672,7 +723,8 @@ class ArchiveUpload(object): sourcedir = os.path.join(self.directory, 'source') if not os.path.exists(sourcedir): - subprocess.check_call(["dpkg-source", "--no-copy", "-x", dsc_path, sourcedir], shell=False) + devnull = open('/dev/null', 'w') + daklib.daksubprocess.check_call(["dpkg-source", "--no-copy", "--no-check", "-x", dsc_path, sourcedir], shell=False, stdout=devnull) if not os.path.isdir(sourcedir): raise Exception("{0} is not a directory after extracting source package".format(sourcedir)) return sourcedir @@ -686,7 +738,7 @@ class ArchiveUpload(object): if src == suite_name: suite_name = dst if rtype != "silent-map": - self.warnings.append('Mapping {0} to {0}.'.format(src, dst)) + self.warnings.append('Mapping {0} to {1}.'.format(src, dst)) elif rtype == "ignore": ignored = fields[1] if suite_name == ignored: @@ -695,7 +747,7 @@ class ArchiveUpload(object): elif rtype == "reject": rejected = fields[1] if suite_name == rejected: - self.reject_reasons.append('Uploads to {0} are not accepted.'.format(suite)) + raise checks.Reject('Uploads to {0} are not accepted.'.format(rejected)) ## XXX: propup-version and map-unreleased not yet implemented return suite_name @@ -716,6 +768,23 @@ class ArchiveUpload(object): suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names)) return suites + def _check_new_binary_overrides(self, suite): + new = False + + binaries = self.changes.binaries + source = self.changes.source + if source is not None and not source.package_list.fallback: + packages = source.package_list.packages_for_suite(suite) + binaries = [ entry for entry in packages ] + + for b in binaries: + override = self._binary_override(suite, b) + if override is None: + self.warnings.append('binary:{0} is NEW.'.format(b.name)) + new = True + + return new + def _check_new(self, suite): """Check if upload is NEW @@ -727,17 +796,16 @@ class ArchiveUpload(object): @return: C{True} if the upload is NEW, C{False} otherwise """ session = self.session + new = False # Check for missing overrides - for b in self.changes.binaries: - override = self._binary_override(suite, b) - if override is None: - return True - + if self._check_new_binary_overrides(suite): + new = True if self.changes.source is not None: override = self._source_override(suite, self.changes.source) if override is None: - return True + self.warnings.append('source:{0} is NEW.'.format(self.changes.source.dsc['Source'])) + new = True # Check if we reference a file only in a tainted archive files = self.changes.files.values() @@ -751,7 +819,10 @@ class ArchiveUpload(object): in_untainted_archive = (query_untainted.first() is not None) if in_archive and not in_untainted_archive: - return True + self.warnings.append('{0} is only available in NEW.'.format(f.filename)) + new = True + + return new def _final_suites(self): session = self.session @@ -775,7 +846,7 @@ class ArchiveUpload(object): @type suite: L{daklib.dbconn.Suite} @param suite: suite to get override for - @type binary: L{daklib.upload.Binary} + @type binary: L{daklib.upload.Binary} or L{daklib.packagelist.PackageListEntry} @param binary: binary to get override for @rtype: L{daklib.dbconn.Override} or C{None} @@ -784,8 +855,12 @@ class ArchiveUpload(object): if suite.overridesuite is not None: suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one() - query = self.session.query(Override).filter_by(suite=suite, package=binary.control['Package']) \ - .join(Component).filter(Component.component_name == binary.component) \ + mapped_component = get_mapped_component(binary.component) + if mapped_component is None: + return None + + query = self.session.query(Override).filter_by(suite=suite, package=binary.name) \ + .join(Component).filter(Component.component_name == mapped_component.component_name) \ .join(OverrideType).filter(OverrideType.overridetype == binary.type) try: @@ -808,10 +883,13 @@ class ArchiveUpload(object): if suite.overridesuite is not None: suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one() - # XXX: component for source? query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \ .join(OverrideType).filter(OverrideType.overridetype == 'dsc') + component = source_component_from_package_list(source.package_list, suite) + if component is not None: + query = query.filter(Override.component == component) + try: return query.one() except NoResultFound: @@ -853,28 +931,36 @@ class ArchiveUpload(object): assert self.changes.valid_signature try: + # Validate signatures and hashes before we do any real work: for chk in ( - checks.SignatureCheck, + checks.SignatureAndHashesCheck, + checks.SignatureTimestampCheck, checks.ChangesCheck, - checks.TransitionCheck, - checks.UploadBlockCheck, - checks.HashesCheck, + checks.ExternalHashesCheck, checks.SourceCheck, checks.BinaryCheck, checks.BinaryTimestampCheck, - checks.ACLCheck, checks.SingleDistributionCheck, - checks.NoSourceOnlyCheck, - checks.LintianCheck, ): chk().check(self) final_suites = self._final_suites() if len(final_suites) == 0: - self.reject_reasons.append('Ended with no suite to install to.') + self.reject_reasons.append('No target suite found. Please check your target distribution and that you uploaded to the right archive.') return False + self.final_suites = final_suites + + for chk in ( + checks.TransitionCheck, + checks.ACLCheck, + checks.NoSourceOnlyCheck, + checks.LintianCheck, + ): + chk().check(self) + for chk in ( + checks.ACLCheck, checks.SourceFormatCheck, checks.SuiteArchitectureCheck, checks.VersionCheck, @@ -885,7 +971,7 @@ class ArchiveUpload(object): if len(self.reject_reasons) != 0: return False - self.final_suites = final_suites + self._checked = True return True except checks.Reject as e: self.reject_reasons.append(unicode(e)) @@ -939,7 +1025,7 @@ class ArchiveUpload(object): if suite.copychanges: src = os.path.join(self.directory, self.changes.filename) dst = os.path.join(suite.archive.path, 'dists', suite.suite_name, self.changes.filename) - self.transaction.fs.copy(src, dst) + self.transaction.fs.copy(src, dst, mode=suite.archive.mode) return (db_source, db_binaries) @@ -971,8 +1057,11 @@ class ArchiveUpload(object): db_changes.changelog_id = changelog_id db_changes.closes = self.changes.closed_bugs - self.transaction.session.add(db_changes) - self.transaction.session.flush() + try: + self.transaction.session.add(db_changes) + self.transaction.session.flush() + except sqlalchemy.exc.IntegrityError: + raise ArchiveException('{0} is already known.'.format(self.changes.filename)) return db_changes @@ -987,7 +1076,7 @@ class ArchiveUpload(object): self.transaction.session.flush() dst = os.path.join(policy_queue.path, self.changes.filename) - self.transaction.fs.copy(self.changes.path, dst) + self.transaction.fs.copy(self.changes.path, dst, mode=policy_queue.change_perms) return u @@ -1002,6 +1091,7 @@ class ArchiveUpload(object): assert len(self.reject_reasons) == 0 assert self.changes.valid_signature assert self.final_suites is not None + assert self._checked byhand = self.changes.byhand_files if len(byhand) == 0: @@ -1017,26 +1107,41 @@ class ArchiveUpload(object): remaining = [] for f in byhand: - parts = f.filename.split('_', 2) - if len(parts) != 3: - print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename) - remaining.append(f) - continue + if '_' in f.filename: + parts = f.filename.split('_', 2) + if len(parts) != 3: + print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename) + remaining.append(f) + continue - package, version, archext = parts - arch, ext = archext.split('.', 1) + package, version, archext = parts + arch, ext = archext.split('.', 1) + else: + parts = f.filename.split('.') + if len(parts) < 2: + print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename) + remaining.append(f) + continue - rule = automatic_byhand_packages.get(package) - if rule is None: + package = parts[0] + version = '0' + arch = 'all' + ext = parts[-1] + + try: + rule = automatic_byhand_packages.subtree(package) + except KeyError: remaining.append(f) continue - if rule['Source'] != control['Source'] or rule['Section'] != f.section or rule['Extension'] != ext: + if rule['Source'] != self.changes.source_name \ + or rule['Section'] != f.section \ + or ('Extension' in rule and rule['Extension'] != ext): remaining.append(f) continue script = rule['Script'] - retcode = subprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False) + retcode = daklib.daksubprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False) if retcode != 0: print "W: error processing {0}.".format(f.filename) remaining.append(f) @@ -1062,7 +1167,7 @@ class ArchiveUpload(object): src = os.path.join(self.directory, hashed_file.filename) dst = os.path.join(policy_queue.path, hashed_file.filename) - fs.copy(src, dst) + fs.copy(src, dst, mode=policy_queue.change_perms) return byhand_file @@ -1092,7 +1197,7 @@ class ArchiveUpload(object): for binary in self.changes.binaries: control = binary.control source_package, source_version = binary.source - line = " ".join([control['Package'], control['Version'], source_package, source_version]) + line = " ".join([control['Package'], control['Version'], control['Architecture'], source_package, source_version]) print >>debinfo, line debinfo.close() @@ -1112,6 +1217,7 @@ class ArchiveUpload(object): assert len(self.reject_reasons) == 0 assert self.changes.valid_signature assert self.final_suites is not None + assert self._checked assert not self.new db_changes = self._install_changes() @@ -1127,17 +1233,25 @@ class ArchiveUpload(object): if policy_queue is not None: redirected_suite = policy_queue.suite + # source can be in the suite we install to or any suite we enhance + source_suite_ids = set([suite.suite_id, redirected_suite.suite_id]) + for enhanced_suite_id, in self.session.query(VersionCheck.reference_id) \ + .filter(VersionCheck.suite_id.in_(source_suite_ids)) \ + .filter(VersionCheck.check == 'Enhances'): + source_suite_ids.add(enhanced_suite_id) + + source_suites = self.session.query(Suite).filter(Suite.suite_id.in_(source_suite_ids)).subquery() + source_component_func = lambda source: self._source_override(overridesuite, source).component binary_component_func = lambda binary: self._binary_component(overridesuite, binary) - (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, extra_source_archives=[suite.archive]) + (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive]) if policy_queue is not None: self._install_policy(policy_queue, suite, db_changes, db_source, db_binaries) # copy to build queues if policy_queue is None or policy_queue.send_to_build_queues: - source_suites = self.session.query(Suite).filter_by(suite_id=suite.suite_id).subquery() for build_queue in suite.copy_queues: self._install_to_suite(build_queue.suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive]) @@ -1160,16 +1274,22 @@ class ArchiveUpload(object): binaries = self.changes.binaries byhand = self.changes.byhand_files - new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one() - if len(byhand) > 0: - new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one() - new_suite = new_queue.suite - # we need a suite to guess components suites = list(self.final_suites) assert len(suites) == 1, "NEW uploads must be to a single suite" suite = suites[0] + # decide which NEW queue to use + if suite.new_queue is None: + new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one() + else: + new_queue = suite.new_queue + if len(byhand) > 0: + # There is only one global BYHAND queue + new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one() + new_suite = new_queue.suite + + def binary_component_func(binary): return self._binary_component(suite, binary, only_overrides=False) @@ -1186,8 +1306,9 @@ class ArchiveUpload(object): source_component_name = guess break if source_component_name is None: - raise Exception('Could not guess source component.') - source_component = self.session.query(Component).filter_by(component_name=source_component_name).one() + source_component = self.session.query(Component).order_by(Component.component_id).first() + else: + source_component = self.session.query(Component).filter_by(component_name=source_component_name).one() source_component_func = lambda source: source_component db_changes = self._install_changes()