X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=daklib%2Fchecks.py;h=3fb883c539270a0e9f4a8f7689af60d9302efe2c;hb=01b9a74483f93f1d87a3a44ab748a9ea3b4e7dbf;hp=3e7a73d4fdc2b2797bef281a298c9d3c86e5b3cc;hpb=394eaf39d605f9b0e916d25d3d5cd223b0c5246e;p=dak.git diff --git a/daklib/checks.py b/daklib/checks.py index 3e7a73d4..3fb883c5 100644 --- a/daklib/checks.py +++ b/daklib/checks.py @@ -24,6 +24,7 @@ Please read the documentation for the L{Check} class for the interface. """ from daklib.config import Config +import daklib.daksubprocess from daklib.dbconn import * import daklib.dbconn as dbconn from daklib.regexes import * @@ -37,11 +38,19 @@ import apt_pkg from apt_pkg import version_compare import errno import os +import subprocess +import textwrap import time import yaml -# TODO: replace by subprocess -import commands +def check_fields_for_valid_utf8(filename, control): + """Check all fields of a control file for valid UTF-8""" + for field in control.keys(): + try: + field.decode('utf-8') + control[field].decode('utf-8') + except UnicodeDecodeError: + raise Reject('{0}: The {1} field is not valid UTF-8'.format(filename, field)) class Reject(Exception): """exception raised by failing checks""" @@ -100,6 +109,18 @@ class Check(object): return False class SignatureAndHashesCheck(Check): + def check_replay(self, upload): + # Use private session as we want to remember having seen the .changes + # in all cases. + session = DBConn().session() + history = SignatureHistory.from_signed_file(upload.changes) + r = history.query(session) + if r is not None: + raise Reject('Signature for changes file was already seen at {0}.\nPlease refresh the signature of the changes file if you want to upload it again.'.format(r.seen)) + session.add(history) + session.commit() + return True + """Check signature of changes and dsc file (if included in upload) Make sure the signature is valid and done by a known user. @@ -108,6 +129,7 @@ class SignatureAndHashesCheck(Check): changes = upload.changes if not changes.valid_signature: raise Reject("Signature for .changes not valid.") + self.check_replay(upload) self._check_hashes(upload, changes.filename, changes.files.itervalues()) source = None @@ -160,6 +182,8 @@ class ChangesCheck(Check): if field not in control: raise Reject('{0}: misses mandatory field {1}'.format(fn, field)) + check_fields_for_valid_utf8(fn, control) + source_match = re_field_source.match(control['Source']) if not source_match: raise Reject('{0}: Invalid Source field'.format(fn)) @@ -264,6 +288,8 @@ class BinaryCheck(Check): if field not in control: raise Reject('{0}: Missing mandatory field {0}.'.format(fn, field)) + check_fields_for_valid_utf8(fn, control) + # check fields package = control['Package'] @@ -393,6 +419,8 @@ class SourceCheck(Check): control = source.dsc dsc_fn = source._dsc_file.filename + check_fields_for_valid_utf8(dsc_fn, control) + # check fields if not re_field_package.match(control['Source']): raise Reject('{0}: Invalid Source field'.format(dsc_fn)) @@ -537,16 +565,18 @@ class TransitionCheck(Check): if transitions is None: return True + session = upload.session + control = upload.changes.changes source = re_field_source.match(control['Source']).group('package') for trans in transitions: t = transitions[trans] - source = t["source"] + transition_source = t["source"] expected = t["new"] # Will be None if nothing is in testing. - current = get_source_in_suite(source, "testing", session) + current = get_source_in_suite(transition_source, "testing", session) if current is not None: compare = apt_pkg.version_compare(current.version, expected) @@ -573,7 +603,7 @@ currently {1}, we need version {2}). This transition is managed by the Release Team, and {3} is the Release-Team member responsible for it. Please mail debian-release@lists.debian.org or contact {3} directly if you need further assistance. You might want to upload to experimental until this -transition is done.""".format(source, currentlymsg, expected,t["rm"]))) +transition is done.""".format(transition_source, currentlymsg, expected,t["rm"]))) raise Reject(rejectmsg) @@ -587,7 +617,7 @@ transition is done.""".format(source, currentlymsg, expected,t["rm"]))) contents = file(path, 'r').read() try: - transitions = yaml.load(contents) + transitions = yaml.safe_load(contents) return transitions except yaml.YAMLError as msg: utils.warn('Not checking transitions, the transitions file is broken: {0}'.format(msg)) @@ -595,17 +625,52 @@ transition is done.""".format(source, currentlymsg, expected,t["rm"]))) return None class NoSourceOnlyCheck(Check): + def is_source_only_upload(self, upload): + changes = upload.changes + if changes.source is not None and len(changes.binaries) == 0: + return True + return False + """Check for source-only upload Source-only uploads are only allowed if Dinstall::AllowSourceOnlyUploads is set. Otherwise they are rejected. + + Source-only uploads are only accepted for source packages having a + Package-List field that also lists architectures per package. This + check can be disabled via + Dinstall::AllowSourceOnlyUploadsWithoutPackageList. + + Source-only uploads to NEW are only allowed if + Dinstall::AllowSourceOnlyNew is set. + + Uploads not including architecture-independent packages are only + allowed if Dinstall::AllowNoArchIndepUploads is set. + """ def check(self, upload): - if Config().find_b("Dinstall::AllowSourceOnlyUploads"): + if not self.is_source_only_upload(upload): return True + + allow_source_only_uploads = Config().find_b('Dinstall::AllowSourceOnlyUploads') + allow_source_only_uploads_without_package_list = Config().find_b('Dinstall::AllowSourceOnlyUploadsWithoutPackageList') + allow_source_only_new = Config().find_b('Dinstall::AllowSourceOnlyNew') + allow_no_arch_indep_uploads = Config().find_b('Dinstall::AllowNoArchIndepUploads') changes = upload.changes - if changes.source is not None and len(changes.binaries) == 0: + + if not allow_source_only_uploads: raise Reject('Source-only uploads are not allowed.') + if not allow_source_only_uploads_without_package_list \ + and changes.source.package_list.fallback: + raise Reject('Source-only uploads are only allowed if a Package-List field that also list architectures is included in the source package. dpkg (>= 1.17.7) includes this information.') + if not allow_source_only_new and upload.new: + raise Reject('Source-only uploads to NEW are not allowed.') + + if not allow_no_arch_indep_uploads \ + and 'all' not in changes.architectures \ + and changes.source.package_list.has_arch_indep_packages(): + raise Reject('Uploads not including architecture-independent packages are not allowed.') + return True class LintianCheck(Check): @@ -628,7 +693,7 @@ class LintianCheck(Check): with open(tagfile, 'r') as sourcefile: sourcecontent = sourcefile.read() try: - lintiantags = yaml.load(sourcecontent)['lintian'] + lintiantags = yaml.safe_load(sourcecontent)['lintian'] except yaml.YAMLError as msg: raise Exception('Could not read lintian tags file {0}, YAML error: {1}'.format(tagfile, msg)) @@ -642,13 +707,17 @@ class LintianCheck(Check): changespath = os.path.join(upload.directory, changes.filename) try: cmd = [] + result = 0 user = cnf.get('Dinstall::UnprivUser') or None if user is not None: cmd.extend(['sudo', '-H', '-u', user]) - cmd.extend(['LINTIAN_COLL_UNPACKED_SKIP_SIG=1', '/usr/bin/lintian', '--show-overrides', '--tags-from-file', temp_filename, changespath]) - result, output = commands.getstatusoutput(" ".join(cmd)) + cmd.extend(['/usr/bin/lintian', '--show-overrides', '--tags-from-file', temp_filename, changespath]) + output = daklib.daksubprocess.check_output(cmd, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + result = e.returncode + output = e.output finally: os.unlink(temp_filename) @@ -709,23 +778,31 @@ class VersionCheck(Check): else: return db_binary.version - def _version_checks(self, upload, suite, op): + def _version_checks(self, upload, suite, other_suite, op, op_name): session = upload.session if upload.changes.source is not None: source_name = upload.changes.source.dsc['Source'] source_version = upload.changes.source.dsc['Version'] - v = self._highest_source_version(session, source_name, suite) + v = self._highest_source_version(session, source_name, other_suite) if v is not None and not op(version_compare(source_version, v)): - raise Reject('Version check failed (source={0}, version={1}, other-version={2}, suite={3})'.format(source_name, source_version, v, suite.suite_name)) + raise Reject("Version check failed:\n" + "Your upload included the source package {0}, version {1},\n" + "however {3} already has version {2}.\n" + "Uploads to {5} must have a {4} version than present in {3}." + .format(source_name, source_version, v, other_suite.suite_name, op_name, suite.suite_name)) for binary in upload.changes.binaries: binary_name = binary.control['Package'] binary_version = binary.control['Version'] architecture = binary.control['Architecture'] - v = self._highest_binary_version(session, binary_name, suite, architecture) + v = self._highest_binary_version(session, binary_name, other_suite, architecture) if v is not None and not op(version_compare(binary_version, v)): - raise Reject('Version check failed (binary={0}, version={1}, other-version={2}, suite={3})'.format(binary_name, binary_version, v, suite.suite_name)) + raise Reject("Version check failed:\n" + "Your upload included the binary package {0}, version {1}, for {2},\n" + "however {4} already has version {3}.\n" + "Uploads to {6} must have a {5} version than present in {4}." + .format(binary_name, binary_version, architecture, v, other_suite.suite_name, op_name, suite.suite_name)) def per_suite_check(self, upload, suite): session = upload.session @@ -737,13 +814,13 @@ class VersionCheck(Check): must_be_newer_than.append(suite) for s in must_be_newer_than: - self._version_checks(upload, s, lambda result: result > 0) + self._version_checks(upload, suite, s, lambda result: result > 0, 'higher') vc_older = session.query(dbconn.VersionCheck).filter_by(suite=suite, check='MustBeOlderThan') must_be_older_than = [ vc.reference for vc in vc_older ] for s in must_be_older_than: - self._version_checks(upload, s, lambda result: result < 0) + self._version_checks(upload, suite, s, lambda result: result < 0, 'lower') return True