1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
3 # Parts based on code that is
4 # Copyright (C) 2001-2006, James Troup <james@nocrew.org>
5 # Copyright (C) 2009-2010, Joerg Jaspert <joerg@debian.org>
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License along
18 # with this program; if not, write to the Free Software Foundation, Inc.,
19 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21 """module provided pre-acceptance tests
23 Please read the documentation for the `Check` class for the interface.
26 from daklib.config import Config
28 import daklib.dbconn as dbconn
29 from .regexes import *
30 from .textutils import fix_maintainer, ParseMaintError
31 import daklib.lintian as lintian
32 import daklib.utils as utils
35 from apt_pkg import version_compare
39 # TODO: replace by subprocess
42 class Reject(Exception):
43 """exception raised by failing checks"""
47 """base class for checks
49 checks are called by daklib.archive.ArchiveUpload. Failing tests should
50 raise a `daklib.checks.Reject` exception including a human-readable
51 description why the upload should be rejected.
53 def check(self, upload):
57 upload (daklib.archive.ArchiveUpload): upload to check
63 def per_suite_check(self, upload, suite):
64 """do per-suite checks
67 upload (daklib.archive.ArchiveUpload): upload to check
68 suite (daklib.dbconn.Suite): suite to check
76 """allow to force ignore failing test
78 True if it is acceptable to force ignoring a failing test,
83 class SignatureCheck(Check):
84 """Check signature of changes and dsc file (if included in upload)
86 Make sure the signature is valid and done by a known user.
88 def check(self, upload):
89 changes = upload.changes
90 if not changes.valid_signature:
91 raise Reject("Signature for .changes not valid.")
92 if changes.source is not None:
93 if not changes.source.valid_signature:
94 raise Reject("Signature for .dsc not valid.")
95 if changes.source.primary_fingerprint != changes.primary_fingerprint:
96 raise Reject(".changes and .dsc not signed by the same key.")
97 if upload.fingerprint is None or upload.fingerprint.uid is None:
98 raise Reject(".changes signed by unknown key.")
100 class ChangesCheck(Check):
101 """Check changes file for syntax errors."""
102 def check(self, upload):
103 changes = upload.changes
104 control = changes.changes
105 fn = changes.filename
107 for field in ('Distribution', 'Source', 'Binary', 'Architecture', 'Version', 'Maintainer', 'Files', 'Changes', 'Description'):
108 if field not in control:
109 raise Reject('{0}: misses mandatory field {1}'.format(fn, field))
111 source_match = re_field_source.match(control['Source'])
113 raise Reject('{0}: Invalid Source field'.format(fn))
114 version_match = re_field_version.match(control['Version'])
115 if not version_match:
116 raise Reject('{0}: Invalid Version field'.format(fn))
117 version_without_epoch = version_match.group('without_epoch')
119 match = re_file_changes.match(fn)
121 raise Reject('{0}: Does not match re_file_changes'.format(fn))
122 if match.group('package') != source_match.group('package'):
123 raise Reject('{0}: Filename does not match Source field'.format(fn))
124 if match.group('version') != version_without_epoch:
125 raise Reject('{0}: Filename does not match Version field'.format(fn))
127 for bn in changes.binary_names:
128 if not re_field_package.match(bn):
129 raise Reject('{0}: Invalid binary package name {1}'.format(fn, bn))
131 if 'source' in changes.architectures and changes.source is None:
132 raise Reject("Changes has architecture source, but no source found.")
133 if changes.source is not None and 'source' not in changes.architectures:
134 raise Reject("Upload includes source, but changes does not say so.")
137 fix_maintainer(changes.changes['Maintainer'])
138 except ParseMaintError as e:
139 raise Reject('{0}: Failed to parse Maintainer field: {1}'.format(changes.filename, e))
142 changed_by = changes.changes.get('Changed-By')
143 if changed_by is not None:
144 fix_maintainer(changed_by)
145 except ParseMaintError as e:
146 raise Reject('{0}: Failed to parse Changed-By field: {1}'.format(changes.filename, e))
148 if len(changes.files) == 0:
149 raise Reject("Changes includes no files.")
151 for bugnum in changes.closed_bugs:
152 if not re_isanum.match(bugnum):
153 raise Reject('{0}: "{1}" in Closes field is not a number'.format(changes.filename, bugnum))
157 class HashesCheck(Check):
158 """Check hashes in .changes and .dsc are valid."""
159 def check(self, upload):
160 changes = upload.changes
161 for f in changes.files.itervalues():
162 f.check(upload.directory)
163 source = changes.source
164 if source is not None:
165 for f in source.files.itervalues():
166 f.check(upload.directory)
168 class BinaryCheck(Check):
169 """Check binary packages for syntax errors."""
170 def check(self, upload):
171 for binary in upload.changes.binaries:
172 self.check_binary(upload, binary)
174 binary_names = set([ binary.control['Package'] for binary in upload.changes.binaries ])
175 for bn in binary_names:
176 if bn not in upload.changes.binary_names:
177 raise Reject('Package {0} is not mentioned in Binary field in changes'.format(bn))
178 for bn in upload.changes.binary_names:
179 if bn not in binary_names:
180 raise Reject('Binary field in changes has {0}, but was not found in upload'.format(bn))
184 def check_binary(self, upload, binary):
185 fn = binary.hashed_file.filename
186 control = binary.control
188 for field in ('Package', 'Architecture', 'Version', 'Description'):
189 if field not in control:
190 raise Reject('{0}: Missing mandatory field {0}.'.format(fn, field))
194 package = control['Package']
195 if not re_field_package.match(package):
196 raise Reject('{0}: Invalid Package field'.format(fn))
198 version = control['Version']
199 version_match = re_field_version.match(version)
200 if not version_match:
201 raise Reject('{0}: Invalid Version field'.format(fn))
202 version_without_epoch = version_match.group('without_epoch')
204 architecture = control['Architecture']
205 if architecture not in upload.changes.architectures:
206 raise Reject('{0}: Architecture not in Architecture field in changes file'.format(fn))
207 if architecture == 'source':
208 raise Reject('{0}: Architecture "source" invalid for binary packages'.format(fn))
210 source = control.get('Source')
211 if source is not None and not re_field_source.match(source):
212 raise Reject('{0}: Invalid Source field'.format(fn))
216 match = re_file_binary.match(fn)
217 if package != match.group('package'):
218 raise Reject('{0}: filename does not match Package field'.format(fn))
219 if version_without_epoch != match.group('version'):
220 raise Reject('{0}: filename does not match Version field'.format(fn))
221 if architecture != match.group('architecture'):
222 raise Reject('{0}: filename does not match Architecture field'.format(fn))
224 # check dependency field syntax
226 for field in ('Breaks', 'Conflicts', 'Depends', 'Enhances', 'Pre-Depends',
227 'Provides', 'Recommends', 'Replaces', 'Suggests'):
228 value = control.get(field)
229 if value is not None:
230 if value.strip() == '':
231 raise Reject('{0}: empty {1} field'.format(fn, field))
233 apt_pkg.parse_depends(value)
235 raise Reject('{0}: APT could not parse {1} field'.format(fn, field))
237 for field in ('Built-Using',):
238 value = control.get(field)
239 if value is not None:
240 if value.strip() == '':
241 raise Reject('{0}: empty {1} field'.format(fn, field))
243 apt_pkg.parse_src_depends(value)
245 raise Reject('{0}: APT could not parse {1} field'.format(fn, field))
247 class SourceCheck(Check):
248 """Check source package for syntax errors."""
249 def check_filename(self, control, filename, regex):
250 # In case we have an .orig.tar.*, we have to strip the Debian revison
251 # from the version number. So handle this special case first.
253 match = re_file_orig.match(filename)
256 match = regex.match(filename)
259 raise Reject('{0}: does not match regular expression for source filenames'.format(filename))
260 if match.group('package') != control['Source']:
261 raise Reject('{0}: filename does not match Source field'.format(filename))
263 version = control['Version']
265 version = re_field_version_upstream.match(version).group('upstream')
266 version_match = re_field_version.match(version)
267 version_without_epoch = version_match.group('without_epoch')
268 if match.group('version') != version_without_epoch:
269 raise Reject('{0}: filename does not match Version field'.format(filename))
271 def check(self, upload):
272 if upload.changes.source is None:
275 changes = upload.changes.changes
276 source = upload.changes.source
278 dsc_fn = source._dsc_file.filename
281 if not re_field_package.match(control['Source']):
282 raise Reject('{0}: Invalid Source field'.format(dsc_fn))
283 if control['Source'] != changes['Source']:
284 raise Reject('{0}: Source field does not match Source field in changes'.format(dsc_fn))
285 if control['Version'] != changes['Version']:
286 raise Reject('{0}: Version field does not match Version field in changes'.format(dsc_fn))
289 self.check_filename(control, dsc_fn, re_file_dsc)
290 for f in source.files.itervalues():
291 self.check_filename(control, f.filename, re_file_source)
293 # check dependency field syntax
294 for field in ('Build-Conflicts', 'Build-Conflicts-Indep', 'Build-Depends', 'Build-Depends-Arch', 'Build-Depends-Indep'):
295 value = control.get(field)
296 if value is not None:
297 if value.strip() == '':
298 raise Reject('{0}: empty {1} field'.format(dsc_fn, field))
300 apt_pkg.parse_src_depends(value)
301 except Exception as e:
302 raise Reject('{0}: APT could not parse {1} field: {2}'.format(dsc_fn, field, e))
304 # TODO: check all expected files for given source format are included
306 class SingleDistributionCheck(Check):
307 """Check that the .changes targets only a single distribution."""
308 def check(self, upload):
309 if len(upload.changes.distributions) != 1:
310 raise Reject("Only uploads to a single distribution are allowed.")
312 class ACLCheck(Check):
313 """Check the uploader is allowed to upload the packages in .changes"""
314 def _check_dm(self, upload):
315 # This code is not very nice, but hopefully works until we can replace
316 # DM-Upload-Allowed, cf. https://lists.debian.org/debian-project/2012/06/msg00029.html
317 session = upload.session
319 if 'source' not in upload.changes.architectures:
320 raise Reject('DM uploads must include source')
321 distributions = upload.changes.distributions
322 for dist in distributions:
323 if dist not in ('unstable', 'experimental', 'squeeze-backports'):
324 raise Reject("Uploading to {0} is not allowed for DMs.".format(dist))
325 for f in upload.changes.files.itervalues():
326 if f.section == 'byhand' or f.section[:4] == "raw-":
327 raise Reject("Uploading byhand packages is not allowed for DMs.")
329 # Reject NEW packages
330 assert len(distributions) == 1
331 suite = session.query(Suite).filter_by(suite_name=distributions[0]).one()
332 overridesuite = suite
333 if suite.overridesuite is not None:
334 overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
335 if upload._check_new(overridesuite):
336 raise Reject('Uploading NEW packages is not allowed for DMs.')
338 # Check DM-Upload-Allowed
339 last_suites = ['unstable', 'experimental']
340 if suite.suite_name.endswith('-backports'):
341 last_suites = [suite.suite_name]
342 last = session.query(DBSource).filter_by(source=upload.changes.changes['Source']) \
343 .join(DBSource.suites).filter(Suite.suite_name.in_(last_suites)) \
344 .order_by(DBSource.version.desc()).limit(1).first()
346 raise Reject('No existing source found in {0}'.format(' or '.join(last_suites)))
347 if not last.dm_upload_allowed:
348 raise Reject('DM-Upload-Allowed is not set in {0}={1}'.format(last.source, last.version))
350 # check current Changed-by is in last Maintainer or Uploaders
351 uploader_names = [ u.name for u in last.uploaders ]
352 changed_by_field = upload.changes.changes.get('Changed-By', upload.changes.changes['Maintainer'])
353 if changed_by_field not in uploader_names:
354 raise Reject('{0} is not an uploader for {1}={2}'.format(changed_by_field, last.source, last.version))
356 # check Changed-by is the DM
357 changed_by = fix_maintainer(changed_by_field)
358 uid = upload.fingerprint.uid
360 raise Reject('Unknown uid for fingerprint {0}'.format(upload.fingerprint.fingerprint))
361 if uid.uid != changed_by[3] and uid.name != changed_by[2]:
362 raise Reject('DMs are not allowed to sponsor uploads (expected {0} <{1}> as maintainer, but got {2})'.format(uid.name, uid.uid, changed_by_field))
364 # Try to catch hijacks.
365 # This doesn't work correctly. Uploads to experimental can still
366 # "hijack" binaries from unstable. Also one can hijack packages
367 # via buildds (but people who try this should not be DMs).
368 for binary_name in upload.changes.binary_names:
369 binaries = session.query(DBBinary).join(DBBinary.source) \
370 .join(DBBinary.suites).filter(Suite.suite_name.in_(upload.changes.distributions)) \
371 .filter(DBBinary.package == binary_name)
372 for binary in binaries:
373 if binary.source.source != upload.changes.changes['Source']:
374 raise Reject('DMs must not hijack binaries (binary={0}, other-source={1})'.format(binary_name, binary.source.source))
378 def check(self, upload):
379 fingerprint = upload.fingerprint
380 source_acl = fingerprint.source_acl
381 if source_acl is None:
382 if 'source' in upload.changes.architectures:
383 raise Reject('Fingerprint {0} must not upload source'.format(fingerprint.fingerprint))
384 elif source_acl.access_level == 'dm':
385 self._check_dm(upload)
386 elif source_acl.access_level != 'full':
387 raise Reject('Unknown source_acl access level {0} for fingerprint {1}'.format(source_acl.access_level, fingerprint.fingerprint))
389 bin_architectures = set(upload.changes.architectures)
390 bin_architectures.discard('source')
391 binary_acl = fingerprint.binary_acl
392 if binary_acl is None:
393 if len(bin_architectures) > 0:
394 raise Reject('Fingerprint {0} must not upload binary packages'.format(fingerprint.fingerprint))
395 elif binary_acl.access_level == 'map':
396 query = upload.session.query(BinaryACLMap).filter_by(fingerprint=fingerprint)
397 allowed_architectures = [ m.architecture.arch_string for m in query ]
399 for arch in upload.changes.architectures:
400 if arch not in allowed_architectures:
401 raise Reject('Fingerprint {0} must not upload binaries for architecture {1}'.format(fingerprint.fingerprint, arch))
402 elif binary_acl.access_level != 'full':
403 raise Reject('Unknown binary_acl access level {0} for fingerprint {1}'.format(binary_acl.access_level, fingerprint.fingerprint))
407 class NoSourceOnlyCheck(Check):
408 """Check for source-only upload
410 Source-only uploads are only allowed if Dinstall::AllowSourceOnlyUploads is
411 set. Otherwise they are rejected.
413 def check(self, upload):
414 if Config().find_b("Dinstall::AllowSourceOnlyUploads"):
416 changes = upload.changes
417 if changes.source is not None and len(changes.binaries) == 0:
418 raise Reject('Source-only uploads are not allowed.')
421 class LintianCheck(Check):
422 """Check package using lintian"""
423 def check(self, upload):
424 changes = upload.changes
426 # Only check sourceful uploads.
427 if changes.source is None:
429 # Only check uploads to unstable or experimental.
430 if 'unstable' not in changes.distributions and 'experimental' not in changes.distributions:
434 if 'Dinstall::LintianTags' not in cnf:
436 tagfile = cnf['Dinstall::LintianTags']
438 with open(tagfile, 'r') as sourcefile:
439 sourcecontent = sourcefile.read()
441 lintiantags = yaml.load(sourcecontent)['lintian']
442 except yaml.YAMLError as msg:
443 raise Exception('Could not read lintian tags file {0}, YAML error: {1}'.format(tagfile, msg))
445 fd, temp_filename = utils.temp_filename()
446 temptagfile = os.fdopen(fd, 'w')
447 for tags in lintiantags.itervalues():
449 print >>temptagfile, tag
452 changespath = os.path.join(upload.directory, changes.filename)
455 cmd = "lintian --show-overrides --tags-from-file {0} {1}".format(temp_filename, changespath)
456 result, output = commands.getstatusoutput(cmd)
458 os.unlink(temp_filename)
461 utils.warn("lintian failed for %s [return code: %s]." % \
462 (changespath, result))
463 utils.warn(utils.prefix_multi_line_string(output, \
464 " [possible output:] "))
466 parsed_tags = lintian.parse_lintian_output(output)
467 rejects = list(lintian.generate_reject_messages(parsed_tags, lintiantags))
468 if len(rejects) != 0:
469 raise Reject('\n'.join(rejects))
473 class SourceFormatCheck(Check):
474 """Check source format is allowed in the target suite"""
475 def per_suite_check(self, upload, suite):
476 source = upload.changes.source
477 session = upload.session
481 source_format = source.dsc['Format']
482 query = session.query(SrcFormat).filter_by(format_name=source_format).filter(SrcFormat.suites.contains(suite))
483 if query.first() is None:
484 raise Reject('source format {0} is not allowed in suite {1}'.format(source_format, suite.suite_name))
486 class SuiteArchitectureCheck(Check):
487 def per_suite_check(self, upload, suite):
488 session = upload.session
489 for arch in upload.changes.architectures:
490 query = session.query(Architecture).filter_by(arch_string=arch).filter(Architecture.suites.contains(suite))
491 if query.first() is None:
492 raise Reject('Architecture {0} is not allowed in suite {2}'.format(arch, suite.suite_name))
496 class VersionCheck(Check):
497 """Check version constraints"""
498 def _highest_source_version(self, session, source_name, suite):
499 db_source = session.query(DBSource).filter_by(source=source_name) \
500 .filter(DBSource.suites.contains(suite)).order_by(DBSource.version.desc()).first()
501 if db_source is None:
504 return db_source.version
506 def _highest_binary_version(self, session, binary_name, suite, architecture):
507 db_binary = session.query(DBBinary).filter_by(package=binary_name) \
508 .filter(DBBinary.suites.contains(suite)) \
509 .filter(Architecture.arch_string.in_(['all', architecture])) \
510 .order_by(DBBinary.version.desc()).first()
511 if db_binary is None:
514 return db_binary.version
516 def _version_checks(self, upload, suite, expected_result):
517 session = upload.session
519 if upload.changes.source is not None:
520 source_name = upload.changes.source.dsc['Source']
521 source_version = upload.changes.source.dsc['Version']
522 v = self._highest_source_version(session, source_name, suite)
523 if v is not None and version_compare(source_version, v) != expected_result:
524 raise Reject('Version check failed (source={0}, version={1}, suite={2})'.format(source_name, source_version, suite.suite_name))
526 for binary in upload.changes.binaries:
527 binary_name = binary.control['Package']
528 binary_version = binary.control['Version']
529 architecture = binary.control['Architecture']
530 v = self._highest_binary_version(session, binary_name, suite, architecture)
531 if v is not None and version_compare(binary_version, v) != expected_result:
532 raise Reject('Version check failed (binary={0}, version={1}, suite={2})'.format(binary_name, binary_version, suite.suite_name))
534 def per_suite_check(self, upload, suite):
535 session = upload.session
537 vc_newer = session.query(dbconn.VersionCheck).filter_by(suite=suite) \
538 .filter(dbconn.VersionCheck.check.in_(['MustBeNewerThan', 'Enhances']))
539 must_be_newer_than = [ vc.reference for vc in vc_newer ]
540 # Must be newer than old versions in `suite`
541 must_be_newer_than.append(suite)
543 for s in must_be_newer_than:
544 self._version_checks(upload, s, 1)
546 vc_older = session.query(dbconn.VersionCheck).filter_by(suite=suite, check='MustBeOlderThan')
547 must_be_older_than = [ vc.reference for vc in vc_older ]
549 for s in must_be_older_than:
550 self._version_checks(upload, s, -1)