4 Checks Debian packages from Incoming
5 @contact: Debian FTP Master <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
7 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
8 @copyright: 2009 Mark Hymers <mhy@debian.org>
9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 # based on process-unchecked and process-accepted
29 ## pu|pa: locking (daily.lock)
30 ## pu|pa: parse arguments -> list of changes files
31 ## pa: initialize urgency log
32 ## pu|pa: sort changes list
36 ## pu: copy CHG to tempdir
37 ## pu: check CHG signature
38 ## pu: parse changes file
40 ## pu: check distribution (mappings, rejects)
41 ## pu: copy FILES to tempdir
42 ## pu: check whether CHG already exists in CopyChanges
43 ## pu: check whether FILES already exist in one of the policy queues
45 ## pu: extract control information
46 ## pu: various checks on control information
47 ## pu|pa: search for source (in CHG, projectb, policy queues)
48 ## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
49 ## pu|pa: check whether deb already exists in the pool
51 ## pu: various checks on filenames and CHG consistency
52 ## pu: if isdsc: check signature
56 ## //pu: check whether file already exists in the pool
57 ## pu: store what "Component" the package is currently in
58 ## pu: check whether we found everything we were looking for in CHG
60 ## pu: check whether we need and have ONE DSC
62 ## pu: various checks //maybe drop some of the in favor of lintian
63 ## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
64 ## pu: check whether DSC_FILES is consistent with "Format"
65 ## for src in DSC_FILES:
66 ## pu|pa: check whether file already exists in the pool (with special handling for .orig.tar.gz)
67 ## pu: create new tempdir
68 ## pu: create symlink mirror of source
70 ## pu: extract changelog information for BTS
71 ## //pu: create missing .orig symlink
72 ## pu: check with lintian
74 ## pu: check checksums and sizes
75 ## for file in DSC_FILES:
76 ## pu: check checksums and sizes
77 ## pu: CHG: check urgency
79 ## pu: extract contents list and check for dubious timestamps
80 ## pu: check that the uploader is actually allowed to upload the package
82 ### if stable_install:
83 ### pa: remove from p-u
85 ### pa: move CHG to morgue
86 ### pa: append data to ChangeLog
88 ### pa: remove .dak file
90 ### pa: add dsc to db:
91 ### for file in DSC_FILES:
92 ### pa: add file to file
93 ### pa: add file to dsc_files
94 ### pa: create source entry
95 ### pa: update source associations
96 ### pa: update src_uploaders
98 ### pa: add deb to db:
99 ### pa: add file to file
100 ### pa: find source entry
101 ### pa: create binaries entry
102 ### pa: update binary associations
103 ### pa: .orig component move
104 ### pa: move files to pool
106 ### pa: move CHG to done/
107 ### pa: change entry in queue_build
108 ## pu: use dispatch table to choose target queue:
110 ## pu: write .dak file
114 ## pu: run autobyhand script
115 ## pu: if stuff left, do byhand or accept
116 ## elsif targetqueue in (oldstable, stable, embargo, unembargo):
117 ## pu: write .dak file
118 ## pu: check overrides
122 ## pu: write .dak file
123 ## pu: move to ACCEPTED
125 ## pu: create files for BTS
126 ## pu: create entry in queue_build
127 ## pu: check overrides
131 ## Parsing changes (check for duplicates)
135 # New check layout (TODO: Implement)
139 ### version checks (suite)
153 ### src relation check
156 ## Database insertion (? copy from stuff)
157 ### BYHAND / NEW / Policy queues
164 from errno import EACCES, EAGAIN
171 from sqlalchemy.orm.exc import NoResultFound
173 from daklib import daklog
174 from daklib.dbconn import *
175 from daklib.urgencylog import UrgencyLog
176 from daklib.summarystats import SummaryStats
177 from daklib.config import Config
178 import daklib.utils as utils
179 from daklib.textutils import fix_maintainer
180 from daklib.regexes import *
182 import daklib.archive
186 ###############################################################################
191 ###############################################################################
193 def usage (exit_code=0):
194 print """Usage: dak process-upload [OPTION]... [CHANGES]...
195 -a, --automatic automatic run
196 -d, --directory <DIR> process uploads in <DIR>
197 -h, --help show this help and exit.
198 -n, --no-action don't do anything
199 -p, --no-lock don't check lockfile !! for cron.daily only !!
200 -s, --no-mail don't send any mail
201 -V, --version display the version number and exit"""
204 ###############################################################################
206 def try_or_reject(function):
207 """Try to call function or reject the upload if that fails
209 def wrapper(directory, upload, *args, **kwargs):
210 reason = 'No exception caught. This should not happen.'
213 return function(directory, upload, *args, **kwargs)
214 except (daklib.archive.ArchiveException, daklib.checks.Reject) as e:
216 except Exception as e:
217 reason = "There was an uncaught exception when processing your upload:\n{0}\nAny original reject reason follows below.".format(traceback.format_exc())
221 return real_reject(directory, upload, reason=reason)
222 except Exception as e:
223 reason = "In addition there was an exception when rejecting the package:\n{0}\nPrevious reasons:\n{1}".format(traceback.format_exc(), reason)
225 return real_reject(directory, upload, reason=reason, notify=False)
227 raise Exception('Rejecting upload failed after multiple tries. Giving up. Last reason:\n{0}'.format(reason))
231 def subst_for_upload(upload):
234 changes = upload.changes
235 control = upload.changes.changes
237 if upload.final_suites is None or len(upload.final_suites) == 0:
238 suite_name = '(unknown)'
241 for suite in upload.final_suites:
242 if suite.policy_queue:
243 suite_names.append("{0}->{1}".format(suite.suite_name, suite.policy_queue.queue_name))
245 suite_names.append(suite.suite_name)
246 suite_name = ','.join(suite_names)
248 maintainer_field = control.get('Maintainer', cnf['Dinstall::MyEmailAddress'])
249 changed_by_field = control.get('Changed-By', maintainer_field)
250 maintainer = fix_maintainer(changed_by_field)
251 if upload.changes.source is not None:
252 addresses = utils.mail_addresses_for_upload(maintainer_field, changed_by_field, changes.primary_fingerprint)
254 addresses = utils.mail_addresses_for_upload(maintainer_field, maintainer_field, changes.primary_fingerprint)
256 # debian-{devel-,}-changes@lists.debian.org toggles writes access based on this header
257 bcc = 'X-DAK: dak process-upload'
258 if 'Dinstall::Bcc' in cnf:
259 bcc = '{0}\nBcc: {1}'.format(bcc, cnf['Dinstall::Bcc'])
262 '__DISTRO__': cnf['Dinstall::MyDistribution'],
263 '__ADMIN_ADDRESS__': cnf['Dinstall::MyAdminAddress'],
265 '__CHANGES_FILENAME__': upload.changes.filename,
267 '__SOURCE__': control.get('Source', '(unknown)'),
268 '__ARCHITECTURE__': control.get('Architecture', '(unknown)'),
269 '__VERSION__': control.get('Version', '(unknown)'),
271 '__SUITE__': suite_name,
273 '__DAK_ADDRESS__': cnf['Dinstall::MyEmailAddress'],
274 '__MAINTAINER_FROM__': maintainer[1],
275 '__MAINTAINER_TO__': ", ".join(addresses),
276 '__MAINTAINER__': changed_by_field,
279 '__BUG_SERVER__': cnf.get('Dinstall::BugServer'),
281 '__FILE_CONTENTS__': open(upload.changes.path, 'r').read(),
284 override_maintainer = cnf.get('Dinstall::OverrideMaintainer')
285 if override_maintainer:
286 subst['__MAINTAINER_TO__'] = subst['__MAINTAINER_FROM__'] = override_maintainer
291 def accept(directory, upload):
294 Logger.log(['ACCEPT', upload.changes.filename])
298 accepted_to_real_suite = False
299 for suite in upload.final_suites:
300 accepted_to_real_suite = accepted_to_real_suite or suite.policy_queue is None
302 sourceful_upload = 'source' in upload.changes.architectures
304 control = upload.changes.changes
305 if sourceful_upload and not Options['No-Action']:
306 urgency = control.get('Urgency')
307 if urgency not in cnf.value_list('Urgency::Valid'):
308 urgency = cnf['Urgency::Default']
309 UrgencyLog().log(control['Source'], control['Version'], urgency)
311 # send mail to maintainer
312 subst = subst_for_upload(upload)
313 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.accepted'))
314 utils.send_mail(message)
316 # send mail to announce lists and tracking server
317 if accepted_to_real_suite and sourceful_upload:
318 subst = subst_for_upload(upload)
320 for suite in upload.final_suites:
321 if suite.policy_queue is not None:
323 announce.update(suite.announce or [])
324 announce_address = ", ".join(announce)
326 tracking = cnf.get('Dinstall::TrackingServer')
327 if tracking and 'source' in upload.changes.architectures:
328 announce_address = '{0}\nBcc: {1}@{2}'.format(announce_address, control['Source'], tracking)
330 subst['__ANNOUNCE_LIST_ADDRESS__'] = announce_address
332 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.announce'))
333 utils.send_mail(message)
335 # Only close bugs for uploads that were not redirected to a policy queue.
336 # process-policy will close bugs for those once they are accepted.
337 subst = subst_for_upload(upload)
338 if accepted_to_real_suite and cnf.find_b('Dinstall::CloseBugs') and sourceful_upload:
339 for bugnum in upload.changes.closed_bugs:
340 subst['__BUG_NUMBER__'] = str(bugnum)
342 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.bug-close'))
343 utils.send_mail(message)
345 del subst['__BUG_NUMBER__']
347 # Move .changes to done, but only for uploads that were accepted to a
348 # real suite. process-policy will handle this for uploads to queues.
349 if accepted_to_real_suite:
350 src = os.path.join(upload.directory, upload.changes.filename)
352 now = datetime.datetime.now()
353 donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
354 dst = os.path.join(donedir, upload.changes.filename)
355 dst = utils.find_next_free(dst)
357 upload.transaction.fs.copy(src, dst, mode=0o644)
359 SummaryStats().accept_count += 1
360 SummaryStats().accept_bytes += upload.changes.bytes
363 def accept_to_new(directory, upload):
366 Logger.log(['ACCEPT-TO-NEW', upload.changes.filename])
368 upload.install_to_new()
369 # TODO: tag bugs pending
371 subst = subst_for_upload(upload)
372 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.new'))
373 utils.send_mail(message)
375 SummaryStats().accept_count += 1
376 SummaryStats().accept_bytes += upload.changes.bytes
379 def reject(directory, upload, reason=None, notify=True):
380 real_reject(directory, upload, reason, notify)
382 def real_reject(directory, upload, reason=None, notify=True):
383 # XXX: rejection itself should go to daklib.archive.ArchiveUpload
386 Logger.log(['REJECT', upload.changes.filename])
388 fs = upload.transaction.fs
389 rejectdir = cnf['Dir::Reject']
391 files = [ f.filename for f in upload.changes.files.itervalues() ]
392 files.append(upload.changes.filename)
395 src = os.path.join(upload.directory, fn)
396 dst = utils.find_next_free(os.path.join(rejectdir, fn))
397 if not os.path.exists(src):
401 if upload.reject_reasons is not None:
404 reason = reason + '\n' + '\n'.join(upload.reject_reasons)
407 reason = '(Unknown reason. Please check logs.)'
409 dst = utils.find_next_free(os.path.join(rejectdir, '{0}.reason'.format(upload.changes.filename)))
415 subst = subst_for_upload(upload)
416 subst['__REJECTOR_ADDRESS__'] = cnf['Dinstall::MyEmailAddress']
417 subst['__MANUAL_REJECT_MESSAGE__'] = ''
418 subst['__REJECT_MESSAGE__'] = reason
419 subst['__CC__'] = 'X-DAK-Rejection: automatic (moo)'
421 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'queue.rejected'))
422 utils.send_mail(message)
424 SummaryStats().reject_count += 1
426 ###############################################################################
428 def action(directory, upload):
429 changes = upload.changes
436 okay = upload.check()
438 summary = changes.changes.get('Changes', '')
442 if changes.source is not None:
443 package_info.append("source:{0}".format(changes.source.dsc['Source']))
444 for binary in changes.binaries:
445 package_info.append("binary:{0}".format(binary.control['Package']))
447 (prompt, answer) = ("", "XXX")
448 if Options["No-Action"] or Options["Automatic"]:
455 print "\n".join(package_info)
458 if len(upload.reject_reasons) > 0:
460 print "\n".join(upload.reject_reasons)
463 path = os.path.join(directory, changes.filename)
464 created = os.stat(path).st_mtime
466 too_new = (now - created < int(cnf['Dinstall::SkipTime']))
469 print "SKIP (too new)"
470 prompt = "[S]kip, Quit ?"
472 prompt = "[R]eject, Skip, Quit ?"
473 if Options["Automatic"]:
476 prompt = "[N]ew, Skip, Quit ?"
477 if Options['Automatic']:
480 prompt = "[A]ccept, Skip, Quit ?"
481 if Options['Automatic']:
484 while prompt.find(answer) == -1:
485 answer = utils.our_raw_input(prompt)
486 m = re_default_answer.match(prompt)
489 answer = answer[:1].upper()
492 reject(directory, upload)
494 # upload.try_autobyhand must not be run with No-Action.
495 if Options['No-Action']:
496 accept(directory, upload)
497 elif upload.try_autobyhand():
498 accept(directory, upload)
500 print "W: redirecting to BYHAND as automatic processing failed."
501 accept_to_new(directory, upload)
503 accept_to_new(directory, upload)
509 if not Options['No-Action']:
514 ###############################################################################
516 def unlink_if_exists(path):
520 if e.errno != errno.ENOENT:
523 def process_it(directory, changes, keyrings, session):
526 print "\n{0}\n".format(changes.filename)
527 Logger.log(["Processing changes file", changes.filename])
529 with daklib.archive.ArchiveUpload(directory, changes, keyrings) as upload:
530 processed = action(directory, upload)
531 if processed and not Options['No-Action']:
532 unlink_if_exists(os.path.join(directory, changes.filename))
533 for fn in changes.files:
534 unlink_if_exists(os.path.join(directory, fn))
536 ###############################################################################
538 def process_changes(changes_filenames):
539 session = DBConn().session()
540 keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority)
541 keyring_files = [ k.keyring_name for k in keyrings ]
544 for fn in changes_filenames:
546 directory, filename = os.path.split(fn)
547 c = daklib.upload.Changes(directory, filename, keyring_files)
548 changes.append([directory, c])
549 except Exception as e:
550 Logger.log([filename, "Error while loading changes: {0}".format(e)])
552 changes.sort(key=lambda x: x[1])
554 for directory, c in changes:
555 process_it(directory, c, keyring_files, session)
559 ###############################################################################
562 global Options, Logger
565 summarystats = SummaryStats()
567 Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
568 ('h',"help","Dinstall::Options::Help"),
569 ('n',"no-action","Dinstall::Options::No-Action"),
570 ('p',"no-lock", "Dinstall::Options::No-Lock"),
571 ('s',"no-mail", "Dinstall::Options::No-Mail"),
572 ('d',"directory", "Dinstall::Options::Directory", "HasArg")]
574 for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
575 "version", "directory"]:
576 if not cnf.has_key("Dinstall::Options::%s" % (i)):
577 cnf["Dinstall::Options::%s" % (i)] = ""
579 changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
580 Options = cnf.subtree("Dinstall::Options")
585 # -n/--dry-run invalidates some other options which would involve things happening
586 if Options["No-Action"]:
587 Options["Automatic"] = ""
589 # Check that we aren't going to clash with the daily cron job
590 if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (cnf["Dir::Lock"])) and not Options["No-Lock"]:
591 utils.fubar("Archive maintenance in progress. Try again later.")
593 # Obtain lock if not in no-action mode and initialize the log
594 if not Options["No-Action"]:
595 lock_fd = os.open(os.path.join(cnf["Dir::Lock"], 'dinstall.lock'), os.O_RDWR | os.O_CREAT)
597 fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
599 if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN':
600 utils.fubar("Couldn't obtain lock; assuming another 'dak process-upload' is already running.")
604 # Initialise UrgencyLog() - it will deal with the case where we don't
605 # want to log urgencies
606 urgencylog = UrgencyLog()
608 Logger = daklog.Logger("process-upload", Options["No-Action"])
610 # If we have a directory flag, use it to find our files
611 if cnf["Dinstall::Options::Directory"] != "":
612 # Note that we clobber the list of files we were given in this case
613 # so warn if the user has done both
614 if len(changes_files) > 0:
615 utils.warn("Directory provided so ignoring files given on command line")
617 changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
618 Logger.log(["Using changes files from directory", cnf["Dinstall::Options::Directory"], len(changes_files)])
619 elif not len(changes_files) > 0:
620 utils.fubar("No changes files given and no directory specified")
622 Logger.log(["Using changes files from command-line", len(changes_files)])
624 process_changes(changes_files)
626 if summarystats.accept_count:
628 if summarystats.accept_count > 1:
630 print "Installed %d package %s, %s." % (summarystats.accept_count, sets,
631 utils.size_type(int(summarystats.accept_bytes)))
632 Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
634 if summarystats.reject_count:
636 if summarystats.reject_count > 1:
638 print "Rejected %d package %s." % (summarystats.reject_count, sets)
639 Logger.log(["rejected", summarystats.reject_count])
641 if not Options["No-Action"]:
646 ###############################################################################
648 if __name__ == '__main__':