2 # vim:set et ts=4 sw=4:
4 """ Handles NEW and BYHAND packages
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
10 @license: GNU General Public License version 2 or later
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License as published by
14 # the Free Software Foundation; either version 2 of the License, or
15 # (at your option) any later version.
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # GNU General Public License for more details.
22 # You should have received a copy of the GNU General Public License
23 # along with this program; if not, write to the Free Software
24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 ################################################################################
28 # 23:12|<aj> I will not hush!
30 # 23:12|<aj> Where there is injustice in the world, I shall be there!
31 # 23:13|<aj> I shall not be silenced!
32 # 23:13|<aj> The world shall know!
33 # 23:13|<aj> The world *must* know!
34 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
35 # 23:13|<aj> yay powerpuff girls!!
36 # 23:13|<aj> buttercup's my favourite, who's yours?
37 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
38 # 23:14|<aj> *AREN'T YOU*?!
39 # 23:15|<aj> I will not be treated like this.
40 # 23:15|<aj> I shall have my revenge.
41 # 23:15|<aj> I SHALL!!!
43 ################################################################################
45 from __future__ import with_statement
56 import apt_pkg, apt_inst
57 import examine_package
59 from daklib.dbconn import *
60 from daklib.queue import *
61 from daklib import daklog
62 from daklib import utils
63 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum, re_package
64 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
65 from daklib.summarystats import SummaryStats
66 from daklib.config import Config
67 from daklib.changesutils import *
76 ################################################################################
77 ################################################################################
78 ################################################################################
80 def recheck(upload, session):
81 # STU: I'm not sure, but I don't thin kthis is necessary any longer: upload.recheck(session)
82 if len(upload.rejects) > 0:
84 if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
87 print "REJECT\n%s" % '\n'.join(upload.rejects)
88 prompt = "[R]eject, Skip, Quit ?"
90 while prompt.find(answer) == -1:
91 answer = utils.our_raw_input(prompt)
92 m = re_default_answer.match(prompt)
95 answer = answer[:1].upper()
98 upload.do_reject(manual=0, reject_message='\n'.join(upload.rejects))
99 upload.pkg.remove_known_changes(session=session)
110 ################################################################################
112 class Section_Completer:
113 def __init__ (self, session):
116 for s, in session.query(Section.section):
117 self.sections.append(s)
119 def complete(self, text, state):
123 for word in self.sections:
125 self.matches.append(word)
127 return self.matches[state]
131 ############################################################
133 class Priority_Completer:
134 def __init__ (self, session):
137 for p, in session.query(Priority.priority):
138 self.priorities.append(p)
140 def complete(self, text, state):
144 for word in self.priorities:
146 self.matches.append(word)
148 return self.matches[state]
152 ################################################################################
154 def print_new (new, upload, indexed, file=sys.stdout):
158 for pkg in new.keys():
160 section = new[pkg]["section"]
161 priority = new[pkg]["priority"]
162 if new[pkg]["section id"] == -1:
165 if new[pkg]["priority id"] == -1:
169 line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
171 line = "%-20s %-20s %-20s" % (pkg, priority, section)
172 line = line.strip()+'\n'
174 notes = get_new_comments(upload.pkg.changes.get("source"))
176 print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
177 % (note.author, note.version, note.notedate, note.comment)
179 return broken, len(notes) > 0
181 ################################################################################
183 def index_range (index):
187 return "1-%s" % (index)
189 ################################################################################
190 ################################################################################
192 def edit_new (new, upload):
193 # Write the current data to a temporary file
194 (fd, temp_filename) = utils.temp_filename()
195 temp_file = os.fdopen(fd, 'w')
196 print_new (new, upload, indexed=0, file=temp_file)
198 # Spawn an editor on that file
199 editor = os.environ.get("EDITOR","vi")
200 result = os.system("%s %s" % (editor, temp_filename))
202 utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
203 # Read the edited data back in
204 temp_file = utils.open_file(temp_filename)
205 lines = temp_file.readlines()
207 os.unlink(temp_filename)
214 # Pad the list if necessary
215 s[len(s):3] = [None] * (3-len(s))
216 (pkg, priority, section) = s[:3]
217 if not new.has_key(pkg):
218 utils.warn("Ignoring unknown package '%s'" % (pkg))
220 # Strip off any invalid markers, print_new will readd them.
221 if section.endswith("[!]"):
222 section = section[:-3]
223 if priority.endswith("[!]"):
224 priority = priority[:-3]
225 for f in new[pkg]["files"]:
226 upload.pkg.files[f]["section"] = section
227 upload.pkg.files[f]["priority"] = priority
228 new[pkg]["section"] = section
229 new[pkg]["priority"] = priority
231 ################################################################################
233 def edit_index (new, upload, index):
234 priority = new[index]["priority"]
235 section = new[index]["section"]
236 ftype = new[index]["type"]
239 print "\t".join([index, priority, section])
243 prompt = "[B]oth, Priority, Section, Done ? "
245 prompt = "[S]ection, Done ? "
246 edit_priority = edit_section = 0
248 while prompt.find(answer) == -1:
249 answer = utils.our_raw_input(prompt)
250 m = re_default_answer.match(prompt)
253 answer = answer[:1].upper()
260 edit_priority = edit_section = 1
266 readline.set_completer(Priorities.complete)
268 while not got_priority:
269 new_priority = utils.our_raw_input("New priority: ").strip()
270 if new_priority not in Priorities.priorities:
271 print "E: '%s' is not a valid priority, try again." % (new_priority)
274 priority = new_priority
278 readline.set_completer(Sections.complete)
280 while not got_section:
281 new_section = utils.our_raw_input("New section: ").strip()
282 if new_section not in Sections.sections:
283 print "E: '%s' is not a valid section, try again." % (new_section)
286 section = new_section
288 # Reset the readline completer
289 readline.set_completer(None)
291 for f in new[index]["files"]:
292 upload.pkg.files[f]["section"] = section
293 upload.pkg.files[f]["priority"] = priority
294 new[index]["priority"] = priority
295 new[index]["section"] = section
298 ################################################################################
300 def edit_overrides (new, upload, session):
304 print_new (new, upload, indexed=1)
311 prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
314 while not got_answer:
315 answer = utils.our_raw_input(prompt)
316 if not answer.isdigit():
317 answer = answer[:1].upper()
318 if answer == "E" or answer == "D":
320 elif re_isanum.match (answer):
322 if (answer < 1) or (answer > index):
323 print "%s is not a valid index (%s). Please retry." % (answer, index_range(index))
328 edit_new(new, upload)
332 edit_index (new, upload, new_index[answer])
337 ################################################################################
339 def check_pkg (upload):
341 less_fd = os.popen("less -R -", 'w', 0)
342 stdout_fd = sys.stdout
345 changes = utils.parse_changes (upload.pkg.changes_file)
346 print examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
347 files = upload.pkg.files
348 for f in files.keys():
349 if files[f].has_key("new"):
350 ftype = files[f]["type"]
352 print examine_package.check_deb(changes['distribution'], f)
354 print examine_package.check_dsc(changes['distribution'], f)
356 print examine_package.output_package_relations()
357 sys.stdout = stdout_fd
359 if e.errno == errno.EPIPE:
360 utils.warn("[examine_package] Caught EPIPE; skipping.")
364 except KeyboardInterrupt:
365 utils.warn("[examine_package] Caught C-c; skipping.")
368 ################################################################################
370 ## FIXME: horribly Debian specific
372 def do_bxa_notification(upload):
373 files = upload.pkg.files
375 for f in files.keys():
376 if files[f]["type"] == "deb":
377 control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
379 summary += "Package: %s\n" % (control.Find("Package"))
380 summary += "Description: %s\n" % (control.Find("Description"))
381 upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
382 bxa_mail = utils.TemplateSubst(upload.Subst,Config()["Dir::Templates"]+"/process-new.bxa_notification")
383 utils.send_mail(bxa_mail)
385 ################################################################################
387 def add_overrides (new, upload, session):
388 changes = upload.pkg.changes
389 files = upload.pkg.files
390 srcpkg = changes.get("source")
392 for suite in changes["suite"].keys():
393 suite_id = get_suite(suite).suite_id
394 for pkg in new.keys():
395 component_id = get_component(new[pkg]["component"]).component_id
396 type_id = get_override_type(new[pkg]["type"]).overridetype_id
397 priority_id = new[pkg]["priority id"]
398 section_id = new[pkg]["section id"]
399 Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
400 session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')",
401 { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id})
402 for f in new[pkg]["files"]:
403 if files[f].has_key("new"):
409 if Config().FindB("Dinstall::BXANotify"):
410 do_bxa_notification(upload)
412 ################################################################################
414 def do_new(upload, session):
416 files = upload.pkg.files
417 upload.check_files(not Options["No-Action"])
418 changes = upload.pkg.changes
421 # Check for a valid distribution
422 upload.check_distributions()
424 # Make a copy of distribution we can happily trample on
425 changes["suite"] = copy.copy(changes["distribution"])
427 # The main NEW processing loop
430 # Find out what's new
431 new, byhand = determine_new(upload.pkg.changes_file, changes, files, session=session)
437 if Options["No-Action"] or Options["Automatic"]:
440 (broken, note) = print_new(new, upload, indexed=0)
443 if not broken and not note:
444 prompt = "Add overrides, "
446 print "W: [!] marked entries must be fixed before package can be processed."
448 print "W: note must be removed before package can be processed."
449 prompt += "RemOve all notes, Remove note, "
451 prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
453 while prompt.find(answer) == -1:
454 answer = utils.our_raw_input(prompt)
455 m = re_default_answer.search(prompt)
458 answer = answer[:1].upper()
460 if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]:
461 utils.warn("Trainees can't do that")
464 if answer == 'A' and not Options["Trainee"]:
467 done = add_overrides (new, upload, session)
468 new_accept(upload, Options["No-Action"], session)
469 Logger.log(["NEW ACCEPT: %s" % (upload.pkg.changes_file)])
470 except CantGetLockError:
471 print "Hello? Operator! Give me the number for 911!"
472 print "Dinstall in the locked area, cant process packages, come back later"
475 elif answer == 'E' and not Options["Trainee"]:
476 new = edit_overrides (new, upload, session)
477 elif answer == 'M' and not Options["Trainee"]:
478 aborted = upload.do_reject(manual=1,
479 reject_message=Options["Manual-Reject"],
480 notes=get_new_comments(changes.get("source", ""), session=session))
482 upload.pkg.remove_known_changes(session=session)
484 Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)])
487 edit_note(get_new_comments(changes.get("source", ""), session=session),
488 upload, session, bool(Options["Trainee"]))
489 elif answer == 'P' and not Options["Trainee"]:
490 prod_maintainer(get_new_comments(changes.get("source", ""), session=session),
492 Logger.log(["NEW PROD: %s" % (upload.pkg.changes_file)])
493 elif answer == 'R' and not Options["Trainee"]:
494 confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
496 for c in get_new_comments(changes.get("source", ""), changes.get("version", ""), session=session):
499 elif answer == 'O' and not Options["Trainee"]:
500 confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
502 for c in get_new_comments(changes.get("source", ""), session=session):
512 ################################################################################
513 ################################################################################
514 ################################################################################
516 def usage (exit_code=0):
517 print """Usage: dak process-new [OPTION]... [CHANGES]...
518 -a, --automatic automatic run
519 -b, --no-binaries do not sort binary-NEW packages first
520 -c, --comments show NEW comments
521 -h, --help show this help and exit.
522 -m, --manual-reject=MSG manual reject with `msg'
523 -n, --no-action don't do anything
524 -t, --trainee FTP Trainee mode
525 -V, --version display the version number and exit"""
528 ################################################################################
530 def do_byhand(upload, session):
533 files = upload.pkg.files
537 for f in files.keys():
538 if files[f]["section"] == "byhand":
539 if os.path.exists(f):
540 print "W: %s still present; please process byhand components and try again." % (f)
546 if Options["No-Action"]:
549 if Options["Automatic"] and not Options["No-Action"]:
551 prompt = "[A]ccept, Manual reject, Skip, Quit ?"
553 prompt = "Manual reject, [S]kip, Quit ?"
555 while prompt.find(answer) == -1:
556 answer = utils.our_raw_input(prompt)
557 m = re_default_answer.search(prompt)
560 answer = answer[:1].upper()
563 dbchg = get_dbchange(upload.pkg.changes_file, session)
565 print "Warning: cannot find changes file in database; can't process BYHAND"
571 # Find the file entry in the database
573 for f in dbchg.files:
580 print "Warning: Couldn't find BYHAND item %s in the database to mark it processed" % b
583 Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)])
584 except CantGetLockError:
585 print "Hello? Operator! Give me the number for 911!"
586 print "Dinstall in the locked area, cant process packages, come back later"
588 aborted = upload.do_reject(manual=1,
589 reject_message=Options["Manual-Reject"],
590 notes=get_new_comments(changes.get("source", ""), session=session))
592 upload.pkg.remove_known_changes(session=session)
594 Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
602 ################################################################################
604 def check_daily_lock():
606 Raises CantGetLockError if the dinstall daily.lock exists.
611 os.open(cnf["Process-New::DinstallLockFile"],
612 os.O_RDONLY | os.O_CREAT | os.O_EXCL)
614 if e.errno == errno.EEXIST or e.errno == errno.EACCES:
615 raise CantGetLockError
617 os.unlink(cnf["Process-New::DinstallLockFile"])
620 @contextlib.contextmanager
621 def lock_package(package):
623 Lock C{package} so that noone else jumps in processing it.
625 @type package: string
626 @param package: source package name to lock
629 path = os.path.join(Config()["Process-New::LockDir"], package)
631 fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
633 if e.errno == errno.EEXIST or e.errno == errno.EACCES:
634 user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
635 raise AlreadyLockedError, user
642 class clean_holding(object):
643 def __init__(self,pkg):
649 def __exit__(self, type, value, traceback):
652 for f in self.pkg.files.keys():
653 if os.path.exists(os.path.join(h.holding_dir, f)):
654 os.unlink(os.path.join(h.holding_dir, f))
657 def do_pkg(changes_full_path, session):
658 changes_dir = os.path.dirname(changes_full_path)
659 changes_file = os.path.basename(changes_full_path)
662 u.pkg.changes_file = changes_file
663 (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changes_file)
664 u.load_changes(changes_file)
665 u.pkg.directory = changes_dir
668 origchanges = os.path.abspath(u.pkg.changes_file)
671 bcc = "X-DAK: dak process-new"
672 if cnf.has_key("Dinstall::Bcc"):
673 u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
675 u.Subst["__BCC__"] = bcc
678 for deb_filename, f in files.items():
679 if deb_filename.endswith(".udeb") or deb_filename.endswith(".deb"):
680 u.binary_file_checks(deb_filename, session)
681 u.check_binary_against_db(deb_filename, session)
683 u.source_file_checks(deb_filename, session)
684 u.check_source_against_db(deb_filename, session)
686 u.pkg.changes["suite"] = copy.copy(u.pkg.changes["distribution"])
689 with lock_package(u.pkg.changes["source"]):
690 with clean_holding(u.pkg):
691 if not recheck(u, session):
694 new, byhand = determine_new(u.pkg.changes_file, u.pkg.changes, files, session=session)
696 do_byhand(u, session)
702 new_accept(u, Options["No-Action"], session)
703 except CantGetLockError:
704 print "Hello? Operator! Give me the number for 911!"
705 print "Dinstall in the locked area, cant process packages, come back later"
707 except AlreadyLockedError, e:
708 print "Seems to be locked by %s already, skipping..." % (e)
710 def show_new_comments(changes_files, session):
712 query = """SELECT package, version, comment, author
714 WHERE package IN ('"""
716 for changes in changes_files:
717 sources.add(os.path.basename(changes).split("_")[0])
719 query += "%s') ORDER BY package, version" % "', '".join(sources)
720 r = session.execute(query)
723 print "%s_%s\n%s\n(%s)\n\n\n" % (i[0], i[1], i[2], i[3])
727 ################################################################################
730 accept_count = SummaryStats().accept_count
731 accept_bytes = SummaryStats().accept_bytes
737 sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
738 Logger.log(["total",accept_count,accept_bytes])
740 if not Options["No-Action"] and not Options["Trainee"]:
743 ################################################################################
746 global Options, Logger, Sections, Priorities
749 session = DBConn().session()
751 Arguments = [('a',"automatic","Process-New::Options::Automatic"),
752 ('b',"no-binaries","Process-New::Options::No-Binaries"),
753 ('c',"comments","Process-New::Options::Comments"),
754 ('h',"help","Process-New::Options::Help"),
755 ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
756 ('t',"trainee","Process-New::Options::Trainee"),
757 ('n',"no-action","Process-New::Options::No-Action")]
759 for i in ["automatic", "no-binaries", "comments", "help", "manual-reject", "no-action", "version", "trainee"]:
760 if not cnf.has_key("Process-New::Options::%s" % (i)):
761 cnf["Process-New::Options::%s" % (i)] = ""
763 changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
764 if len(changes_files) == 0:
765 new_queue = get_policy_queue('new', session );
766 changes_paths = [ os.path.join(new_queue.path, j) for j in utils.get_changes_files(new_queue.path) ]
768 changes_paths = [ os.path.abspath(j) for j in changes_files ]
770 Options = cnf.SubTree("Process-New::Options")
775 if not Options["No-Action"]:
777 Logger = daklog.Logger(cnf, "process-new")
778 except CantOpenError, e:
779 Options["Trainee"] = "True"
781 Sections = Section_Completer(session)
782 Priorities = Priority_Completer(session)
783 readline.parse_and_bind("tab: complete")
785 if len(changes_paths) > 1:
786 sys.stderr.write("Sorting changes...\n")
787 changes_files = sort_changes(changes_paths, session, Options["No-Binaries"])
789 if Options["Comments"]:
790 show_new_comments(changes_files, session)
792 for changes_file in changes_files:
793 changes_file = utils.validate_changes_file_arg(changes_file, 0)
796 print "\n" + os.path.basename(changes_file)
798 do_pkg (changes_file, session)
802 ################################################################################
804 if __name__ == '__main__':