]> git.donarmstrong.com Git - dak.git/blob - dak/process_new.py
e3388d25597654893125d0b55ee957621706c4e2
[dak.git] / dak / process_new.py
1 #!/usr/bin/env python
2 # vim:set et ts=4 sw=4:
3
4 """ Handles NEW and BYHAND packages
5
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @license: GNU General Public License version 2 or later
10 """
11 # This program is free software; you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation; either version 2 of the License, or
14 # (at your option) any later version.
15
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19 # GNU General Public License for more details.
20
21 # You should have received a copy of the GNU General Public License
22 # along with this program; if not, write to the Free Software
23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
24
25 ################################################################################
26
27 # 23:12|<aj> I will not hush!
28 # 23:12|<elmo> :>
29 # 23:12|<aj> Where there is injustice in the world, I shall be there!
30 # 23:13|<aj> I shall not be silenced!
31 # 23:13|<aj> The world shall know!
32 # 23:13|<aj> The world *must* know!
33 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
34 # 23:13|<aj> yay powerpuff girls!!
35 # 23:13|<aj> buttercup's my favourite, who's yours?
36 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
37 # 23:14|<aj> *AREN'T YOU*?!
38 # 23:15|<aj> I will not be treated like this.
39 # 23:15|<aj> I shall have my revenge.
40 # 23:15|<aj> I SHALL!!!
41
42 ################################################################################
43
44 from __future__ import with_statement
45
46 import copy
47 import errno
48 import os
49 import readline
50 import stat
51 import sys
52 import time
53 import contextlib
54 import pwd
55 import apt_pkg, apt_inst
56 import examine_package
57 from daklib import database
58 from daklib import logging
59 from daklib import queue
60 from daklib import utils
61 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
62 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
63
64 # Globals
65 Cnf = None       #: Configuration, apt_pkg.Configuration
66 Options = None
67 Upload = None
68 projectB = None  #: database connection, pgobject
69 Logger = None
70
71 Priorities = None
72 Sections = None
73
74 reject_message = ""
75
76 ################################################################################
77 ################################################################################
78 ################################################################################
79
80 def reject (str, prefix="Rejected: "):
81     global reject_message
82     if str:
83         reject_message += prefix + str + "\n"
84
85 def recheck():
86     global reject_message
87     files = Upload.pkg.files
88     reject_message = ""
89
90     for f in files.keys():
91         # The .orig.tar.gz can disappear out from under us is it's a
92         # duplicate of one in the archive.
93         if not files.has_key(f):
94             continue
95         # Check that the source still exists
96         if files[f]["type"] == "deb":
97             source_version = files[f]["source version"]
98             source_package = files[f]["source package"]
99             if not Upload.pkg.changes["architecture"].has_key("source") \
100                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
101                 source_epochless_version = re_no_epoch.sub('', source_version)
102                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
103                 found = 0
104                 for q in ["Accepted", "Embargoed", "Unembargoed", "Newstage"]:
105                     if Cnf.has_key("Dir::Queue::%s" % (q)):
106                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
107                             found = 1
108                 if not found:
109                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
110
111         # Version and file overwrite checks
112         if files[f]["type"] == "deb":
113             reject(Upload.check_binary_against_db(f), "")
114         elif files[f]["type"] == "dsc":
115             reject(Upload.check_source_against_db(f), "")
116             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
117             reject(reject_msg, "")
118
119     if reject_message.find("Rejected") != -1:
120         answer = "XXX"
121         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
122             answer = 'S'
123
124         print "REJECT\n" + reject_message,
125         prompt = "[R]eject, Skip, Quit ?"
126
127         while prompt.find(answer) == -1:
128             answer = utils.our_raw_input(prompt)
129             m = re_default_answer.match(prompt)
130             if answer == "":
131                 answer = m.group(1)
132             answer = answer[:1].upper()
133
134         if answer == 'R':
135             Upload.do_reject(0, reject_message)
136             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
137             return 0
138         elif answer == 'S':
139             return 0
140         elif answer == 'Q':
141             end()
142             sys.exit(0)
143
144     return 1
145
146 ################################################################################
147
148 def indiv_sg_compare (a, b):
149     """Sort by source name, source, version, 'have source', and
150        finally by filename."""
151     # Sort by source version
152     q = apt_pkg.VersionCompare(a["version"], b["version"])
153     if q:
154         return -q
155
156     # Sort by 'have source'
157     a_has_source = a["architecture"].get("source")
158     b_has_source = b["architecture"].get("source")
159     if a_has_source and not b_has_source:
160         return -1
161     elif b_has_source and not a_has_source:
162         return 1
163
164     return cmp(a["filename"], b["filename"])
165
166 ############################################################
167
168 def sg_compare (a, b):
169     a = a[1]
170     b = b[1]
171     """Sort by have note, source already in database and time of oldest upload."""
172     # Sort by have note
173     a_note_state = a["note_state"]
174     b_note_state = b["note_state"]
175     if a_note_state < b_note_state:
176         return -1
177     elif a_note_state > b_note_state:
178         return 1
179     # Sort by source already in database (descending)
180     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
181     if source_in_database:
182         return -source_in_database
183
184     # Sort by time of oldest upload
185     return cmp(a["oldest"], b["oldest"])
186
187 def sort_changes(changes_files):
188     """Sort into source groups, then sort each source group by version,
189     have source, filename.  Finally, sort the source groups by have
190     note, time of oldest upload of each source upload."""
191     if len(changes_files) == 1:
192         return changes_files
193
194     sorted_list = []
195     cache = {}
196     # Read in all the .changes files
197     for filename in changes_files:
198         try:
199             Upload.pkg.changes_file = filename
200             Upload.init_vars()
201             Upload.update_vars()
202             cache[filename] = copy.copy(Upload.pkg.changes)
203             cache[filename]["filename"] = filename
204         except:
205             sorted_list.append(filename)
206             break
207     # Divide the .changes into per-source groups
208     per_source = {}
209     for filename in cache.keys():
210         source = cache[filename]["source"]
211         if not per_source.has_key(source):
212             per_source[source] = {}
213             per_source[source]["list"] = []
214         per_source[source]["list"].append(cache[filename])
215     # Determine oldest time and have note status for each source group
216     for source in per_source.keys():
217         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
218         ql = q.getresult()
219         per_source[source]["source_in_database"] = len(ql)>0
220         source_list = per_source[source]["list"]
221         first = source_list[0]
222         oldest = os.stat(first["filename"])[stat.ST_MTIME]
223         have_note = 0
224         for d in per_source[source]["list"]:
225             mtime = os.stat(d["filename"])[stat.ST_MTIME]
226             if mtime < oldest:
227                 oldest = mtime
228             have_note += (database.has_new_comment(d["source"], d["version"]))
229         per_source[source]["oldest"] = oldest
230         if not have_note:
231             per_source[source]["note_state"] = 0; # none
232         elif have_note < len(source_list):
233             per_source[source]["note_state"] = 1; # some
234         else:
235             per_source[source]["note_state"] = 2; # all
236         per_source[source]["list"].sort(indiv_sg_compare)
237     per_source_items = per_source.items()
238     per_source_items.sort(sg_compare)
239     for i in per_source_items:
240         for j in i[1]["list"]:
241             sorted_list.append(j["filename"])
242     return sorted_list
243
244 ################################################################################
245
246 class Section_Completer:
247     def __init__ (self):
248         self.sections = []
249         self.matches = []
250         q = projectB.query("SELECT section FROM section")
251         for i in q.getresult():
252             self.sections.append(i[0])
253
254     def complete(self, text, state):
255         if state == 0:
256             self.matches = []
257             n = len(text)
258             for word in self.sections:
259                 if word[:n] == text:
260                     self.matches.append(word)
261         try:
262             return self.matches[state]
263         except IndexError:
264             return None
265
266 ############################################################
267
268 class Priority_Completer:
269     def __init__ (self):
270         self.priorities = []
271         self.matches = []
272         q = projectB.query("SELECT priority FROM priority")
273         for i in q.getresult():
274             self.priorities.append(i[0])
275
276     def complete(self, text, state):
277         if state == 0:
278             self.matches = []
279             n = len(text)
280             for word in self.priorities:
281                 if word[:n] == text:
282                     self.matches.append(word)
283         try:
284             return self.matches[state]
285         except IndexError:
286             return None
287
288 ################################################################################
289
290 def print_new (new, indexed, file=sys.stdout):
291     queue.check_valid(new)
292     broken = 0
293     index = 0
294     for pkg in new.keys():
295         index += 1
296         section = new[pkg]["section"]
297         priority = new[pkg]["priority"]
298         if new[pkg]["section id"] == -1:
299             section += "[!]"
300             broken = 1
301         if new[pkg]["priority id"] == -1:
302             priority += "[!]"
303             broken = 1
304         if indexed:
305             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
306         else:
307             line = "%-20s %-20s %-20s" % (pkg, priority, section)
308         line = line.strip()+'\n'
309         file.write(line)
310     note = database.get_new_comments(Upload.pkg.changes.get("source"))
311     if len(note) > 0:
312         for line in note:
313             print line
314     return broken, note
315
316 ################################################################################
317
318 def index_range (index):
319     if index == 1:
320         return "1"
321     else:
322         return "1-%s" % (index)
323
324 ################################################################################
325 ################################################################################
326
327 def edit_new (new):
328     # Write the current data to a temporary file
329     (fd, temp_filename) = utils.temp_filename()
330     temp_file = os.fdopen(fd, 'w')
331     print_new (new, 0, temp_file)
332     temp_file.close()
333     # Spawn an editor on that file
334     editor = os.environ.get("EDITOR","vi")
335     result = os.system("%s %s" % (editor, temp_filename))
336     if result != 0:
337         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
338     # Read the edited data back in
339     temp_file = utils.open_file(temp_filename)
340     lines = temp_file.readlines()
341     temp_file.close()
342     os.unlink(temp_filename)
343     # Parse the new data
344     for line in lines:
345         line = line.strip()
346         if line == "":
347             continue
348         s = line.split()
349         # Pad the list if necessary
350         s[len(s):3] = [None] * (3-len(s))
351         (pkg, priority, section) = s[:3]
352         if not new.has_key(pkg):
353             utils.warn("Ignoring unknown package '%s'" % (pkg))
354         else:
355             # Strip off any invalid markers, print_new will readd them.
356             if section.endswith("[!]"):
357                 section = section[:-3]
358             if priority.endswith("[!]"):
359                 priority = priority[:-3]
360             for f in new[pkg]["files"]:
361                 Upload.pkg.files[f]["section"] = section
362                 Upload.pkg.files[f]["priority"] = priority
363             new[pkg]["section"] = section
364             new[pkg]["priority"] = priority
365
366 ################################################################################
367
368 def edit_index (new, index):
369     priority = new[index]["priority"]
370     section = new[index]["section"]
371     ftype = new[index]["type"]
372     done = 0
373     while not done:
374         print "\t".join([index, priority, section])
375
376         answer = "XXX"
377         if ftype != "dsc":
378             prompt = "[B]oth, Priority, Section, Done ? "
379         else:
380             prompt = "[S]ection, Done ? "
381         edit_priority = edit_section = 0
382
383         while prompt.find(answer) == -1:
384             answer = utils.our_raw_input(prompt)
385             m = re_default_answer.match(prompt)
386             if answer == "":
387                 answer = m.group(1)
388             answer = answer[:1].upper()
389
390         if answer == 'P':
391             edit_priority = 1
392         elif answer == 'S':
393             edit_section = 1
394         elif answer == 'B':
395             edit_priority = edit_section = 1
396         elif answer == 'D':
397             done = 1
398
399         # Edit the priority
400         if edit_priority:
401             readline.set_completer(Priorities.complete)
402             got_priority = 0
403             while not got_priority:
404                 new_priority = utils.our_raw_input("New priority: ").strip()
405                 if new_priority not in Priorities.priorities:
406                     print "E: '%s' is not a valid priority, try again." % (new_priority)
407                 else:
408                     got_priority = 1
409                     priority = new_priority
410
411         # Edit the section
412         if edit_section:
413             readline.set_completer(Sections.complete)
414             got_section = 0
415             while not got_section:
416                 new_section = utils.our_raw_input("New section: ").strip()
417                 if new_section not in Sections.sections:
418                     print "E: '%s' is not a valid section, try again." % (new_section)
419                 else:
420                     got_section = 1
421                     section = new_section
422
423         # Reset the readline completer
424         readline.set_completer(None)
425
426     for f in new[index]["files"]:
427         Upload.pkg.files[f]["section"] = section
428         Upload.pkg.files[f]["priority"] = priority
429     new[index]["priority"] = priority
430     new[index]["section"] = section
431     return new
432
433 ################################################################################
434
435 def edit_overrides (new):
436     print
437     done = 0
438     while not done:
439         print_new (new, 1)
440         new_index = {}
441         index = 0
442         for i in new.keys():
443             index += 1
444             new_index[index] = i
445
446         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
447
448         got_answer = 0
449         while not got_answer:
450             answer = utils.our_raw_input(prompt)
451             if not answer.isdigit():
452                 answer = answer[:1].upper()
453             if answer == "E" or answer == "D":
454                 got_answer = 1
455             elif re_isanum.match (answer):
456                 answer = int(answer)
457                 if (answer < 1) or (answer > index):
458                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
459                 else:
460                     got_answer = 1
461
462         if answer == 'E':
463             edit_new(new)
464         elif answer == 'D':
465             done = 1
466         else:
467             edit_index (new, new_index[answer])
468
469     return new
470
471 ################################################################################
472
473 def edit_note(note):
474     # Write the current data to a temporary file
475     (fd, temp_filename) = utils.temp_filename()
476     editor = os.environ.get("EDITOR","vi")
477     answer = 'E'
478     while answer == 'E':
479         os.system("%s %s" % (editor, temp_filename))
480         temp_file = utils.open_file(temp_filename)
481         newnote = temp_file.read().rstrip()
482         temp_file.close()
483         print "New Note:"
484         print utils.prefix_multi_line_string(newnote,"  ")
485         prompt = "[D]one, Edit, Abandon, Quit ?"
486         answer = "XXX"
487         while prompt.find(answer) == -1:
488             answer = utils.our_raw_input(prompt)
489             m = re_default_answer.search(prompt)
490             if answer == "":
491                 answer = m.group(1)
492             answer = answer[:1].upper()
493     os.unlink(temp_filename)
494     if answer == 'A':
495         return
496     elif answer == 'Q':
497         end()
498         sys.exit(0)
499     database.add_new_comment(Upload.pkg.changes["source"], Upload.pkg.changes["version"], newnote, utils.whoami())
500
501 ################################################################################
502
503 def check_pkg ():
504     try:
505         less_fd = os.popen("less -R -", 'w', 0)
506         stdout_fd = sys.stdout
507         try:
508             sys.stdout = less_fd
509             changes = utils.parse_changes (Upload.pkg.changes_file)
510             examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file)
511             files = Upload.pkg.files
512             for f in files.keys():
513                 if files[f].has_key("new"):
514                     ftype = files[f]["type"]
515                     if ftype == "deb":
516                         examine_package.check_deb(changes['distribution'], f)
517                     elif ftype == "dsc":
518                         examine_package.check_dsc(changes['distribution'], f)
519         finally:
520             examine_package.output_package_relations()
521             sys.stdout = stdout_fd
522     except IOError, e:
523         if e.errno == errno.EPIPE:
524             utils.warn("[examine_package] Caught EPIPE; skipping.")
525             pass
526         else:
527             raise
528     except KeyboardInterrupt:
529         utils.warn("[examine_package] Caught C-c; skipping.")
530         pass
531
532 ################################################################################
533
534 ## FIXME: horribly Debian specific
535
536 def do_bxa_notification():
537     files = Upload.pkg.files
538     summary = ""
539     for f in files.keys():
540         if files[f]["type"] == "deb":
541             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
542             summary += "\n"
543             summary += "Package: %s\n" % (control.Find("Package"))
544             summary += "Description: %s\n" % (control.Find("Description"))
545     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
546     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
547     utils.send_mail(bxa_mail)
548
549 ################################################################################
550
551 def add_overrides (new):
552     changes = Upload.pkg.changes
553     files = Upload.pkg.files
554
555     projectB.query("BEGIN WORK")
556     for suite in changes["suite"].keys():
557         suite_id = database.get_suite_id(suite)
558         for pkg in new.keys():
559             component_id = database.get_component_id(new[pkg]["component"])
560             type_id = database.get_override_type_id(new[pkg]["type"])
561             priority_id = new[pkg]["priority id"]
562             section_id = new[pkg]["section id"]
563             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
564             for f in new[pkg]["files"]:
565                 if files[f].has_key("new"):
566                     del files[f]["new"]
567             del new[pkg]
568
569     projectB.query("COMMIT WORK")
570
571     if Cnf.FindB("Dinstall::BXANotify"):
572         do_bxa_notification()
573
574 ################################################################################
575
576 def prod_maintainer (note):
577     # Here we prepare an editor and get them ready to prod...
578     (fd, temp_filename) = utils.temp_filename()
579     temp_file = os.fdopen(fd, 'w')
580     if len(note) > 0:
581         for line in note:
582             temp_file.write(line)
583     temp_file.close()
584     editor = os.environ.get("EDITOR","vi")
585     answer = 'E'
586     while answer == 'E':
587         os.system("%s %s" % (editor, temp_filename))
588         temp_fh = utils.open_file(temp_filename)
589         prod_message = "".join(temp_fh.readlines())
590         temp_fh.close()
591         print "Prod message:"
592         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
593         prompt = "[P]rod, Edit, Abandon, Quit ?"
594         answer = "XXX"
595         while prompt.find(answer) == -1:
596             answer = utils.our_raw_input(prompt)
597             m = re_default_answer.search(prompt)
598             if answer == "":
599                 answer = m.group(1)
600             answer = answer[:1].upper()
601     os.unlink(temp_filename)
602     if answer == 'A':
603         return
604     elif answer == 'Q':
605         end()
606         sys.exit(0)
607     # Otherwise, do the proding...
608     user_email_address = utils.whoami() + " <%s>" % (
609         Cnf["Dinstall::MyAdminAddress"])
610
611     Subst = Upload.Subst
612
613     Subst["__FROM_ADDRESS__"] = user_email_address
614     Subst["__PROD_MESSAGE__"] = prod_message
615     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
616
617     prod_mail_message = utils.TemplateSubst(
618         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
619
620     # Send the prod mail if appropriate
621     if not Cnf["Dinstall::Options::No-Mail"]:
622         utils.send_mail(prod_mail_message)
623
624     print "Sent proding message"
625
626 ################################################################################
627
628 def do_new():
629     print "NEW\n"
630     files = Upload.pkg.files
631     changes = Upload.pkg.changes
632
633     # Make a copy of distribution we can happily trample on
634     changes["suite"] = copy.copy(changes["distribution"])
635
636     # Fix up the list of target suites
637     for suite in changes["suite"].keys():
638         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
639         if override:
640             (olderr, newerr) = (database.get_suite_id(suite) == -1,
641               database.get_suite_id(override) == -1)
642             if olderr or newerr:
643                 (oinv, newinv) = ("", "")
644                 if olderr: oinv = "invalid "
645                 if newerr: ninv = "invalid "
646                 print "warning: overriding %ssuite %s to %ssuite %s" % (
647                         oinv, suite, ninv, override)
648             del changes["suite"][suite]
649             changes["suite"][override] = 1
650     # Validate suites
651     for suite in changes["suite"].keys():
652         suite_id = database.get_suite_id(suite)
653         if suite_id == -1:
654             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
655
656     # The main NEW processing loop
657     done = 0
658     while not done:
659         # Find out what's new
660         new = queue.determine_new(changes, files, projectB)
661
662         if not new:
663             break
664
665         answer = "XXX"
666         if Options["No-Action"] or Options["Automatic"]:
667             answer = 'S'
668
669         (broken, note) = print_new(new, 0)
670         prompt = ""
671
672         if not broken and not note:
673             prompt = "Add overrides, "
674         if broken:
675             print "W: [!] marked entries must be fixed before package can be processed."
676         if note:
677             print "W: note must be removed before package can be processed."
678             prompt += "Remove note, "
679
680         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
681
682         while prompt.find(answer) == -1:
683             answer = utils.our_raw_input(prompt)
684             m = re_default_answer.search(prompt)
685             if answer == "":
686                 answer = m.group(1)
687             answer = answer[:1].upper()
688
689         if answer == 'A' and not Options["Trainee"]:
690             try:
691                 check_daily_lock()
692                 done = add_overrides (new)
693             except CantGetLockError:
694                 print "Hello? Operator! Give me the number for 911!"
695                 print "Dinstall in the locked area, cant process packages, come back later"
696         elif answer == 'C':
697             check_pkg()
698         elif answer == 'E' and not Options["Trainee"]:
699             new = edit_overrides (new)
700         elif answer == 'M' and not Options["Trainee"]:
701             aborted = Upload.do_reject(manual=1,
702                                        reject_message=Options["Manual-Reject"],
703                                        note=database.get_new_comments(changes.get("source", "")))
704             if not aborted:
705                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
706                 done = 1
707         elif answer == 'N':
708             edit_note(database.get_new_comments(changes.get("source", "")))
709         elif answer == 'P' and not Options["Trainee"]:
710             prod_maintainer(database.get_new_comments(changes.get("source", "")))
711         elif answer == 'R' and not Options["Trainee"]:
712             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
713             if confirm == "y":
714                 database.delete_new_comments(changes.get("source"), changes.get("version"))
715         elif answer == 'S':
716             done = 1
717         elif answer == 'Q':
718             end()
719             sys.exit(0)
720
721 ################################################################################
722 ################################################################################
723 ################################################################################
724
725 def usage (exit_code=0):
726     print """Usage: dak process-new [OPTION]... [CHANGES]...
727   -a, --automatic           automatic run
728   -h, --help                show this help and exit.
729   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
730   -m, --manual-reject=MSG   manual reject with `msg'
731   -n, --no-action           don't do anything
732   -t, --trainee             FTP Trainee mode
733   -V, --version             display the version number and exit"""
734     sys.exit(exit_code)
735
736 ################################################################################
737
738 def init():
739     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
740
741     Cnf = utils.get_conf()
742
743     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
744                  ('h',"help","Process-New::Options::Help"),
745                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
746                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
747                  ('t',"trainee","Process-New::Options::Trainee"),
748                  ('n',"no-action","Process-New::Options::No-Action")]
749
750     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]:
751         if not Cnf.has_key("Process-New::Options::%s" % (i)):
752             Cnf["Process-New::Options::%s" % (i)] = ""
753
754     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
755     if len(changes_files) == 0 and not Cnf.get("Process-New::Options::Comments-Dir",""):
756         changes_files = utils.get_changes_files(Cnf["Dir::Queue::New"])
757
758     Options = Cnf.SubTree("Process-New::Options")
759
760     if Options["Help"]:
761         usage()
762
763     Upload = queue.Upload(Cnf)
764
765     if not Options["No-Action"]:
766         try:
767             Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
768         except CantOpenError, e:
769             Options["Trainee"] = "Oh yes"
770
771     projectB = Upload.projectB
772
773     Sections = Section_Completer()
774     Priorities = Priority_Completer()
775     readline.parse_and_bind("tab: complete")
776
777     return changes_files
778
779 ################################################################################
780
781 def do_byhand():
782     done = 0
783     while not done:
784         files = Upload.pkg.files
785         will_install = 1
786         byhand = []
787
788         for f in files.keys():
789             if files[f]["type"] == "byhand":
790                 if os.path.exists(f):
791                     print "W: %s still present; please process byhand components and try again." % (f)
792                     will_install = 0
793                 else:
794                     byhand.append(f)
795
796         answer = "XXXX"
797         if Options["No-Action"]:
798             answer = "S"
799         if will_install:
800             if Options["Automatic"] and not Options["No-Action"]:
801                 answer = 'A'
802             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
803         else:
804             prompt = "Manual reject, [S]kip, Quit ?"
805
806         while prompt.find(answer) == -1:
807             answer = utils.our_raw_input(prompt)
808             m = re_default_answer.search(prompt)
809             if answer == "":
810                 answer = m.group(1)
811             answer = answer[:1].upper()
812
813         if answer == 'A':
814             try:
815                 check_daily_lock()
816                 done = 1
817                 for f in byhand:
818                     del files[f]
819             except CantGetLockError:
820                 print "Hello? Operator! Give me the number for 911!"
821                 print "Dinstall in the locked area, cant process packages, come back later"
822         elif answer == 'M':
823             Upload.do_reject(1, Options["Manual-Reject"])
824             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
825             done = 1
826         elif answer == 'S':
827             done = 1
828         elif answer == 'Q':
829             end()
830             sys.exit(0)
831
832 ################################################################################
833
834 def check_daily_lock():
835     """
836     Raises CantGetLockError if the dinstall daily.lock exists.
837     """
838
839     try:
840         os.open(Cnf["Process-New::DinstallLockFile"],  os.O_RDONLY | os.O_CREAT | os.O_EXCL)
841     except OSError, e:
842         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
843             raise CantGetLockError
844
845     os.unlink(Cnf["Process-New::DinstallLockFile"])
846
847
848 @contextlib.contextmanager
849 def lock_package(package):
850     """
851     Lock C{package} so that noone else jumps in processing it.
852
853     @type package: string
854     @param package: source package name to lock
855     """
856
857     path = os.path.join(Cnf["Process-New::LockDir"], package)
858     try:
859         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
860     except OSError, e:
861         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
862             user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
863             raise AlreadyLockedError, user
864
865     try:
866         yield fd
867     finally:
868         os.unlink(path)
869
870 def move_to_dir (dest, perms=0660, changesperms=0664):
871     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
872     file_keys = Upload.pkg.files.keys()
873     for f in file_keys:
874         utils.move (f, dest, perms=perms)
875
876 def is_source_in_queue_dir(qdir):
877     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
878                 and x.endswith(".changes") ]
879     for entry in entries:
880         # read the .dak
881         u = queue.Upload(Cnf)
882         u.pkg.changes_file = os.path.join(qdir, entry)
883         u.update_vars()
884         if not u.pkg.changes["architecture"].has_key("source"):
885             # another binary upload, ignore
886             continue
887         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
888             # another version, ignore
889             continue
890         # found it!
891         return True
892     return False
893
894 def move_to_holding(suite, queue_dir):
895     print "Moving to %s holding area." % (suite.upper(),)
896     if Options["No-Action"]:
897         return
898     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
899     Upload.dump_vars(queue_dir)
900     move_to_dir(queue_dir, perms=0664)
901     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
902
903 def _accept():
904     if Options["No-Action"]:
905         return
906     (summary, short_summary) = Upload.build_summaries()
907     Upload.accept(summary, short_summary, targetdir=Cnf["Dir::Queue::Newstage"])
908     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
909
910 def do_accept_stableupdate(suite, q):
911     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
912     if not Upload.pkg.changes["architecture"].has_key("source"):
913         # It is not a sourceful upload.  So its source may be either in p-u
914         # holding, in new, in accepted or already installed.
915         if is_source_in_queue_dir(queue_dir):
916             # It's in p-u holding, so move it there.
917             print "Binary-only upload, source in %s." % (q,)
918             move_to_holding(suite, queue_dir)
919         elif Upload.source_exists(Upload.pkg.changes["source"],
920                 Upload.pkg.changes["version"]):
921             # dak tells us that there is source available.  At time of
922             # writing this means that it is installed, so put it into
923             # accepted.
924             print "Binary-only upload, source installed."
925             _accept()
926         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
927             # The source is in accepted, the binary cleared NEW: accept it.
928             print "Binary-only upload, source in accepted."
929             _accept()
930         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
931             # It's in NEW.  We expect the source to land in p-u holding
932             # pretty soon.
933             print "Binary-only upload, source in new."
934             move_to_holding(suite, queue_dir)
935         elif is_source_in_queue_dir(Cnf["Dir::Queue::Newstage"]):
936             # It's in newstage.  Accept into the holding area
937             print "Binary-only upload, source in newstage."
938             move_to_holding(suite, queue_dir)
939         else:
940             # No case applicable.  Bail out.  Return will cause the upload
941             # to be skipped.
942             print "ERROR"
943             print "Stable update failed.  Source not found."
944             return
945     else:
946         # We are handling a sourceful upload.  Move to accepted if currently
947         # in p-u holding and to p-u holding otherwise.
948         if is_source_in_queue_dir(queue_dir):
949             print "Sourceful upload in %s, accepting." % (q,)
950             _accept()
951         else:
952             move_to_holding(suite, queue_dir)
953
954 def do_accept():
955     print "ACCEPT"
956     if not Options["No-Action"]:
957         (summary, short_summary) = Upload.build_summaries()
958     if Cnf.FindB("Dinstall::SecurityQueueHandling"):
959         Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
960         move_to_dir(Cnf["Dir::Queue::Embargoed"])
961         Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
962         # Check for override disparities
963         Upload.Subst["__SUMMARY__"] = summary
964     else:
965         # Stable updates need to be copied to proposed-updates holding
966         # area instead of accepted.  Sourceful uploads need to go
967         # to it directly, binaries only if the source has not yet been
968         # accepted into p-u.
969         for suite, q in [("proposed-updates", "ProposedUpdates"),
970                 ("oldstable-proposed-updates", "OldProposedUpdates")]:
971             if not Upload.pkg.changes["distribution"].has_key(suite):
972                 continue
973             return do_accept_stableupdate(suite, q)
974         # Just a normal upload, accept it...
975         _accept()
976
977 def check_status(files):
978     new = byhand = 0
979     for f in files.keys():
980         if files[f]["type"] == "byhand":
981             byhand = 1
982         elif files[f].has_key("new"):
983             new = 1
984     return (new, byhand)
985
986 def do_pkg(changes_file):
987     Upload.pkg.changes_file = changes_file
988     Upload.init_vars()
989     Upload.update_vars()
990     Upload.update_subst()
991     files = Upload.pkg.files
992
993     try:
994         with lock_package(Upload.pkg.changes["source"]):
995             if not recheck():
996                 return
997
998             (new, byhand) = check_status(files)
999             if new or byhand:
1000                 if new:
1001                     do_new()
1002                 if byhand:
1003                     do_byhand()
1004                 (new, byhand) = check_status(files)
1005
1006             if not new and not byhand:
1007                 try:
1008                     check_daily_lock()
1009                     do_accept()
1010                 except CantGetLockError:
1011                     print "Hello? Operator! Give me the number for 911!"
1012                     print "Dinstall in the locked area, cant process packages, come back later"
1013     except AlreadyLockedError, e:
1014         print "Seems to be locked by %s already, skipping..." % (e)
1015
1016 ################################################################################
1017
1018 def end():
1019     accept_count = Upload.accept_count
1020     accept_bytes = Upload.accept_bytes
1021
1022     if accept_count:
1023         sets = "set"
1024         if accept_count > 1:
1025             sets = "sets"
1026         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
1027         Logger.log(["total",accept_count,accept_bytes])
1028
1029     if not Options["No-Action"] and not Options["Trainee"]:
1030         Logger.close()
1031
1032 ################################################################################
1033
1034 def do_comments(dir, opref, npref, line, fn):
1035     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
1036         lines = open("%s/%s" % (dir, comm)).readlines()
1037         if len(lines) == 0 or lines[0] != line + "\n": continue
1038         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
1039                                 and x.endswith(".changes") ]
1040         changes_files = sort_changes(changes_files)
1041         for f in changes_files:
1042             f = utils.validate_changes_file_arg(f, 0)
1043             if not f: continue
1044             print "\n" + f
1045             fn(f, "".join(lines[1:]))
1046
1047         if opref != npref and not Options["No-Action"]:
1048             newcomm = npref + comm[len(opref):]
1049             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
1050
1051 ################################################################################
1052
1053 def comment_accept(changes_file, comments):
1054     Upload.pkg.changes_file = changes_file
1055     Upload.init_vars()
1056     Upload.update_vars()
1057     Upload.update_subst()
1058     files = Upload.pkg.files
1059
1060     if not recheck():
1061         return # dak wants to REJECT, crap
1062
1063     (new, byhand) = check_status(files)
1064     if not new and not byhand:
1065         do_accept()
1066
1067 ################################################################################
1068
1069 def comment_reject(changes_file, comments):
1070     Upload.pkg.changes_file = changes_file
1071     Upload.init_vars()
1072     Upload.update_vars()
1073     Upload.update_subst()
1074
1075     if not recheck():
1076         pass # dak has its own reasons to reject as well, which is fine
1077
1078     reject(comments)
1079     print "REJECT\n" + reject_message,
1080     if not Options["No-Action"]:
1081         Upload.do_reject(0, reject_message)
1082         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1083
1084 ################################################################################
1085
1086 def main():
1087     changes_files = init()
1088     if len(changes_files) > 50:
1089         sys.stderr.write("Sorting changes...\n")
1090     changes_files = sort_changes(changes_files)
1091
1092     # Kill me now? **FIXME**
1093     Cnf["Dinstall::Options::No-Mail"] = ""
1094     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1095     if Cnf.has_key("Dinstall::Bcc"):
1096         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1097     else:
1098         Upload.Subst["__BCC__"] = bcc
1099
1100     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1101     if commentsdir:
1102         if changes_files != []:
1103             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1104             sys.exit(1)
1105         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1106         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1107     else:
1108         for changes_file in changes_files:
1109             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1110             if not changes_file:
1111                 continue
1112             print "\n" + changes_file
1113
1114             do_pkg (changes_file)
1115
1116     end()
1117
1118 ################################################################################
1119
1120 if __name__ == '__main__':
1121     main()