]> git.donarmstrong.com Git - dak.git/blob - dak/process_new.py
0e1d5c036fae285788d26d606c984263da033a1a
[dak.git] / dak / process_new.py
1 #!/usr/bin/env python
2
3 # Handles NEW and BYHAND packages
4 # Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
5
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
10
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14 # GNU General Public License for more details.
15
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
19
20 ################################################################################
21
22 # 23:12|<aj> I will not hush!
23 # 23:12|<elmo> :>
24 # 23:12|<aj> Where there is injustice in the world, I shall be there!
25 # 23:13|<aj> I shall not be silenced!
26 # 23:13|<aj> The world shall know!
27 # 23:13|<aj> The world *must* know!
28 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
29 # 23:13|<aj> yay powerpuff girls!!
30 # 23:13|<aj> buttercup's my favourite, who's yours?
31 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
32 # 23:14|<aj> *AREN'T YOU*?!
33 # 23:15|<aj> I will not be treated like this.
34 # 23:15|<aj> I shall have my revenge.
35 # 23:15|<aj> I SHALL!!!
36
37 ################################################################################
38
39 import copy, errno, os, readline, stat, sys, time
40 import apt_pkg, apt_inst
41 import examine_package
42 from daklib import database
43 from daklib import logging
44 from daklib import queue
45 from daklib import utils
46
47 # Globals
48 Cnf = None
49 Options = None
50 Upload = None
51 projectB = None
52 Logger = None
53
54 Priorities = None
55 Sections = None
56
57 reject_message = ""
58
59 ################################################################################
60 ################################################################################
61 ################################################################################
62
63 def reject (str, prefix="Rejected: "):
64     global reject_message
65     if str:
66         reject_message += prefix + str + "\n"
67
68 def recheck():
69     global reject_message
70     files = Upload.pkg.files
71     reject_message = ""
72
73     for f in files.keys():
74         # The .orig.tar.gz can disappear out from under us is it's a
75         # duplicate of one in the archive.
76         if not files.has_key(f):
77             continue
78         # Check that the source still exists
79         if files[f]["type"] == "deb":
80             source_version = files[f]["source version"]
81             source_package = files[f]["source package"]
82             if not Upload.pkg.changes["architecture"].has_key("source") \
83                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
84                 source_epochless_version = utils.re_no_epoch.sub('', source_version)
85                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
86                 found = 0
87                 for q in ["Accepted", "Embargoed", "Unembargoed"]:
88                     if Cnf.has_key("Dir::Queue::%s" % (q)):
89                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
90                             found = 1
91                 if not found:
92                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
93
94         # Version and file overwrite checks
95         if files[f]["type"] == "deb":
96             reject(Upload.check_binary_against_db(f))
97         elif files[f]["type"] == "dsc":
98             reject(Upload.check_source_against_db(f))
99             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
100             reject(reject_msg, "")
101
102     if reject_message.find("Rejected") != -1:
103         answer = "XXX"
104         if Options["No-Action"] or Options["Automatic"]:
105             answer = 'S'
106
107         print "REJECT\n" + reject_message,
108         prompt = "[R]eject, Skip, Quit ?"
109
110         while prompt.find(answer) == -1:
111             answer = utils.our_raw_input(prompt)
112             m = queue.re_default_answer.match(prompt)
113             if answer == "":
114                 answer = m.group(1)
115             answer = answer[:1].upper()
116
117         if answer == 'R':
118             Upload.do_reject(0, reject_message)
119             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
120             return 0
121         elif answer == 'S':
122             return 0
123         elif answer == 'Q':
124             end()
125             sys.exit(0)
126
127     return 1
128
129 ################################################################################
130
131 def indiv_sg_compare (a, b):
132     """Sort by source name, source, version, 'have source', and
133        finally by filename."""
134     # Sort by source version
135     q = apt_pkg.VersionCompare(a["version"], b["version"])
136     if q:
137         return -q
138
139     # Sort by 'have source'
140     a_has_source = a["architecture"].get("source")
141     b_has_source = b["architecture"].get("source")
142     if a_has_source and not b_has_source:
143         return -1
144     elif b_has_source and not a_has_source:
145         return 1
146
147     return cmp(a["filename"], b["filename"])
148
149 ############################################################
150
151 def sg_compare (a, b):
152     a = a[1]
153     b = b[1]
154     """Sort by have note, source already in database and time of oldest upload."""
155     # Sort by have note
156     a_note_state = a["note_state"]
157     b_note_state = b["note_state"]
158     if a_note_state < b_note_state:
159         return -1
160     elif a_note_state > b_note_state:
161         return 1
162     # Sort by source already in database (descending)
163     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
164     if source_in_database:
165         return -source_in_database
166
167     # Sort by time of oldest upload
168     return cmp(a["oldest"], b["oldest"])
169
170 def sort_changes(changes_files):
171     """Sort into source groups, then sort each source group by version,
172     have source, filename.  Finally, sort the source groups by have
173     note, time of oldest upload of each source upload."""
174     if len(changes_files) == 1:
175         return changes_files
176
177     sorted_list = []
178     cache = {}
179     # Read in all the .changes files
180     for filename in changes_files:
181         try:
182             Upload.pkg.changes_file = filename
183             Upload.init_vars()
184             Upload.update_vars()
185             cache[filename] = copy.copy(Upload.pkg.changes)
186             cache[filename]["filename"] = filename
187         except:
188             sorted_list.append(filename)
189             break
190     # Divide the .changes into per-source groups
191     per_source = {}
192     for filename in cache.keys():
193         source = cache[filename]["source"]
194         if not per_source.has_key(source):
195             per_source[source] = {}
196             per_source[source]["list"] = []
197         per_source[source]["list"].append(cache[filename])
198     # Determine oldest time and have note status for each source group
199     for source in per_source.keys():
200         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
201         ql = q.getresult()
202         per_source[source]["source_in_database"] = len(ql)>0
203         source_list = per_source[source]["list"]
204         first = source_list[0]
205         oldest = os.stat(first["filename"])[stat.ST_MTIME]
206         have_note = 0
207         for d in per_source[source]["list"]:
208             mtime = os.stat(d["filename"])[stat.ST_MTIME]
209             if mtime < oldest:
210                 oldest = mtime
211             have_note += (d.has_key("process-new note"))
212         per_source[source]["oldest"] = oldest
213         if not have_note:
214             per_source[source]["note_state"] = 0; # none
215         elif have_note < len(source_list):
216             per_source[source]["note_state"] = 1; # some
217         else:
218             per_source[source]["note_state"] = 2; # all
219         per_source[source]["list"].sort(indiv_sg_compare)
220     per_source_items = per_source.items()
221     per_source_items.sort(sg_compare)
222     for i in per_source_items:
223         for j in i[1]["list"]:
224             sorted_list.append(j["filename"])
225     return sorted_list
226
227 ################################################################################
228
229 class Section_Completer:
230     def __init__ (self):
231         self.sections = []
232         q = projectB.query("SELECT section FROM section")
233         for i in q.getresult():
234             self.sections.append(i[0])
235
236     def complete(self, text, state):
237         if state == 0:
238             self.matches = []
239             n = len(text)
240             for word in self.sections:
241                 if word[:n] == text:
242                     self.matches.append(word)
243         try:
244             return self.matches[state]
245         except IndexError:
246             return None
247
248 ############################################################
249
250 class Priority_Completer:
251     def __init__ (self):
252         self.priorities = []
253         q = projectB.query("SELECT priority FROM priority")
254         for i in q.getresult():
255             self.priorities.append(i[0])
256
257     def complete(self, text, state):
258         if state == 0:
259             self.matches = []
260             n = len(text)
261             for word in self.priorities:
262                 if word[:n] == text:
263                     self.matches.append(word)
264         try:
265             return self.matches[state]
266         except IndexError:
267             return None
268
269 ################################################################################
270
271 def print_new (new, indexed, file=sys.stdout):
272     queue.check_valid(new)
273     broken = 0
274     index = 0
275     for pkg in new.keys():
276         index += 1
277         section = new[pkg]["section"]
278         priority = new[pkg]["priority"]
279         if new[pkg]["section id"] == -1:
280             section += "[!]"
281             broken = 1
282         if new[pkg]["priority id"] == -1:
283             priority += "[!]"
284             broken = 1
285         if indexed:
286             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
287         else:
288             line = "%-20s %-20s %-20s" % (pkg, priority, section)
289         line = line.strip()+'\n'
290         file.write(line)
291     note = Upload.pkg.changes.get("process-new note")
292     if note:
293         print "*"*75
294         print note
295         print "*"*75
296     return broken, note
297
298 ################################################################################
299
300 def index_range (index):
301     if index == 1:
302         return "1"
303     else:
304         return "1-%s" % (index)
305
306 ################################################################################
307 ################################################################################
308
309 def edit_new (new):
310     # Write the current data to a temporary file
311     temp_filename = utils.temp_filename()
312     temp_file = utils.open_file(temp_filename, 'w')
313     print_new (new, 0, temp_file)
314     temp_file.close()
315     # Spawn an editor on that file
316     editor = os.environ.get("EDITOR","vi")
317     result = os.system("%s %s" % (editor, temp_filename))
318     if result != 0:
319         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
320     # Read the edited data back in
321     temp_file = utils.open_file(temp_filename)
322     lines = temp_file.readlines()
323     temp_file.close()
324     os.unlink(temp_filename)
325     # Parse the new data
326     for line in lines:
327         line = line.strip()
328         if line == "":
329             continue
330         s = line.split()
331         # Pad the list if necessary
332         s[len(s):3] = [None] * (3-len(s))
333         (pkg, priority, section) = s[:3]
334         if not new.has_key(pkg):
335             utils.warn("Ignoring unknown package '%s'" % (pkg))
336         else:
337             # Strip off any invalid markers, print_new will readd them.
338             if section.endswith("[!]"):
339                 section = section[:-3]
340             if priority.endswith("[!]"):
341                 priority = priority[:-3]
342             for f in new[pkg]["files"]:
343                 Upload.pkg.files[f]["section"] = section
344                 Upload.pkg.files[f]["priority"] = priority
345             new[pkg]["section"] = section
346             new[pkg]["priority"] = priority
347
348 ################################################################################
349
350 def edit_index (new, index):
351     priority = new[index]["priority"]
352     section = new[index]["section"]
353     ftype = new[index]["type"]
354     done = 0
355     while not done:
356         print "\t".join([index, priority, section])
357
358         answer = "XXX"
359         if ftype != "dsc":
360             prompt = "[B]oth, Priority, Section, Done ? "
361         else:
362             prompt = "[S]ection, Done ? "
363         edit_priority = edit_section = 0
364
365         while prompt.find(answer) == -1:
366             answer = utils.our_raw_input(prompt)
367             m = queue.re_default_answer.match(prompt)
368             if answer == "":
369                 answer = m.group(1)
370             answer = answer[:1].upper()
371
372         if answer == 'P':
373             edit_priority = 1
374         elif answer == 'S':
375             edit_section = 1
376         elif answer == 'B':
377             edit_priority = edit_section = 1
378         elif answer == 'D':
379             done = 1
380
381         # Edit the priority
382         if edit_priority:
383             readline.set_completer(Priorities.complete)
384             got_priority = 0
385             while not got_priority:
386                 new_priority = utils.our_raw_input("New priority: ").strip()
387                 if new_priority not in Priorities.priorities:
388                     print "E: '%s' is not a valid priority, try again." % (new_priority)
389                 else:
390                     got_priority = 1
391                     priority = new_priority
392
393         # Edit the section
394         if edit_section:
395             readline.set_completer(Sections.complete)
396             got_section = 0
397             while not got_section:
398                 new_section = utils.our_raw_input("New section: ").strip()
399                 if new_section not in Sections.sections:
400                     print "E: '%s' is not a valid section, try again." % (new_section)
401                 else:
402                     got_section = 1
403                     section = new_section
404
405         # Reset the readline completer
406         readline.set_completer(None)
407
408     for f in new[index]["files"]:
409         Upload.pkg.files[f]["section"] = section
410         Upload.pkg.files[f]["priority"] = priority
411     new[index]["priority"] = priority
412     new[index]["section"] = section
413     return new
414
415 ################################################################################
416
417 def edit_overrides (new):
418     print
419     done = 0
420     while not done:
421         print_new (new, 1)
422         new_index = {}
423         index = 0
424         for i in new.keys():
425             index += 1
426             new_index[index] = i
427
428         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
429
430         got_answer = 0
431         while not got_answer:
432             answer = utils.our_raw_input(prompt)
433             if not answer.isdigit():
434                 answer = answer[:1].upper()
435             if answer == "E" or answer == "D":
436                 got_answer = 1
437             elif queue.re_isanum.match (answer):
438                 answer = int(answer)
439                 if (answer < 1) or (answer > index):
440                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
441                 else:
442                     got_answer = 1
443
444         if answer == 'E':
445             edit_new(new)
446         elif answer == 'D':
447             done = 1
448         else:
449             edit_index (new, new_index[answer])
450
451     return new
452
453 ################################################################################
454
455 def edit_note(note):
456     # Write the current data to a temporary file
457     temp_filename = utils.temp_filename()
458     temp_file = utils.open_file(temp_filename, 'w')
459     temp_file.write(note)
460     temp_file.close()
461     editor = os.environ.get("EDITOR","vi")
462     answer = 'E'
463     while answer == 'E':
464         os.system("%s %s" % (editor, temp_filename))
465         temp_file = utils.open_file(temp_filename)
466         note = temp_file.read().rstrip()
467         temp_file.close()
468         print "Note:"
469         print utils.prefix_multi_line_string(note,"  ")
470         prompt = "[D]one, Edit, Abandon, Quit ?"
471         answer = "XXX"
472         while prompt.find(answer) == -1:
473             answer = utils.our_raw_input(prompt)
474             m = queue.re_default_answer.search(prompt)
475             if answer == "":
476                 answer = m.group(1)
477             answer = answer[:1].upper()
478     os.unlink(temp_filename)
479     if answer == 'A':
480         return
481     elif answer == 'Q':
482         end()
483         sys.exit(0)
484     Upload.pkg.changes["process-new note"] = note
485     Upload.dump_vars(Cnf["Dir::Queue::New"])
486
487 ################################################################################
488
489 def check_pkg ():
490     try:
491         less_fd = os.popen("less -R -", 'w', 0)
492         stdout_fd = sys.stdout
493         try:
494             sys.stdout = less_fd
495             examine_package.display_changes(Upload.pkg.changes_file)
496             files = Upload.pkg.files
497             for f in files.keys():
498                 if files[f].has_key("new"):
499                     ftype = files[f]["type"]
500                     if ftype == "deb":
501                         examine_package.check_deb(f)
502                     elif ftype == "dsc":
503                         examine_package.check_dsc(f)
504         finally:
505             sys.stdout = stdout_fd
506     except IOError, e:
507         if e.errno == errno.EPIPE:
508             utils.warn("[examine_package] Caught EPIPE; skipping.")
509             pass
510         else:
511             raise
512     except KeyboardInterrupt:
513         utils.warn("[examine_package] Caught C-c; skipping.")
514         pass
515
516 ################################################################################
517
518 ## FIXME: horribly Debian specific
519
520 def do_bxa_notification():
521     files = Upload.pkg.files
522     summary = ""
523     for f in files.keys():
524         if files[f]["type"] == "deb":
525             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
526             summary += "\n"
527             summary += "Package: %s\n" % (control.Find("Package"))
528             summary += "Description: %s\n" % (control.Find("Description"))
529     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
530     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
531     utils.send_mail(bxa_mail)
532
533 ################################################################################
534
535 def add_overrides (new):
536     changes = Upload.pkg.changes
537     files = Upload.pkg.files
538
539     projectB.query("BEGIN WORK")
540     for suite in changes["suite"].keys():
541         suite_id = database.get_suite_id(suite)
542         for pkg in new.keys():
543             component_id = database.get_component_id(new[pkg]["component"])
544             type_id = database.get_override_type_id(new[pkg]["type"])
545             priority_id = new[pkg]["priority id"]
546             section_id = new[pkg]["section id"]
547             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
548             for f in new[pkg]["files"]:
549                 if files[f].has_key("new"):
550                     del files[f]["new"]
551             del new[pkg]
552
553     projectB.query("COMMIT WORK")
554
555     if Cnf.FindB("Dinstall::BXANotify"):
556         do_bxa_notification()
557
558 ################################################################################
559
560 def prod_maintainer ():
561     # Here we prepare an editor and get them ready to prod...
562     temp_filename = utils.temp_filename()
563     editor = os.environ.get("EDITOR","vi")
564     answer = 'E'
565     while answer == 'E':
566         os.system("%s %s" % (editor, temp_filename))
567         f = utils.open_file(temp_filename)
568         prod_message = "".join(f.readlines())
569         f.close()
570         print "Prod message:"
571         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
572         prompt = "[P]rod, Edit, Abandon, Quit ?"
573         answer = "XXX"
574         while prompt.find(answer) == -1:
575             answer = utils.our_raw_input(prompt)
576             m = queue.re_default_answer.search(prompt)
577             if answer == "":
578                 answer = m.group(1)
579             answer = answer[:1].upper()
580         os.unlink(temp_filename)
581         if answer == 'A':
582             return
583         elif answer == 'Q':
584             end()
585             sys.exit(0)
586     # Otherwise, do the proding...
587     user_email_address = utils.whoami() + " <%s>" % (
588         Cnf["Dinstall::MyAdminAddress"])
589
590     Subst = Upload.Subst
591
592     Subst["__FROM_ADDRESS__"] = user_email_address
593     Subst["__PROD_MESSAGE__"] = prod_message
594     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
595
596     prod_mail_message = utils.TemplateSubst(
597         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
598
599     # Send the prod mail if appropriate
600     if not Cnf["Dinstall::Options::No-Mail"]:
601         utils.send_mail(prod_mail_message)
602
603     print "Sent proding message"
604
605 ################################################################################
606
607 def do_new():
608     print "NEW\n"
609     files = Upload.pkg.files
610     changes = Upload.pkg.changes
611
612     # Make a copy of distribution we can happily trample on
613     changes["suite"] = copy.copy(changes["distribution"])
614
615     # Fix up the list of target suites
616     for suite in changes["suite"].keys():
617         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
618         if override:
619             (olderr, newerr) = (database.get_suite_id(suite) == -1,
620               database.get_suite_id(override) == -1)
621             if olderr or newerr:
622                 (oinv, newinv) = ("", "")
623                 if olderr: oinv = "invalid "
624                 if newerr: ninv = "invalid "
625                 print "warning: overriding %ssuite %s to %ssuite %s" % (
626                         oinv, suite, ninv, override)
627             del changes["suite"][suite]
628             changes["suite"][override] = 1
629     # Validate suites
630     for suite in changes["suite"].keys():
631         suite_id = database.get_suite_id(suite)
632         if suite_id == -1:
633             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
634
635     # The main NEW processing loop
636     done = 0
637     while not done:
638         # Find out what's new
639         new = queue.determine_new(changes, files, projectB)
640
641         if not new:
642             break
643
644         answer = "XXX"
645         if Options["No-Action"] or Options["Automatic"]:
646             answer = 'S'
647
648         (broken, note) = print_new(new, 0)
649         prompt = ""
650
651         if not broken and not note:
652             prompt = "Add overrides, "
653         if broken:
654             print "W: [!] marked entries must be fixed before package can be processed."
655         if note:
656             print "W: note must be removed before package can be processed."
657             prompt += "Remove note, "
658
659         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
660
661         while prompt.find(answer) == -1:
662             answer = utils.our_raw_input(prompt)
663             m = queue.re_default_answer.search(prompt)
664             if answer == "":
665                 answer = m.group(1)
666             answer = answer[:1].upper()
667
668         if answer == 'A':
669             done = add_overrides (new)
670         elif answer == 'C':
671             check_pkg()
672         elif answer == 'E':
673             new = edit_overrides (new)
674         elif answer == 'M':
675             aborted = Upload.do_reject(1, Options["Manual-Reject"])
676             if not aborted:
677                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
678                 done = 1
679         elif answer == 'N':
680             edit_note(changes.get("process-new note", ""))
681         elif answer == 'P':
682             prod_maintainer()
683         elif answer == 'R':
684             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
685             if confirm == "y":
686                 del changes["process-new note"]
687         elif answer == 'S':
688             done = 1
689         elif answer == 'Q':
690             end()
691             sys.exit(0)
692
693 ################################################################################
694 ################################################################################
695 ################################################################################
696
697 def usage (exit_code=0):
698     print """Usage: dak process-new [OPTION]... [CHANGES]...
699   -a, --automatic           automatic run
700   -h, --help                show this help and exit.
701   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
702   -m, --manual-reject=MSG   manual reject with `msg'
703   -n, --no-action           don't do anything
704   -V, --version             display the version number and exit"""
705     sys.exit(exit_code)
706
707 ################################################################################
708
709 def init():
710     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
711
712     Cnf = utils.get_conf()
713
714     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
715                  ('h',"help","Process-New::Options::Help"),
716                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
717                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
718                  ('n',"no-action","Process-New::Options::No-Action")]
719
720     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir"]:
721         if not Cnf.has_key("Process-New::Options::%s" % (i)):
722             Cnf["Process-New::Options::%s" % (i)] = ""
723
724     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
725     Options = Cnf.SubTree("Process-New::Options")
726
727     if Options["Help"]:
728         usage()
729
730     Upload = queue.Upload(Cnf)
731
732     if not Options["No-Action"]:
733         Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
734
735     projectB = Upload.projectB
736
737     Sections = Section_Completer()
738     Priorities = Priority_Completer()
739     readline.parse_and_bind("tab: complete")
740
741     return changes_files
742
743 ################################################################################
744
745 def do_byhand():
746     done = 0
747     while not done:
748         files = Upload.pkg.files
749         will_install = 1
750         byhand = []
751
752         for f in files.keys():
753             if files[f]["type"] == "byhand":
754                 if os.path.exists(f):
755                     print "W: %s still present; please process byhand components and try again." % (f)
756                     will_install = 0
757                 else:
758                     byhand.append(f)
759
760         answer = "XXXX"
761         if Options["No-Action"]:
762             answer = "S"
763         if will_install:
764             if Options["Automatic"] and not Options["No-Action"]:
765                 answer = 'A'
766             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
767         else:
768             prompt = "Manual reject, [S]kip, Quit ?"
769
770         while prompt.find(answer) == -1:
771             answer = utils.our_raw_input(prompt)
772             m = queue.re_default_answer.search(prompt)
773             if answer == "":
774                 answer = m.group(1)
775             answer = answer[:1].upper()
776
777         if answer == 'A':
778             done = 1
779             for f in byhand:
780                 del files[f]
781         elif answer == 'M':
782             Upload.do_reject(1, Options["Manual-Reject"])
783             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
784             done = 1
785         elif answer == 'S':
786             done = 1
787         elif answer == 'Q':
788             end()
789             sys.exit(0)
790
791 ################################################################################
792
793 def get_accept_lock():
794     retry = 0
795     while retry < 10:
796         try:
797             os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
798             retry = 10
799         except OSError, e:
800             if e.errno == errno.EACCES or e.errno == errno.EEXIST:
801                 retry += 1
802                 if (retry >= 10):
803                     utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
804                 else:
805                     print("Unable to get accepted lock (try %d of 10)" % retry)
806                 time.sleep(60)
807             else:
808                 raise
809
810 def move_to_dir (dest, perms=0660, changesperms=0664):
811     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
812     file_keys = Upload.pkg.files.keys()
813     for f in file_keys:
814         utils.move (f, dest, perms=perms)
815
816 def is_source_in_queue_dir(qdir):
817     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
818                 and x.endswith(".changes") ]
819     for entry in entries:
820         # read the .dak
821         u = queue.Upload(Cnf)
822         u.pkg.changes_file = os.path.join(qdir, entry)
823         u.update_vars()
824         if not Upload.pkg.changes["architecture"].has_key("source"):
825             # another binary upload, ignore
826             continue
827         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
828             # another version, ignore
829             continue
830         # found it!
831         return True
832     return False
833
834 def move_to_holding(suite, queue_dir):
835     print "Moving to %s holding area." % (suite.upper(),)
836     if Options["No-Action"]:
837         return
838     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
839     Upload.dump_vars(queue_dir)
840     move_to_dir(queue_dir)
841     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
842
843 def do_accept_stableupdate(suite, q):
844     (summary, short_summary) = Upload.build_summaries()
845     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
846     if not Upload.pkg.changes["architecture"].has_key("source"):
847         # It is not a sourceful upload.  So its source may be either in p-u
848         # holding, in new, in accepted or already installed.
849         if is_source_in_queue_dir(queue_dir):
850             # It's in p-u holding, so move it there.
851             print "Binary-only upload, source in %s." % (q,)
852             move_to_holding(suite, queue_dir)
853         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
854             # It's in NEW.  We expect the source to land in p-u holding
855             # pretty soon.
856             print "Binary-only upload, source in new."
857             move_to_holding(suite, queue_dir)
858         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
859             # The source is in accepted, the binary cleared NEW: accept it.
860             print "Binary-only upload, source in accepted."
861             Upload.accept(summary, short_summary)
862             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
863         elif Upload.source_exists(Upload.pkg.changes["source"],
864                 Upload.pkg.changes["version"]):
865             # dak tells us that there is source available.  At time of
866             # writing this means that it is installed, so put it into
867             # accepted.
868             print "Binary-only upload, source installed."
869             Upload.accept(summary, short_summary)
870             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
871     else:
872         # We are handling a sourceful upload.  Move to accepted if currently
873         # in p-u holding and to p-u holding otherwise.
874         if is_source_in_queue_dir(queue_dir):
875             print "Sourceful upload in %s, accepting." % (q,)
876             Upload.accept(summary, short_summary)
877             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
878         else:
879             move_to_holding(suite, queue_dir)
880
881 def do_accept():
882     print "ACCEPT"
883     if not Options["No-Action"]:
884         get_accept_lock()
885         (summary, short_summary) = Upload.build_summaries()
886     try:
887         if Cnf.FindB("Dinstall::SecurityQueueHandling"):
888             Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
889             move_to_dir(Cnf["Dir::Queue::Embargoed"])
890             Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
891             # Check for override disparities
892             Upload.Subst["__SUMMARY__"] = summary
893         else:
894             # Stable updates need to be copied to proposed-updates holding
895             # area instead of accepted.  Sourceful uploads need to go
896             # to it directly, binaries only if the source has not yet been
897             # accepted into p-u.
898             for suite, q in [("proposed-updates", "ProposedUpdates"),
899                     ("oldstable-proposed-updates", "OldProposedUpdates")]:
900                 if not Upload.pkg.changes["distribution"].has_key(suite):
901                     continue
902                 return do_accept_stableupdate(suite, q)
903             # Just a normal upload, accept it...
904             Upload.accept(summary, short_summary)
905             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
906     finally:
907         if not Options["No-Action"]:
908             os.unlink(Cnf["Process-New::AcceptedLockFile"])
909
910 def check_status(files):
911     new = byhand = 0
912     for f in files.keys():
913         if files[f]["type"] == "byhand":
914             byhand = 1
915         elif files[f].has_key("new"):
916             new = 1
917     return (new, byhand)
918
919 def do_pkg(changes_file):
920     Upload.pkg.changes_file = changes_file
921     Upload.init_vars()
922     Upload.update_vars()
923     Upload.update_subst()
924     files = Upload.pkg.files
925
926     if not recheck():
927         return
928
929     (new, byhand) = check_status(files)
930     if new or byhand:
931         if new:
932             do_new()
933         if byhand:
934             do_byhand()
935         (new, byhand) = check_status(files)
936
937     if not new and not byhand:
938         do_accept()
939
940 ################################################################################
941
942 def end():
943     accept_count = Upload.accept_count
944     accept_bytes = Upload.accept_bytes
945
946     if accept_count:
947         sets = "set"
948         if accept_count > 1:
949             sets = "sets"
950         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
951         Logger.log(["total",accept_count,accept_bytes])
952
953     if not Options["No-Action"]:
954         Logger.close()
955
956 ################################################################################
957
958 def do_comments(dir, opref, npref, line, fn):
959     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
960         lines = open("%s/%s" % (dir, comm)).readlines()
961         if len(lines) == 0 or lines[0] != line + "\n": continue
962         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
963                                 and x.endswith(".changes") ]
964         changes_files = sort_changes(changes_files)
965         for f in changes_files:
966             f = utils.validate_changes_file_arg(f, 0)
967             if not f: continue
968             print "\n" + f
969             fn(f, "".join(lines[1:]))
970
971         if opref != npref and not Options["No-Action"]:
972             newcomm = npref + comm[len(opref):]
973             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
974
975 ################################################################################
976
977 def comment_accept(changes_file, comments):
978     Upload.pkg.changes_file = changes_file
979     Upload.init_vars()
980     Upload.update_vars()
981     Upload.update_subst()
982     files = Upload.pkg.files
983
984     if not recheck():
985         return # dak wants to REJECT, crap
986
987     (new, byhand) = check_status(files)
988     if not new and not byhand:
989         do_accept()
990
991 ################################################################################
992
993 def comment_reject(changes_file, comments):
994     Upload.pkg.changes_file = changes_file
995     Upload.init_vars()
996     Upload.update_vars()
997     Upload.update_subst()
998
999     if not recheck():
1000         pass # dak has its own reasons to reject as well, which is fine
1001
1002     reject(comments)
1003     print "REJECT\n" + reject_message,
1004     if not Options["No-Action"]:
1005         Upload.do_reject(0, reject_message)
1006         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1007
1008 ################################################################################
1009
1010 def main():
1011     changes_files = init()
1012     if len(changes_files) > 50:
1013         sys.stderr.write("Sorting changes...\n")
1014     changes_files = sort_changes(changes_files)
1015
1016     # Kill me now? **FIXME**
1017     Cnf["Dinstall::Options::No-Mail"] = ""
1018     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1019     if Cnf.has_key("Dinstall::Bcc"):
1020         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1021     else:
1022         Upload.Subst["__BCC__"] = bcc
1023
1024     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1025     if commentsdir:
1026         if changes_files != []:
1027             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1028             sys.exit(1)
1029         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1030         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1031     else:
1032         for changes_file in changes_files:
1033             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1034             if not changes_file:
1035                 continue
1036             print "\n" + changes_file
1037             do_pkg (changes_file)
1038
1039     end()
1040
1041 ################################################################################
1042
1043 if __name__ == '__main__':
1044     main()