dak/process_new.py

   1 #!/usr/bin/env python
   2
   3 # Handles NEW and BYHAND packages
   4 # Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   5
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19
  20 ################################################################################
  21
  22 # 23:12|<aj> I will not hush!
  23 # 23:12|<elmo> :>
  24 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  25 # 23:13|<aj> I shall not be silenced!
  26 # 23:13|<aj> The world shall know!
  27 # 23:13|<aj> The world *must* know!
  28 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  29 # 23:13|<aj> yay powerpuff girls!!
  30 # 23:13|<aj> buttercup's my favourite, who's yours?
  31 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  32 # 23:14|<aj> *AREN'T YOU*?!
  33 # 23:15|<aj> I will not be treated like this.
  34 # 23:15|<aj> I shall have my revenge.
  35 # 23:15|<aj> I SHALL!!!
  36
  37 ################################################################################
  38
  39 import copy, errno, os, readline, stat, sys, time
  40 import apt_pkg, apt_inst
  41 import examine_package
  42 from daklib import database
  43 from daklib import logging
  44 from daklib import queue
  45 from daklib import utils
  46
  47 # Globals
  48 Cnf = None
  49 Options = None
  50 Upload = None
  51 projectB = None
  52 Logger = None
  53
  54 Priorities = None
  55 Sections = None
  56
  57 reject_message = ""
  58
  59 ################################################################################
  60 ################################################################################
  61 ################################################################################
  62
  63 def reject (str, prefix="Rejected: "):
  64     global reject_message
  65     if str:
  66         reject_message += prefix + str + "\n"
  67
  68 def recheck():
  69     global reject_message
  70     files = Upload.pkg.files
  71     reject_message = ""
  72
  73     for f in files.keys():
  74         # The .orig.tar.gz can disappear out from under us is it's a
  75         # duplicate of one in the archive.
  76         if not files.has_key(f):
  77             continue
  78         # Check that the source still exists
  79         if files[f]["type"] == "deb":
  80             source_version = files[f]["source version"]
  81             source_package = files[f]["source package"]
  82             if not Upload.pkg.changes["architecture"].has_key("source") \
  83                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
  84                 source_epochless_version = utils.re_no_epoch.sub('', source_version)
  85                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
  86                 found = 0
  87                 for q in ["Accepted", "Embargoed", "Unembargoed"]:
  88                     if Cnf.has_key("Dir::Queue::%s" % (q)):
  89                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
  90                             found = 1
  91                 if not found:
  92                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
  93
  94         # Version and file overwrite checks
  95         if files[f]["type"] == "deb":
  96             reject(Upload.check_binary_against_db(f))
  97         elif files[f]["type"] == "dsc":
  98             reject(Upload.check_source_against_db(f))
  99             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
 100             reject(reject_msg, "")
 101
 102     if reject_message.find("Rejected") != -1:
 103         answer = "XXX"
 104         if Options["No-Action"] or Options["Automatic"]:
 105             answer = 'S'
 106
 107         print "REJECT\n" + reject_message,
 108         prompt = "[R]eject, Skip, Quit ?"
 109
 110         while prompt.find(answer) == -1:
 111             answer = utils.our_raw_input(prompt)
 112             m = queue.re_default_answer.match(prompt)
 113             if answer == "":
 114                 answer = m.group(1)
 115             answer = answer[:1].upper()
 116
 117         if answer == 'R':
 118             Upload.do_reject(0, reject_message)
 119             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 120             return 0
 121         elif answer == 'S':
 122             return 0
 123         elif answer == 'Q':
 124             end()
 125             sys.exit(0)
 126
 127     return 1
 128
 129 ################################################################################
 130
 131 def indiv_sg_compare (a, b):
 132     """Sort by source name, source, version, 'have source', and
 133        finally by filename."""
 134     # Sort by source version
 135     q = apt_pkg.VersionCompare(a["version"], b["version"])
 136     if q:
 137         return -q
 138
 139     # Sort by 'have source'
 140     a_has_source = a["architecture"].get("source")
 141     b_has_source = b["architecture"].get("source")
 142     if a_has_source and not b_has_source:
 143         return -1
 144     elif b_has_source and not a_has_source:
 145         return 1
 146
 147     return cmp(a["filename"], b["filename"])
 148
 149 ############################################################
 150
 151 def sg_compare (a, b):
 152     a = a[1]
 153     b = b[1]
 154     """Sort by have note, source already in database and time of oldest upload."""
 155     # Sort by have note
 156     a_note_state = a["note_state"]
 157     b_note_state = b["note_state"]
 158     if a_note_state < b_note_state:
 159         return -1
 160     elif a_note_state > b_note_state:
 161         return 1
 162     # Sort by source already in database (descending)
 163     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 164     if source_in_database:
 165         return -source_in_database
 166
 167     # Sort by time of oldest upload
 168     return cmp(a["oldest"], b["oldest"])
 169
 170 def sort_changes(changes_files):
 171     """Sort into source groups, then sort each source group by version,
 172     have source, filename.  Finally, sort the source groups by have
 173     note, time of oldest upload of each source upload."""
 174     if len(changes_files) == 1:
 175         return changes_files
 176
 177     sorted_list = []
 178     cache = {}
 179     # Read in all the .changes files
 180     for filename in changes_files:
 181         try:
 182             Upload.pkg.changes_file = filename
 183             Upload.init_vars()
 184             Upload.update_vars()
 185             cache[filename] = copy.copy(Upload.pkg.changes)
 186             cache[filename]["filename"] = filename
 187         except:
 188             sorted_list.append(filename)
 189             break
 190     # Divide the .changes into per-source groups
 191     per_source = {}
 192     for filename in cache.keys():
 193         source = cache[filename]["source"]
 194         if not per_source.has_key(source):
 195             per_source[source] = {}
 196             per_source[source]["list"] = []
 197         per_source[source]["list"].append(cache[filename])
 198     # Determine oldest time and have note status for each source group
 199     for source in per_source.keys():
 200         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
 201         ql = q.getresult()
 202         per_source[source]["source_in_database"] = len(ql)>0
 203         source_list = per_source[source]["list"]
 204         first = source_list[0]
 205         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 206         have_note = 0
 207         for d in per_source[source]["list"]:
 208             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 209             if mtime < oldest:
 210                 oldest = mtime
 211             have_note += (d.has_key("process-new note"))
 212         per_source[source]["oldest"] = oldest
 213         if not have_note:
 214             per_source[source]["note_state"] = 0; # none
 215         elif have_note < len(source_list):
 216             per_source[source]["note_state"] = 1; # some
 217         else:
 218             per_source[source]["note_state"] = 2; # all
 219         per_source[source]["list"].sort(indiv_sg_compare)
 220     per_source_items = per_source.items()
 221     per_source_items.sort(sg_compare)
 222     for i in per_source_items:
 223         for j in i[1]["list"]:
 224             sorted_list.append(j["filename"])
 225     return sorted_list
 226
 227 ################################################################################
 228
 229 class Section_Completer:
 230     def __init__ (self):
 231         self.sections = []
 232         q = projectB.query("SELECT section FROM section")
 233         for i in q.getresult():
 234             self.sections.append(i[0])
 235
 236     def complete(self, text, state):
 237         if state == 0:
 238             self.matches = []
 239             n = len(text)
 240             for word in self.sections:
 241                 if word[:n] == text:
 242                     self.matches.append(word)
 243         try:
 244             return self.matches[state]
 245         except IndexError:
 246             return None
 247
 248 ############################################################
 249
 250 class Priority_Completer:
 251     def __init__ (self):
 252         self.priorities = []
 253         q = projectB.query("SELECT priority FROM priority")
 254         for i in q.getresult():
 255             self.priorities.append(i[0])
 256
 257     def complete(self, text, state):
 258         if state == 0:
 259             self.matches = []
 260             n = len(text)
 261             for word in self.priorities:
 262                 if word[:n] == text:
 263                     self.matches.append(word)
 264         try:
 265             return self.matches[state]
 266         except IndexError:
 267             return None
 268
 269 ################################################################################
 270
 271 def print_new (new, indexed, file=sys.stdout):
 272     queue.check_valid(new)
 273     broken = 0
 274     index = 0
 275     for pkg in new.keys():
 276         index += 1
 277         section = new[pkg]["section"]
 278         priority = new[pkg]["priority"]
 279         if new[pkg]["section id"] == -1:
 280             section += "[!]"
 281             broken = 1
 282         if new[pkg]["priority id"] == -1:
 283             priority += "[!]"
 284             broken = 1
 285         if indexed:
 286             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 287         else:
 288             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 289         line = line.strip()+'\n'
 290         file.write(line)
 291     note = Upload.pkg.changes.get("process-new note")
 292     if note:
 293         print "*"*75
 294         print note
 295         print "*"*75
 296     return broken, note
 297
 298 ################################################################################
 299
 300 def index_range (index):
 301     if index == 1:
 302         return "1"
 303     else:
 304         return "1-%s" % (index)
 305
 306 ################################################################################
 307 ################################################################################
 308
 309 def edit_new (new):
 310     # Write the current data to a temporary file
 311     temp_filename = utils.temp_filename()
 312     temp_file = utils.open_file(temp_filename, 'w')
 313     print_new (new, 0, temp_file)
 314     temp_file.close()
 315     # Spawn an editor on that file
 316     editor = os.environ.get("EDITOR","vi")
 317     result = os.system("%s %s" % (editor, temp_filename))
 318     if result != 0:
 319         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 320     # Read the edited data back in
 321     temp_file = utils.open_file(temp_filename)
 322     lines = temp_file.readlines()
 323     temp_file.close()
 324     os.unlink(temp_filename)
 325     # Parse the new data
 326     for line in lines:
 327         line = line.strip()
 328         if line == "":
 329             continue
 330         s = line.split()
 331         # Pad the list if necessary
 332         s[len(s):3] = [None] * (3-len(s))
 333         (pkg, priority, section) = s[:3]
 334         if not new.has_key(pkg):
 335             utils.warn("Ignoring unknown package '%s'" % (pkg))
 336         else:
 337             # Strip off any invalid markers, print_new will readd them.
 338             if section.endswith("[!]"):
 339                 section = section[:-3]
 340             if priority.endswith("[!]"):
 341                 priority = priority[:-3]
 342             for f in new[pkg]["files"]:
 343                 Upload.pkg.files[f]["section"] = section
 344                 Upload.pkg.files[f]["priority"] = priority
 345             new[pkg]["section"] = section
 346             new[pkg]["priority"] = priority
 347
 348 ################################################################################
 349
 350 def edit_index (new, index):
 351     priority = new[index]["priority"]
 352     section = new[index]["section"]
 353     ftype = new[index]["type"]
 354     done = 0
 355     while not done:
 356         print "\t".join([index, priority, section])
 357
 358         answer = "XXX"
 359         if ftype != "dsc":
 360             prompt = "[B]oth, Priority, Section, Done ? "
 361         else:
 362             prompt = "[S]ection, Done ? "
 363         edit_priority = edit_section = 0
 364
 365         while prompt.find(answer) == -1:
 366             answer = utils.our_raw_input(prompt)
 367             m = queue.re_default_answer.match(prompt)
 368             if answer == "":
 369                 answer = m.group(1)
 370             answer = answer[:1].upper()
 371
 372         if answer == 'P':
 373             edit_priority = 1
 374         elif answer == 'S':
 375             edit_section = 1
 376         elif answer == 'B':
 377             edit_priority = edit_section = 1
 378         elif answer == 'D':
 379             done = 1
 380
 381         # Edit the priority
 382         if edit_priority:
 383             readline.set_completer(Priorities.complete)
 384             got_priority = 0
 385             while not got_priority:
 386                 new_priority = utils.our_raw_input("New priority: ").strip()
 387                 if new_priority not in Priorities.priorities:
 388                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 389                 else:
 390                     got_priority = 1
 391                     priority = new_priority
 392
 393         # Edit the section
 394         if edit_section:
 395             readline.set_completer(Sections.complete)
 396             got_section = 0
 397             while not got_section:
 398                 new_section = utils.our_raw_input("New section: ").strip()
 399                 if new_section not in Sections.sections:
 400                     print "E: '%s' is not a valid section, try again." % (new_section)
 401                 else:
 402                     got_section = 1
 403                     section = new_section
 404
 405         # Reset the readline completer
 406         readline.set_completer(None)
 407
 408     for f in new[index]["files"]:
 409         Upload.pkg.files[f]["section"] = section
 410         Upload.pkg.files[f]["priority"] = priority
 411     new[index]["priority"] = priority
 412     new[index]["section"] = section
 413     return new
 414
 415 ################################################################################
 416
 417 def edit_overrides (new):
 418     print
 419     done = 0
 420     while not done:
 421         print_new (new, 1)
 422         new_index = {}
 423         index = 0
 424         for i in new.keys():
 425             index += 1
 426             new_index[index] = i
 427
 428         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 429
 430         got_answer = 0
 431         while not got_answer:
 432             answer = utils.our_raw_input(prompt)
 433             if not answer.isdigit():
 434                 answer = answer[:1].upper()
 435             if answer == "E" or answer == "D":
 436                 got_answer = 1
 437             elif queue.re_isanum.match (answer):
 438                 answer = int(answer)
 439                 if (answer < 1) or (answer > index):
 440                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 441                 else:
 442                     got_answer = 1
 443
 444         if answer == 'E':
 445             edit_new(new)
 446         elif answer == 'D':
 447             done = 1
 448         else:
 449             edit_index (new, new_index[answer])
 450
 451     return new
 452
 453 ################################################################################
 454
 455 def edit_note(note):
 456     # Write the current data to a temporary file
 457     temp_filename = utils.temp_filename()
 458     temp_file = utils.open_file(temp_filename, 'w')
 459     temp_file.write(note)
 460     temp_file.close()
 461     editor = os.environ.get("EDITOR","vi")
 462     answer = 'E'
 463     while answer == 'E':
 464         os.system("%s %s" % (editor, temp_filename))
 465         temp_file = utils.open_file(temp_filename)
 466         note = temp_file.read().rstrip()
 467         temp_file.close()
 468         print "Note:"
 469         print utils.prefix_multi_line_string(note,"  ")
 470         prompt = "[D]one, Edit, Abandon, Quit ?"
 471         answer = "XXX"
 472         while prompt.find(answer) == -1:
 473             answer = utils.our_raw_input(prompt)
 474             m = queue.re_default_answer.search(prompt)
 475             if answer == "":
 476                 answer = m.group(1)
 477             answer = answer[:1].upper()
 478     os.unlink(temp_filename)
 479     if answer == 'A':
 480         return
 481     elif answer == 'Q':
 482         end()
 483         sys.exit(0)
 484     Upload.pkg.changes["process-new note"] = note
 485     Upload.dump_vars(Cnf["Dir::Queue::New"])
 486
 487 ################################################################################
 488
 489 def check_pkg ():
 490     try:
 491         less_fd = os.popen("less -R -", 'w', 0)
 492         stdout_fd = sys.stdout
 493         try:
 494             sys.stdout = less_fd
 495             examine_package.display_changes(Upload.pkg.changes_file)
 496             files = Upload.pkg.files
 497             for f in files.keys():
 498                 if files[f].has_key("new"):
 499                     ftype = files[f]["type"]
 500                     if ftype == "deb":
 501                         examine_package.check_deb(f)
 502                     elif ftype == "dsc":
 503                         examine_package.check_dsc(f)
 504         finally:
 505             sys.stdout = stdout_fd
 506     except IOError, e:
 507         if e.errno == errno.EPIPE:
 508             utils.warn("[examine_package] Caught EPIPE; skipping.")
 509             pass
 510         else:
 511             raise
 512     except KeyboardInterrupt:
 513         utils.warn("[examine_package] Caught C-c; skipping.")
 514         pass
 515
 516 ################################################################################
 517
 518 ## FIXME: horribly Debian specific
 519
 520 def do_bxa_notification():
 521     files = Upload.pkg.files
 522     summary = ""
 523     for f in files.keys():
 524         if files[f]["type"] == "deb":
 525             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 526             summary += "\n"
 527             summary += "Package: %s\n" % (control.Find("Package"))
 528             summary += "Description: %s\n" % (control.Find("Description"))
 529     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 530     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 531     utils.send_mail(bxa_mail)
 532
 533 ################################################################################
 534
 535 def add_overrides (new):
 536     changes = Upload.pkg.changes
 537     files = Upload.pkg.files
 538
 539     projectB.query("BEGIN WORK")
 540     for suite in changes["suite"].keys():
 541         suite_id = database.get_suite_id(suite)
 542         for pkg in new.keys():
 543             component_id = database.get_component_id(new[pkg]["component"])
 544             type_id = database.get_override_type_id(new[pkg]["type"])
 545             priority_id = new[pkg]["priority id"]
 546             section_id = new[pkg]["section id"]
 547             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 548             for f in new[pkg]["files"]:
 549                 if files[f].has_key("new"):
 550                     del files[f]["new"]
 551             del new[pkg]
 552
 553     projectB.query("COMMIT WORK")
 554
 555     if Cnf.FindB("Dinstall::BXANotify"):
 556         do_bxa_notification()
 557
 558 ################################################################################
 559
 560 def prod_maintainer ():
 561     # Here we prepare an editor and get them ready to prod...
 562     temp_filename = utils.temp_filename()
 563     editor = os.environ.get("EDITOR","vi")
 564     answer = 'E'
 565     while answer == 'E':
 566         os.system("%s %s" % (editor, temp_filename))
 567         f = utils.open_file(temp_filename)
 568         prod_message = "".join(f.readlines())
 569         f.close()
 570         print "Prod message:"
 571         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 572         prompt = "[P]rod, Edit, Abandon, Quit ?"
 573         answer = "XXX"
 574         while prompt.find(answer) == -1:
 575             answer = utils.our_raw_input(prompt)
 576             m = queue.re_default_answer.search(prompt)
 577             if answer == "":
 578                 answer = m.group(1)
 579             answer = answer[:1].upper()
 580         os.unlink(temp_filename)
 581         if answer == 'A':
 582             return
 583         elif answer == 'Q':
 584             end()
 585             sys.exit(0)
 586     # Otherwise, do the proding...
 587     user_email_address = utils.whoami() + " <%s>" % (
 588         Cnf["Dinstall::MyAdminAddress"])
 589
 590     Subst = Upload.Subst
 591
 592     Subst["__FROM_ADDRESS__"] = user_email_address
 593     Subst["__PROD_MESSAGE__"] = prod_message
 594     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 595
 596     prod_mail_message = utils.TemplateSubst(
 597         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 598
 599     # Send the prod mail if appropriate
 600     if not Cnf["Dinstall::Options::No-Mail"]:
 601         utils.send_mail(prod_mail_message)
 602
 603     print "Sent proding message"
 604
 605 ################################################################################
 606
 607 def do_new():
 608     print "NEW\n"
 609     files = Upload.pkg.files
 610     changes = Upload.pkg.changes
 611
 612     # Make a copy of distribution we can happily trample on
 613     changes["suite"] = copy.copy(changes["distribution"])
 614
 615     # Fix up the list of target suites
 616     for suite in changes["suite"].keys():
 617         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 618         if override:
 619             (olderr, newerr) = (database.get_suite_id(suite) == -1,
 620               database.get_suite_id(override) == -1)
 621             if olderr or newerr:
 622                 (oinv, newinv) = ("", "")
 623                 if olderr: oinv = "invalid "
 624                 if newerr: ninv = "invalid "
 625                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 626                         oinv, suite, ninv, override)
 627             del changes["suite"][suite]
 628             changes["suite"][override] = 1
 629     # Validate suites
 630     for suite in changes["suite"].keys():
 631         suite_id = database.get_suite_id(suite)
 632         if suite_id == -1:
 633             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 634
 635     # The main NEW processing loop
 636     done = 0
 637     while not done:
 638         # Find out what's new
 639         new = queue.determine_new(changes, files, projectB)
 640
 641         if not new:
 642             break
 643
 644         answer = "XXX"
 645         if Options["No-Action"] or Options["Automatic"]:
 646             answer = 'S'
 647
 648         (broken, note) = print_new(new, 0)
 649         prompt = ""
 650
 651         if not broken and not note:
 652             prompt = "Add overrides, "
 653         if broken:
 654             print "W: [!] marked entries must be fixed before package can be processed."
 655         if note:
 656             print "W: note must be removed before package can be processed."
 657             prompt += "Remove note, "
 658
 659         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 660
 661         while prompt.find(answer) == -1:
 662             answer = utils.our_raw_input(prompt)
 663             m = queue.re_default_answer.search(prompt)
 664             if answer == "":
 665                 answer = m.group(1)
 666             answer = answer[:1].upper()
 667
 668         if answer == 'A':
 669             done = add_overrides (new)
 670         elif answer == 'C':
 671             check_pkg()
 672         elif answer == 'E':
 673             new = edit_overrides (new)
 674         elif answer == 'M':
 675             aborted = Upload.do_reject(1, Options["Manual-Reject"])
 676             if not aborted:
 677                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 678                 done = 1
 679         elif answer == 'N':
 680             edit_note(changes.get("process-new note", ""))
 681         elif answer == 'P':
 682             prod_maintainer()
 683         elif answer == 'R':
 684             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 685             if confirm == "y":
 686                 del changes["process-new note"]
 687         elif answer == 'S':
 688             done = 1
 689         elif answer == 'Q':
 690             end()
 691             sys.exit(0)
 692
 693 ################################################################################
 694 ################################################################################
 695 ################################################################################
 696
 697 def usage (exit_code=0):
 698     print """Usage: dak process-new [OPTION]... [CHANGES]...
 699   -a, --automatic           automatic run
 700   -h, --help                show this help and exit.
 701   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 702   -m, --manual-reject=MSG   manual reject with `msg'
 703   -n, --no-action           don't do anything
 704   -V, --version             display the version number and exit"""
 705     sys.exit(exit_code)
 706
 707 ################################################################################
 708
 709 def init():
 710     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 711
 712     Cnf = utils.get_conf()
 713
 714     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 715                  ('h',"help","Process-New::Options::Help"),
 716                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
 717                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 718                  ('n',"no-action","Process-New::Options::No-Action")]
 719
 720     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir"]:
 721         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 722             Cnf["Process-New::Options::%s" % (i)] = ""
 723
 724     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 725     Options = Cnf.SubTree("Process-New::Options")
 726
 727     if Options["Help"]:
 728         usage()
 729
 730     Upload = queue.Upload(Cnf)
 731
 732     if not Options["No-Action"]:
 733         Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
 734
 735     projectB = Upload.projectB
 736
 737     Sections = Section_Completer()
 738     Priorities = Priority_Completer()
 739     readline.parse_and_bind("tab: complete")
 740
 741     return changes_files
 742
 743 ################################################################################
 744
 745 def do_byhand():
 746     done = 0
 747     while not done:
 748         files = Upload.pkg.files
 749         will_install = 1
 750         byhand = []
 751
 752         for f in files.keys():
 753             if files[f]["type"] == "byhand":
 754                 if os.path.exists(f):
 755                     print "W: %s still present; please process byhand components and try again." % (f)
 756                     will_install = 0
 757                 else:
 758                     byhand.append(f)
 759
 760         answer = "XXXX"
 761         if Options["No-Action"]:
 762             answer = "S"
 763         if will_install:
 764             if Options["Automatic"] and not Options["No-Action"]:
 765                 answer = 'A'
 766             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 767         else:
 768             prompt = "Manual reject, [S]kip, Quit ?"
 769
 770         while prompt.find(answer) == -1:
 771             answer = utils.our_raw_input(prompt)
 772             m = queue.re_default_answer.search(prompt)
 773             if answer == "":
 774                 answer = m.group(1)
 775             answer = answer[:1].upper()
 776
 777         if answer == 'A':
 778             done = 1
 779             for f in byhand:
 780                 del files[f]
 781         elif answer == 'M':
 782             Upload.do_reject(1, Options["Manual-Reject"])
 783             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 784             done = 1
 785         elif answer == 'S':
 786             done = 1
 787         elif answer == 'Q':
 788             end()
 789             sys.exit(0)
 790
 791 ################################################################################
 792
 793 def get_accept_lock():
 794     retry = 0
 795     while retry < 10:
 796         try:
 797             os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 798             retry = 10
 799         except OSError, e:
 800             if e.errno == errno.EACCES or e.errno == errno.EEXIST:
 801                 retry += 1
 802                 if (retry >= 10):
 803                     utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
 804                 else:
 805                     print("Unable to get accepted lock (try %d of 10)" % retry)
 806                 time.sleep(60)
 807             else:
 808                 raise
 809
 810 def move_to_dir (dest, perms=0660, changesperms=0664):
 811     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
 812     file_keys = Upload.pkg.files.keys()
 813     for f in file_keys:
 814         utils.move (f, dest, perms=perms)
 815
 816 def is_source_in_queue_dir(qdir):
 817     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 818                 and x.endswith(".changes") ]
 819     for entry in entries:
 820         # read the .dak
 821         u = queue.Upload(Cnf)
 822         u.pkg.changes_file = os.path.join(qdir, entry)
 823         u.update_vars()
 824         if not Upload.pkg.changes["architecture"].has_key("source"):
 825             # another binary upload, ignore
 826             continue
 827         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 828             # another version, ignore
 829             continue
 830         # found it!
 831         return True
 832     return False
 833
 834 def move_to_holding(suite, queue_dir):
 835     print "Moving to %s holding area." % (suite.upper(),)
 836     if Options["No-Action"]:
 837         return
 838     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 839     Upload.dump_vars(queue_dir)
 840     move_to_dir(queue_dir)
 841     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 842
 843 def do_accept_stableupdate(suite, q):
 844     (summary, short_summary) = Upload.build_summaries()
 845     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
 846     if not Upload.pkg.changes["architecture"].has_key("source"):
 847         # It is not a sourceful upload.  So its source may be either in p-u
 848         # holding, in new, in accepted or already installed.
 849         if is_source_in_queue_dir(queue_dir):
 850             # It's in p-u holding, so move it there.
 851             print "Binary-only upload, source in %s." % (q,)
 852             move_to_holding(suite, queue_dir)
 853         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 854             # It's in NEW.  We expect the source to land in p-u holding
 855             # pretty soon.
 856             print "Binary-only upload, source in new."
 857             move_to_holding(suite, queue_dir)
 858         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 859             # The source is in accepted, the binary cleared NEW: accept it.
 860             print "Binary-only upload, source in accepted."
 861             Upload.accept(summary, short_summary)
 862             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 863         elif Upload.source_exists(Upload.pkg.changes["source"],
 864                 Upload.pkg.changes["version"]):
 865             # dak tells us that there is source available.  At time of
 866             # writing this means that it is installed, so put it into
 867             # accepted.
 868             print "Binary-only upload, source installed."
 869             Upload.accept(summary, short_summary)
 870             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 871     else:
 872         # We are handling a sourceful upload.  Move to accepted if currently
 873         # in p-u holding and to p-u holding otherwise.
 874         if is_source_in_queue_dir(queue_dir):
 875             print "Sourceful upload in %s, accepting." % (q,)
 876             Upload.accept(summary, short_summary)
 877             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 878         else:
 879             move_to_holding(suite, queue_dir)
 880
 881 def do_accept():
 882     print "ACCEPT"
 883     if not Options["No-Action"]:
 884         get_accept_lock()
 885         (summary, short_summary) = Upload.build_summaries()
 886     try:
 887         if Cnf.FindB("Dinstall::SecurityQueueHandling"):
 888             Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
 889             move_to_dir(Cnf["Dir::Queue::Embargoed"])
 890             Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
 891             # Check for override disparities
 892             Upload.Subst["__SUMMARY__"] = summary
 893         else:
 894             # Stable updates need to be copied to proposed-updates holding
 895             # area instead of accepted.  Sourceful uploads need to go
 896             # to it directly, binaries only if the source has not yet been
 897             # accepted into p-u.
 898             for suite, q in [("proposed-updates", "ProposedUpdates"),
 899                     ("oldstable-proposed-updates", "OldProposedUpdates")]:
 900                 if not Upload.pkg.changes["distribution"].has_key(suite):
 901                     continue
 902                 return do_accept_stableupdate(suite, q)
 903             # Just a normal upload, accept it...
 904             Upload.accept(summary, short_summary)
 905             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 906     finally:
 907         if not Options["No-Action"]:
 908             os.unlink(Cnf["Process-New::AcceptedLockFile"])
 909
 910 def check_status(files):
 911     new = byhand = 0
 912     for f in files.keys():
 913         if files[f]["type"] == "byhand":
 914             byhand = 1
 915         elif files[f].has_key("new"):
 916             new = 1
 917     return (new, byhand)
 918
 919 def do_pkg(changes_file):
 920     Upload.pkg.changes_file = changes_file
 921     Upload.init_vars()
 922     Upload.update_vars()
 923     Upload.update_subst()
 924     files = Upload.pkg.files
 925
 926     if not recheck():
 927         return
 928
 929     (new, byhand) = check_status(files)
 930     if new or byhand:
 931         if new:
 932             do_new()
 933         if byhand:
 934             do_byhand()
 935         (new, byhand) = check_status(files)
 936
 937     if not new and not byhand:
 938         do_accept()
 939
 940 ################################################################################
 941
 942 def end():
 943     accept_count = Upload.accept_count
 944     accept_bytes = Upload.accept_bytes
 945
 946     if accept_count:
 947         sets = "set"
 948         if accept_count > 1:
 949             sets = "sets"
 950         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
 951         Logger.log(["total",accept_count,accept_bytes])
 952
 953     if not Options["No-Action"]:
 954         Logger.close()
 955
 956 ################################################################################
 957
 958 def do_comments(dir, opref, npref, line, fn):
 959     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
 960         lines = open("%s/%s" % (dir, comm)).readlines()
 961         if len(lines) == 0 or lines[0] != line + "\n": continue
 962         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
 963                                 and x.endswith(".changes") ]
 964         changes_files = sort_changes(changes_files)
 965         for f in changes_files:
 966             f = utils.validate_changes_file_arg(f, 0)
 967             if not f: continue
 968             print "\n" + f
 969             fn(f, "".join(lines[1:]))
 970
 971         if opref != npref and not Options["No-Action"]:
 972             newcomm = npref + comm[len(opref):]
 973             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
 974
 975 ################################################################################
 976
 977 def comment_accept(changes_file, comments):
 978     Upload.pkg.changes_file = changes_file
 979     Upload.init_vars()
 980     Upload.update_vars()
 981     Upload.update_subst()
 982     files = Upload.pkg.files
 983
 984     if not recheck():
 985         return # dak wants to REJECT, crap
 986
 987     (new, byhand) = check_status(files)
 988     if not new and not byhand:
 989         do_accept()
 990
 991 ################################################################################
 992
 993 def comment_reject(changes_file, comments):
 994     Upload.pkg.changes_file = changes_file
 995     Upload.init_vars()
 996     Upload.update_vars()
 997     Upload.update_subst()
 998
 999     if not recheck():
1000         pass # dak has its own reasons to reject as well, which is fine
1001
1002     reject(comments)
1003     print "REJECT\n" + reject_message,
1004     if not Options["No-Action"]:
1005         Upload.do_reject(0, reject_message)
1006         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1007
1008 ################################################################################
1009
1010 def main():
1011     changes_files = init()
1012     if len(changes_files) > 50:
1013         sys.stderr.write("Sorting changes...\n")
1014     changes_files = sort_changes(changes_files)
1015
1016     # Kill me now? **FIXME**
1017     Cnf["Dinstall::Options::No-Mail"] = ""
1018     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1019     if Cnf.has_key("Dinstall::Bcc"):
1020         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1021     else:
1022         Upload.Subst["__BCC__"] = bcc
1023
1024     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1025     if commentsdir:
1026         if changes_files != []:
1027             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1028             sys.exit(1)
1029         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1030         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1031     else:
1032         for changes_file in changes_files:
1033             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1034             if not changes_file:
1035                 continue
1036             print "\n" + changes_file
1037             do_pkg (changes_file)
1038
1039     end()
1040
1041 ################################################################################
1042
1043 if __name__ == '__main__':
1044     main()