From: Joerg Jaspert Date: Thu, 24 Mar 2011 10:33:44 +0000 (+0100) Subject: Merge remote-tracking branch 'ansgar/package-set' into merge X-Git-Tag: debian-r/squeeze~249 X-Git-Url: https://git.donarmstrong.com/?a=commitdiff_plain;h=40970b8f6af57da3b1a65237e7bc6af7ba77ec73;hp=3342185a1c804f3fe03822c21bad57ce9db009ed;p=dak.git Merge remote-tracking branch 'ansgar/package-set' into merge * ansgar/package-set: process-new: allow overrides to work for Package-Set Make use of Package-Set in determine_new queue.Upload: separate code to load .dsc from check_dsc utils.py: Add parser for Package-Set field in .dsc process-upload: Dinstall::LintianTags may not be set Signed-off-by: Joerg Jaspert --- diff --git a/dak/process_new.py b/dak/process_new.py index 740ce6e8..dfdb2e97 100755 --- a/dak/process_new.py +++ b/dak/process_new.py @@ -424,11 +424,18 @@ def do_new(upload, session): # Make a copy of distribution we can happily trample on changes["suite"] = copy.copy(changes["distribution"]) + # Try to get an included dsc + dsc = None + (status, _) = upload.load_dsc() + if status: + dsc = upload.pkg.dsc + # The main NEW processing loop done = 0 + new = {} while not done: # Find out what's new - new, byhand = determine_new(upload.pkg.changes_file, changes, files, session=session) + new, byhand = determine_new(upload.pkg.changes_file, changes, files, dsc=dsc, session=session, new=new) if not new: break @@ -667,6 +674,12 @@ def do_pkg(changes_full_path, session): u.logger = Logger origchanges = os.path.abspath(u.pkg.changes_file) + # Try to get an included dsc + dsc = None + (status, _) = u.load_dsc() + if status: + dsc = u.pkg.dsc + cnf = Config() bcc = "X-DAK: dak process-new" if cnf.has_key("Dinstall::Bcc"): @@ -691,7 +704,7 @@ def do_pkg(changes_full_path, session): if not recheck(u, session): return - new, byhand = determine_new(u.pkg.changes_file, u.pkg.changes, files, session=session) + new, byhand = determine_new(u.pkg.changes_file, u.pkg.changes, files, dsc=dsc, session=session) if byhand: do_byhand(u, session) elif new: diff --git a/dak/show_new.py b/dak/show_new.py index e95096e8..8405aeef 100755 --- a/dak/show_new.py +++ b/dak/show_new.py @@ -180,7 +180,7 @@ def do_pkg(changes_file): u.check_source_against_db(deb_filename, session) u.pkg.changes["suite"] = u.pkg.changes["distribution"] - new, byhand = determine_new(u.pkg.changes_file, u.pkg.changes, files, 0, session) + new, byhand = determine_new(u.pkg.changes_file, u.pkg.changes, files, 0, dsc=u.pkg.dsc, session=session) outfile = open(os.path.join(cnf["Show-New::HTMLPath"],htmlname),"w") diff --git a/daklib/queue.py b/daklib/queue.py index 52483cca..84ac6c98 100755 --- a/daklib/queue.py +++ b/daklib/queue.py @@ -51,7 +51,7 @@ from holding import Holding from urgencylog import UrgencyLog from dbconn import * from summarystats import SummaryStats -from utils import parse_changes, check_dsc_files +from utils import parse_changes, check_dsc_files, build_package_set from textutils import fix_maintainer from lintian import parse_lintian_output, generate_reject_messages from contents import UnpackedSource @@ -102,7 +102,7 @@ def get_type(f, session): # Determine what parts in a .changes are NEW -def determine_new(filename, changes, files, warn=1, session = None): +def determine_new(filename, changes, files, warn=1, session = None, dsc = None, new = {}): """ Determine what parts in a C{changes} file are NEW. @@ -118,19 +118,30 @@ def determine_new(filename, changes, files, warn=1, session = None): @type warn: bool @param warn: Warn if overrides are added for (old)stable + @type dsc: Upload.Pkg.dsc dict + @param dsc: (optional); Dsc dictionary + + @type new: dict + @param new: new packages as returned by a previous call to this function, but override information may have changed + @rtype: dict @return: dictionary of NEW components. """ # TODO: This should all use the database instead of parsing the changes # file again - new = {} byhand = {} dbchg = get_dbchange(filename, session) if dbchg is None: print "Warning: cannot find changes file in database; won't check byhand" + # Try to get the Package-Set field from an included .dsc file (if possible). + if dsc: + for package, entry in build_package_set(dsc, session).items(): + if not new.has_key(package): + new[package] = entry + # Build up a list of potentially new things for name, f in files.items(): # Keep a record of byhand elements @@ -1105,40 +1116,76 @@ class Upload(object): self.rejects.append("source only uploads are not supported.") ########################################################################### - def check_dsc(self, action=True, session=None): - """Returns bool indicating whether or not the source changes are valid""" - # Ensure there is source to check - if not self.pkg.changes["architecture"].has_key("source"): - return True - # Find the .dsc + def __dsc_filename(self): + """ + Returns: (Status, Dsc_Filename) + where + Status: Boolean; True when there was no error, False otherwise + Dsc_Filename: String; name of the dsc file if Status is True, reason for the error otherwise + """ dsc_filename = None - for f, entry in self.pkg.files.items(): - if entry["type"] == "dsc": + + # find the dsc + for name, entry in self.pkg.files.items(): + if entry.has_key("type") and entry["type"] == "dsc": if dsc_filename: - self.rejects.append("can not process a .changes file with multiple .dsc's.") - return False + return False, "cannot process a .changes file with multiple .dsc's." else: - dsc_filename = f + dsc_filename = name - # If there isn't one, we have nothing to do. (We have reject()ed the upload already) if not dsc_filename: - self.rejects.append("source uploads must contain a dsc file") - return False + return False, "source uploads must contain a dsc file" + + return True, dsc_filename + + def load_dsc(self, action=True, signing_rules=1): + """ + Find and load the dsc from self.pkg.files into self.dsc + + Returns: (Status, Reason) + where + Status: Boolean; True when there was no error, False otherwise + Reason: String; When Status is False this describes the error + """ + + # find the dsc + (status, dsc_filename) = self.__dsc_filename() + if not status: + # If status is false, dsc_filename has the reason + return False, dsc_filename - # Parse the .dsc file try: - self.pkg.dsc.update(utils.parse_changes(dsc_filename, signing_rules=1, dsc_file=1)) + self.pkg.dsc.update(utils.parse_changes(dsc_filename, signing_rules=signing_rules, dsc_file=1)) except CantOpenError: - # if not -n copy_to_holding() will have done this for us... if not action: - self.rejects.append("%s: can't read file." % (dsc_filename)) + return False, "%s: can't read file." % (dsc_filename) except ParseChangesError, line: - self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line)) + return False, "%s: parse error, can't grok: %s." % (dsc_filename, line) except InvalidDscError, line: - self.rejects.append("%s: syntax error on line %s." % (dsc_filename, line)) + return False, "%s: syntax error on line %s." % (dsc_filename, line) except ChangesUnicodeError: - self.rejects.append("%s: dsc file not proper utf-8." % (dsc_filename)) + return False, "%s: dsc file not proper utf-8." % (dsc_filename) + + return True, None + + ########################################################################### + + def check_dsc(self, action=True, session=None): + """Returns bool indicating whether or not the source changes are valid""" + # Ensure there is source to check + if not self.pkg.changes["architecture"].has_key("source"): + return True + + (status, reason) = self.load_dsc(action=action) + if not status: + self.rejects.append(reason) + return False + (status, dsc_filename) = self.__dsc_filename() + if not status: + # If status is false, dsc_filename has the reason + self.rejects.append(dsc_filename) + return False # Build up the file list of files mentioned by the .dsc try: @@ -1495,7 +1542,7 @@ class Upload(object): # If we do not have a tagfile, don't do anything tagfile = cnf.get("Dinstall::LintianTags") - if tagfile is None: + if not tagfile: return # Parse the yaml file diff --git a/daklib/utils.py b/daklib/utils.py index b5e090da..0b2d8c87 100755 --- a/daklib/utils.py +++ b/daklib/utils.py @@ -39,7 +39,7 @@ import re import email as modemail import subprocess -from dbconn import DBConn, get_architecture, get_component, get_suite +from dbconn import DBConn, get_architecture, get_component, get_suite, get_override_type from dak_exceptions import * from textutils import fix_maintainer from regexes import re_html_escaping, html_escaping, re_single_line_field, \ @@ -588,6 +588,46 @@ def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"): ################################################################################ +# see http://bugs.debian.org/619131 +def build_package_set(dsc, session = None): + if not dsc.has_key("package-set"): + return {} + + packages = {} + + for line in dsc["package-set"].split("\n"): + if not line: + break + + (name, section, priority) = line.split() + (section, component) = extract_component_from_section(section) + + package_type = "deb" + if name.find(":") != -1: + (package_type, name) = name.split(":", 1) + if package_type == "src": + package_type = "dsc" + + # Validate type if we have a session + if session and get_override_type(package_type, session) is None: + # Maybe just warn and ignore? exit(1) might be a bit hard... + utils.fubar("invalid type (%s) in Package-Set." % (package_type)) + + if section == "": + section = "-" + if priority == "": + priority = "-" + + if package_type == "dsc": + priority = "source" + + if not packages.has_key(name) or packages[name]["type"] == "dsc": + packages[name] = dict(priority=priority, section=section, type=package_type, component=component, files=[]) + + return packages + +################################################################################ + def send_mail (message, filename=""): """sendmail wrapper, takes _either_ a message string or a file as arguments"""