2 # vim:set et ts=4 sw=4:
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
8 @license: GNU General Public License version 2 or later
11 # This program is free software; you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation; either version 2 of the License, or
14 # (at your option) any later version.
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
21 # You should have received a copy of the GNU General Public License
22 # along with this program; if not, write to the Free Software
23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
39 import email as modemail
42 from dbconn import DBConn, get_architecture, get_component, get_suite
43 from dak_exceptions import *
44 from textutils import fix_maintainer
45 from regexes import re_html_escaping, html_escaping, re_single_line_field, \
46 re_multi_line_field, re_srchasver, re_taint_free, \
47 re_gpg_uid, re_re_mark, re_whitespace_comment, re_issource
49 from formats import parse_format, validate_changes_format
50 from srcformats import get_format_from_string
51 from collections import defaultdict
53 ################################################################################
55 default_config = "/etc/dak/dak.conf" #: default dak config, defines host properties
56 default_apt_config = "/etc/dak/apt.conf" #: default apt config, not normally used
58 alias_cache = None #: Cache for email alias checks
59 key_uid_email_cache = {} #: Cache for email addresses from gpg key uids
61 # (hashname, function, earliest_changes_version)
62 known_hashes = [("sha1", apt_pkg.sha1sum, (1, 8)),
63 ("sha256", apt_pkg.sha256sum, (1, 8))] #: hashes we accept for entries in .changes/.dsc
65 # Monkeypatch commands.getstatusoutput as it may not return the correct exit
66 # code in lenny's Python. This also affects commands.getoutput and
68 def dak_getstatusoutput(cmd):
69 pipe = subprocess.Popen(cmd, shell=True, universal_newlines=True,
70 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
72 output = "".join(pipe.stdout.readlines())
74 if output[-1:] == '\n':
82 commands.getstatusoutput = dak_getstatusoutput
84 ################################################################################
87 """ Escape html chars """
88 return re_html_escaping.sub(lambda x: html_escaping.get(x.group(0)), s)
90 ################################################################################
92 def open_file(filename, mode='r'):
94 Open C{file}, return fileobject.
96 @type filename: string
97 @param filename: path/filename to open
100 @param mode: open mode
103 @return: open fileobject
105 @raise CantOpenError: If IOError is raised by open, reraise it as CantOpenError.
109 f = open(filename, mode)
111 raise CantOpenError, filename
114 ################################################################################
116 def our_raw_input(prompt=""):
118 sys.stdout.write(prompt)
124 sys.stderr.write("\nUser interrupt (^D).\n")
127 ################################################################################
129 def extract_component_from_section(section):
132 if section.find('/') != -1:
133 component = section.split('/')[0]
135 # Expand default component
137 if Cnf.has_key("Component::%s" % section):
142 return (section, component)
144 ################################################################################
146 def parse_deb822(contents, signing_rules=0):
150 # Split the lines in the input, keeping the linebreaks.
151 lines = contents.splitlines(True)
154 raise ParseChangesError, "[Empty changes file]"
156 # Reindex by line number so we can easily verify the format of
162 indexed_lines[index] = line[:-1]
166 num_of_lines = len(indexed_lines.keys())
169 while index < num_of_lines:
171 line = indexed_lines[index]
173 if signing_rules == 1:
175 if index > num_of_lines:
176 raise InvalidDscError, index
177 line = indexed_lines[index]
178 if not line.startswith("-----BEGIN PGP SIGNATURE"):
179 raise InvalidDscError, index
184 if line.startswith("-----BEGIN PGP SIGNATURE"):
186 if line.startswith("-----BEGIN PGP SIGNED MESSAGE"):
188 if signing_rules == 1:
189 while index < num_of_lines and line != "":
191 line = indexed_lines[index]
193 # If we're not inside the signed data, don't process anything
194 if signing_rules >= 0 and not inside_signature:
196 slf = re_single_line_field.match(line)
198 field = slf.groups()[0].lower()
199 changes[field] = slf.groups()[1]
203 changes[field] += '\n'
205 mlf = re_multi_line_field.match(line)
208 raise ParseChangesError, "'%s'\n [Multi-line field continuing on from nothing?]" % (line)
209 if first == 1 and changes[field] != "":
210 changes[field] += '\n'
212 changes[field] += mlf.groups()[0] + '\n'
216 if signing_rules == 1 and inside_signature:
217 raise InvalidDscError, index
219 changes["filecontents"] = "".join(lines)
221 if changes.has_key("source"):
222 # Strip the source version in brackets from the source field,
223 # put it in the "source-version" field instead.
224 srcver = re_srchasver.search(changes["source"])
226 changes["source"] = srcver.group(1)
227 changes["source-version"] = srcver.group(2)
230 raise ParseChangesError, error
234 ################################################################################
236 def parse_changes(filename, signing_rules=0):
238 Parses a changes file and returns a dictionary where each field is a
239 key. The mandatory first argument is the filename of the .changes
242 signing_rules is an optional argument:
244 - If signing_rules == -1, no signature is required.
245 - If signing_rules == 0 (the default), a signature is required.
246 - If signing_rules == 1, it turns on the same strict format checking
249 The rules for (signing_rules == 1)-mode are:
251 - The PGP header consists of "-----BEGIN PGP SIGNED MESSAGE-----"
252 followed by any PGP header data and must end with a blank line.
254 - The data section must end with a blank line and must be followed by
255 "-----BEGIN PGP SIGNATURE-----".
258 changes_in = open_file(filename)
259 content = changes_in.read()
262 unicode(content, 'utf-8')
264 raise ChangesUnicodeError, "Changes file not proper utf-8"
265 return parse_deb822(content, signing_rules)
267 ################################################################################
269 def hash_key(hashname):
270 return '%ssum' % hashname
272 ################################################################################
274 def create_hash(where, files, hashname, hashfunc):
276 create_hash extends the passed files dict with the given hash by
277 iterating over all files on disk and passing them to the hashing
282 for f in files.keys():
284 file_handle = open_file(f)
285 except CantOpenError:
286 rejmsg.append("Could not open file %s for checksumming" % (f))
289 files[f][hash_key(hashname)] = hashfunc(file_handle)
294 ################################################################################
296 def check_hash(where, files, hashname, hashfunc):
298 check_hash checks the given hash in the files dict against the actual
299 files on disk. The hash values need to be present consistently in
300 all file entries. It does not modify its input in any way.
304 for f in files.keys():
308 file_handle = open_file(f)
310 # Check for the hash entry, to not trigger a KeyError.
311 if not files[f].has_key(hash_key(hashname)):
312 rejmsg.append("%s: misses %s checksum in %s" % (f, hashname,
316 # Actually check the hash for correctness.
317 if hashfunc(file_handle) != files[f][hash_key(hashname)]:
318 rejmsg.append("%s: %s check failed in %s" % (f, hashname,
320 except CantOpenError:
321 # TODO: This happens when the file is in the pool.
322 # warn("Cannot open file %s" % f)
329 ################################################################################
331 def check_size(where, files):
333 check_size checks the file sizes in the passed files dict against the
338 for f in files.keys():
343 # TODO: This happens when the file is in the pool.
347 actual_size = entry[stat.ST_SIZE]
348 size = int(files[f]["size"])
349 if size != actual_size:
350 rejmsg.append("%s: actual file size (%s) does not match size (%s) in %s"
351 % (f, actual_size, size, where))
354 ################################################################################
356 def check_dsc_files(dsc_filename, dsc=None, dsc_files=None):
358 Verify that the files listed in the Files field of the .dsc are
359 those expected given the announced Format.
361 @type dsc_filename: string
362 @param dsc_filename: path of .dsc file
365 @param dsc: the content of the .dsc parsed by C{parse_changes()}
367 @type dsc_files: dict
368 @param dsc_files: the file list returned by C{build_file_list()}
371 @return: all errors detected
375 # Parse the file if needed
377 dsc = parse_changes(dsc_filename, signing_rules=1);
379 if dsc_files is None:
380 dsc_files = build_file_list(dsc, is_a_dsc=1)
382 # Ensure .dsc lists proper set of source files according to the format
384 has = defaultdict(lambda: 0)
387 (r'orig.tar.gz', ('orig_tar_gz', 'orig_tar')),
388 (r'diff.gz', ('debian_diff',)),
389 (r'tar.gz', ('native_tar_gz', 'native_tar')),
390 (r'debian\.tar\.(gz|bz2)', ('debian_tar',)),
391 (r'orig\.tar\.(gz|bz2)', ('orig_tar',)),
392 (r'tar\.(gz|bz2)', ('native_tar',)),
393 (r'orig-.+\.tar\.(gz|bz2)', ('more_orig_tar',)),
396 for f in dsc_files.keys():
397 m = re_issource.match(f)
399 rejmsg.append("%s: %s in Files field not recognised as source."
403 # Populate 'has' dictionary by resolving keys in lookup table
405 for regex, keys in ftype_lookup:
406 if re.match(regex, m.group(3)):
412 # File does not match anything in lookup table; reject
414 reject("%s: unexpected source file '%s'" % (dsc_filename, f))
416 # Check for multiple files
417 for file_type in ('orig_tar', 'native_tar', 'debian_tar', 'debian_diff'):
418 if has[file_type] > 1:
419 rejmsg.append("%s: lists multiple %s" % (dsc_filename, file_type))
421 # Source format specific tests
423 format = get_format_from_string(dsc['format'])
425 '%s: %s' % (dsc_filename, x) for x in format.reject_msgs(has)
428 except UnknownFormatError:
429 # Not an error here for now
434 ################################################################################
436 def check_hash_fields(what, manifest):
438 check_hash_fields ensures that there are no checksum fields in the
439 given dict that we do not know about.
443 hashes = map(lambda x: x[0], known_hashes)
444 for field in manifest:
445 if field.startswith("checksums-"):
446 hashname = field.split("-",1)[1]
447 if hashname not in hashes:
448 rejmsg.append("Unsupported checksum field for %s "\
449 "in %s" % (hashname, what))
452 ################################################################################
454 def _ensure_changes_hash(changes, format, version, files, hashname, hashfunc):
455 if format >= version:
456 # The version should contain the specified hash.
459 # Import hashes from the changes
460 rejmsg = parse_checksums(".changes", files, changes, hashname)
464 # We need to calculate the hash because it can't possibly
467 return func(".changes", files, hashname, hashfunc)
469 # We could add the orig which might be in the pool to the files dict to
470 # access the checksums easily.
472 def _ensure_dsc_hash(dsc, dsc_files, hashname, hashfunc):
474 ensure_dsc_hashes' task is to ensure that each and every *present* hash
475 in the dsc is correct, i.e. identical to the changes file and if necessary
476 the pool. The latter task is delegated to check_hash.
480 if not dsc.has_key('Checksums-%s' % (hashname,)):
482 # Import hashes from the dsc
483 parse_checksums(".dsc", dsc_files, dsc, hashname)
485 rejmsg.extend(check_hash(".dsc", dsc_files, hashname, hashfunc))
488 ################################################################################
490 def parse_checksums(where, files, manifest, hashname):
492 field = 'checksums-%s' % hashname
493 if not field in manifest:
495 for line in manifest[field].split('\n'):
498 clist = line.strip().split(' ')
500 checksum, size, checkfile = clist
502 rejmsg.append("Cannot parse checksum line [%s]" % (line))
504 if not files.has_key(checkfile):
505 # TODO: check for the file's entry in the original files dict, not
506 # the one modified by (auto)byhand and other weird stuff
507 # rejmsg.append("%s: not present in files but in checksums-%s in %s" %
508 # (file, hashname, where))
510 if not files[checkfile]["size"] == size:
511 rejmsg.append("%s: size differs for files and checksums-%s entry "\
512 "in %s" % (checkfile, hashname, where))
514 files[checkfile][hash_key(hashname)] = checksum
515 for f in files.keys():
516 if not files[f].has_key(hash_key(hashname)):
517 rejmsg.append("%s: no entry in checksums-%s in %s" % (checkfile,
521 ################################################################################
523 # Dropped support for 1.4 and ``buggy dchanges 3.4'' (?!) compared to di.pl
525 def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"):
528 # Make sure we have a Files: field to parse...
529 if not changes.has_key(field):
530 raise NoFilesFieldError
532 # Validate .changes Format: field
534 validate_changes_format(parse_format(changes['format']), field)
536 includes_section = (not is_a_dsc) and field == "files"
538 # Parse each entry/line:
539 for i in changes[field].split('\n'):
543 section = priority = ""
546 (md5, size, section, priority, name) = s
548 (md5, size, name) = s
550 raise ParseChangesError, i
557 (section, component) = extract_component_from_section(section)
559 files[name] = Dict(size=size, section=section,
560 priority=priority, component=component)
561 files[name][hashname] = md5
565 ################################################################################
567 def send_mail (message, filename=""):
568 """sendmail wrapper, takes _either_ a message string or a file as arguments"""
570 # If we've been passed a string dump it into a temporary file
572 (fd, filename) = tempfile.mkstemp()
573 os.write (fd, message)
576 if Cnf.has_key("Dinstall::MailWhiteList") and \
577 Cnf["Dinstall::MailWhiteList"] != "":
578 message_in = open_file(filename)
579 message_raw = modemail.message_from_file(message_in)
583 whitelist_in = open_file(Cnf["Dinstall::MailWhiteList"])
585 for line in whitelist_in:
586 if not re_whitespace_comment.match(line):
587 if re_re_mark.match(line):
588 whitelist.append(re.compile(re_re_mark.sub("", line.strip(), 1)))
590 whitelist.append(re.compile(re.escape(line.strip())))
595 fields = ["To", "Bcc", "Cc"]
598 value = message_raw.get(field, None)
601 for item in value.split(","):
602 (rfc822_maint, rfc2047_maint, name, email) = fix_maintainer(item.strip())
608 if not mail_whitelisted:
609 print "Skipping %s since it's not in %s" % (item, Cnf["Dinstall::MailWhiteList"])
613 # Doesn't have any mail in whitelist so remove the header
615 del message_raw[field]
617 message_raw.replace_header(field, ', '.join(match))
619 # Change message fields in order if we don't have a To header
620 if not message_raw.has_key("To"):
623 if message_raw.has_key(field):
624 message_raw[fields[-1]] = message_raw[field]
625 del message_raw[field]
628 # Clean up any temporary files
629 # and return, as we removed all recipients.
631 os.unlink (filename);
634 fd = os.open(filename, os.O_RDWR|os.O_EXCL, 0700);
635 os.write (fd, message_raw.as_string(True));
639 (result, output) = commands.getstatusoutput("%s < %s" % (Cnf["Dinstall::SendmailCommand"], filename))
641 raise SendmailFailedError, output
643 # Clean up any temporary files
647 ################################################################################
649 def poolify (source, component):
652 if source[:3] == "lib":
653 return component + source[:4] + '/' + source + '/'
655 return component + source[:1] + '/' + source + '/'
657 ################################################################################
659 def move (src, dest, overwrite = 0, perms = 0664):
660 if os.path.exists(dest) and os.path.isdir(dest):
663 dest_dir = os.path.dirname(dest)
664 if not os.path.exists(dest_dir):
665 umask = os.umask(00000)
666 os.makedirs(dest_dir, 02775)
668 #print "Moving %s to %s..." % (src, dest)
669 if os.path.exists(dest) and os.path.isdir(dest):
670 dest += '/' + os.path.basename(src)
671 # Don't overwrite unless forced to
672 if os.path.exists(dest):
674 fubar("Can't move %s to %s - file already exists." % (src, dest))
676 if not os.access(dest, os.W_OK):
677 fubar("Can't move %s to %s - can't write to existing file." % (src, dest))
678 shutil.copy2(src, dest)
679 os.chmod(dest, perms)
682 def copy (src, dest, overwrite = 0, perms = 0664):
683 if os.path.exists(dest) and os.path.isdir(dest):
686 dest_dir = os.path.dirname(dest)
687 if not os.path.exists(dest_dir):
688 umask = os.umask(00000)
689 os.makedirs(dest_dir, 02775)
691 #print "Copying %s to %s..." % (src, dest)
692 if os.path.exists(dest) and os.path.isdir(dest):
693 dest += '/' + os.path.basename(src)
694 # Don't overwrite unless forced to
695 if os.path.exists(dest):
697 raise FileExistsError
699 if not os.access(dest, os.W_OK):
700 raise CantOverwriteError
701 shutil.copy2(src, dest)
702 os.chmod(dest, perms)
704 ################################################################################
707 res = socket.gethostbyaddr(socket.gethostname())
708 database_hostname = Cnf.get("Config::" + res[0] + "::DatabaseHostname")
709 if database_hostname:
710 return database_hostname
714 def which_conf_file ():
715 if os.getenv("DAK_CONFIG"):
716 print(os.getenv("DAK_CONFIG"))
717 return os.getenv("DAK_CONFIG")
719 res = socket.gethostbyaddr(socket.gethostname())
720 # In case we allow local config files per user, try if one exists
721 if Cnf.FindB("Config::" + res[0] + "::AllowLocalConfig"):
722 homedir = os.getenv("HOME")
723 confpath = os.path.join(homedir, "/etc/dak.conf")
724 if os.path.exists(confpath):
725 apt_pkg.ReadConfigFileISC(Cnf,default_config)
727 # We are still in here, so there is no local config file or we do
728 # not allow local files. Do the normal stuff.
729 if Cnf.get("Config::" + res[0] + "::DakConfig"):
730 return Cnf["Config::" + res[0] + "::DakConfig"]
732 return default_config
734 def which_apt_conf_file ():
735 res = socket.gethostbyaddr(socket.gethostname())
736 # In case we allow local config files per user, try if one exists
737 if Cnf.FindB("Config::" + res[0] + "::AllowLocalConfig"):
738 homedir = os.getenv("HOME")
739 confpath = os.path.join(homedir, "/etc/dak.conf")
740 if os.path.exists(confpath):
741 apt_pkg.ReadConfigFileISC(Cnf,default_config)
743 if Cnf.get("Config::" + res[0] + "::AptConfig"):
744 return Cnf["Config::" + res[0] + "::AptConfig"]
746 return default_apt_config
748 def which_alias_file():
749 hostname = socket.gethostbyaddr(socket.gethostname())[0]
750 aliasfn = '/var/lib/misc/'+hostname+'/forward-alias'
751 if os.path.exists(aliasfn):
756 ################################################################################
758 def TemplateSubst(subst_map, filename):
759 """ Perform a substition of template """
760 templatefile = open_file(filename)
761 template = templatefile.read()
762 for k, v in subst_map.iteritems():
763 template = template.replace(k, str(v))
767 ################################################################################
769 def fubar(msg, exit_code=1):
770 sys.stderr.write("E: %s\n" % (msg))
774 sys.stderr.write("W: %s\n" % (msg))
776 ################################################################################
778 # Returns the user name with a laughable attempt at rfc822 conformancy
779 # (read: removing stray periods).
781 return pwd.getpwuid(os.getuid())[4].split(',')[0].replace('.', '')
784 return pwd.getpwuid(os.getuid())[0]
786 ################################################################################
796 return ("%d%s" % (c, t))
798 ################################################################################
800 def cc_fix_changes (changes):
801 o = changes.get("architecture", "")
803 del changes["architecture"]
804 changes["architecture"] = {}
806 changes["architecture"][j] = 1
808 def changes_compare (a, b):
809 """ Sort by source name, source version, 'have source', and then by filename """
811 a_changes = parse_changes(a)
816 b_changes = parse_changes(b)
820 cc_fix_changes (a_changes)
821 cc_fix_changes (b_changes)
823 # Sort by source name
824 a_source = a_changes.get("source")
825 b_source = b_changes.get("source")
826 q = cmp (a_source, b_source)
830 # Sort by source version
831 a_version = a_changes.get("version", "0")
832 b_version = b_changes.get("version", "0")
833 q = apt_pkg.VersionCompare(a_version, b_version)
837 # Sort by 'have source'
838 a_has_source = a_changes["architecture"].get("source")
839 b_has_source = b_changes["architecture"].get("source")
840 if a_has_source and not b_has_source:
842 elif b_has_source and not a_has_source:
845 # Fall back to sort by filename
848 ################################################################################
850 def find_next_free (dest, too_many=100):
853 while os.path.exists(dest) and extra < too_many:
854 dest = orig_dest + '.' + repr(extra)
856 if extra >= too_many:
857 raise NoFreeFilenameError
860 ################################################################################
862 def result_join (original, sep = '\t'):
864 for i in xrange(len(original)):
865 if original[i] == None:
866 resultlist.append("")
868 resultlist.append(original[i])
869 return sep.join(resultlist)
871 ################################################################################
873 def prefix_multi_line_string(str, prefix, include_blank_lines=0):
875 for line in str.split('\n'):
877 if line or include_blank_lines:
878 out += "%s%s\n" % (prefix, line)
879 # Strip trailing new line
884 ################################################################################
886 def validate_changes_file_arg(filename, require_changes=1):
888 'filename' is either a .changes or .dak file. If 'filename' is a
889 .dak file, it's changed to be the corresponding .changes file. The
890 function then checks if the .changes file a) exists and b) is
891 readable and returns the .changes filename if so. If there's a
892 problem, the next action depends on the option 'require_changes'
895 - If 'require_changes' == -1, errors are ignored and the .changes
896 filename is returned.
897 - If 'require_changes' == 0, a warning is given and 'None' is returned.
898 - If 'require_changes' == 1, a fatal error is raised.
903 orig_filename = filename
904 if filename.endswith(".dak"):
905 filename = filename[:-4]+".changes"
907 if not filename.endswith(".changes"):
908 error = "invalid file type; not a changes file"
910 if not os.access(filename,os.R_OK):
911 if os.path.exists(filename):
912 error = "permission denied"
914 error = "file not found"
917 if require_changes == 1:
918 fubar("%s: %s." % (orig_filename, error))
919 elif require_changes == 0:
920 warn("Skipping %s - %s" % (orig_filename, error))
922 else: # We only care about the .dak file
927 ################################################################################
930 return (arch != "source" and arch != "all")
932 ################################################################################
934 def join_with_commas_and(list):
935 if len(list) == 0: return "nothing"
936 if len(list) == 1: return list[0]
937 return ", ".join(list[:-1]) + " and " + list[-1]
939 ################################################################################
944 (pkg, version, constraint) = atom
946 pp_dep = "%s (%s %s)" % (pkg, constraint, version)
949 pp_deps.append(pp_dep)
950 return " |".join(pp_deps)
952 ################################################################################
957 ################################################################################
959 def parse_args(Options):
960 """ Handle -a, -c and -s arguments; returns them as SQL constraints """
961 # XXX: This should go away and everything which calls it be converted
962 # to use SQLA properly. For now, we'll just fix it not to use
963 # the old Pg interface though
964 session = DBConn().session()
968 for suitename in split_args(Options["Suite"]):
969 suite = get_suite(suitename, session=session)
970 if suite.suite_id is None:
971 warn("suite '%s' not recognised." % (suite.suite_name))
973 suite_ids_list.append(suite.suite_id)
975 con_suites = "AND su.id IN (%s)" % ", ".join([ str(i) for i in suite_ids_list ])
977 fubar("No valid suite given.")
982 if Options["Component"]:
983 component_ids_list = []
984 for componentname in split_args(Options["Component"]):
985 component = get_component(componentname, session=session)
986 if component is None:
987 warn("component '%s' not recognised." % (componentname))
989 component_ids_list.append(component.component_id)
990 if component_ids_list:
991 con_components = "AND c.id IN (%s)" % ", ".join([ str(i) for i in component_ids_list ])
993 fubar("No valid component given.")
997 # Process architecture
998 con_architectures = ""
1000 if Options["Architecture"]:
1002 for archname in split_args(Options["Architecture"]):
1003 if archname == "source":
1006 arch = get_architecture(archname, session=session)
1008 warn("architecture '%s' not recognised." % (archname))
1010 arch_ids_list.append(arch.arch_id)
1012 con_architectures = "AND a.id IN (%s)" % ", ".join([ str(i) for i in arch_ids_list ])
1014 if not check_source:
1015 fubar("No valid architecture given.")
1019 return (con_suites, con_architectures, con_components, check_source)
1021 ################################################################################
1023 # Inspired(tm) by Bryn Keller's print_exc_plus (See
1024 # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52215)
1027 tb = sys.exc_info()[2]
1034 frame = frame.f_back
1036 traceback.print_exc()
1038 print "\nFrame %s in %s at line %s" % (frame.f_code.co_name,
1039 frame.f_code.co_filename,
1041 for key, value in frame.f_locals.items():
1042 print "\t%20s = " % key,
1046 print "<unable to print>"
1048 ################################################################################
1050 def try_with_debug(function):
1058 ################################################################################
1060 def arch_compare_sw (a, b):
1062 Function for use in sorting lists of architectures.
1064 Sorts normally except that 'source' dominates all others.
1067 if a == "source" and b == "source":
1076 ################################################################################
1078 def split_args (s, dwim=1):
1080 Split command line arguments which can be separated by either commas
1081 or whitespace. If dwim is set, it will complain about string ending
1082 in comma since this usually means someone did 'dak ls -a i386, m68k
1083 foo' or something and the inevitable confusion resulting from 'm68k'
1084 being treated as an argument is undesirable.
1087 if s.find(",") == -1:
1090 if s[-1:] == "," and dwim:
1091 fubar("split_args: found trailing comma, spurious space maybe?")
1094 ################################################################################
1096 def Dict(**dict): return dict
1098 ########################################
1100 def gpgv_get_status_output(cmd, status_read, status_write):
1102 Our very own version of commands.getouputstatus(), hacked to support
1106 cmd = ['/bin/sh', '-c', cmd]
1107 p2cread, p2cwrite = os.pipe()
1108 c2pread, c2pwrite = os.pipe()
1109 errout, errin = os.pipe()
1119 for i in range(3, 256):
1120 if i != status_write:
1126 os.execvp(cmd[0], cmd)
1132 os.dup2(c2pread, c2pwrite)
1133 os.dup2(errout, errin)
1135 output = status = ""
1137 i, o, e = select.select([c2pwrite, errin, status_read], [], [])
1140 r = os.read(fd, 8196)
1142 more_data.append(fd)
1143 if fd == c2pwrite or fd == errin:
1145 elif fd == status_read:
1148 fubar("Unexpected file descriptor [%s] returned from select\n" % (fd))
1150 pid, exit_status = os.waitpid(pid, 0)
1152 os.close(status_write)
1153 os.close(status_read)
1163 return output, status, exit_status
1165 ################################################################################
1167 def process_gpgv_output(status):
1168 # Process the status-fd output
1171 for line in status.split('\n'):
1175 split = line.split()
1177 internal_error += "gpgv status line is malformed (< 2 atoms) ['%s'].\n" % (line)
1179 (gnupg, keyword) = split[:2]
1180 if gnupg != "[GNUPG:]":
1181 internal_error += "gpgv status line is malformed (incorrect prefix '%s').\n" % (gnupg)
1184 if keywords.has_key(keyword) and keyword not in [ "NODATA", "SIGEXPIRED", "KEYEXPIRED" ]:
1185 internal_error += "found duplicate status token ('%s').\n" % (keyword)
1188 keywords[keyword] = args
1190 return (keywords, internal_error)
1192 ################################################################################
1194 def retrieve_key (filename, keyserver=None, keyring=None):
1196 Retrieve the key that signed 'filename' from 'keyserver' and
1197 add it to 'keyring'. Returns nothing on success, or an error message
1201 # Defaults for keyserver and keyring
1203 keyserver = Cnf["Dinstall::KeyServer"]
1205 keyring = Cnf.ValueList("Dinstall::GPGKeyring")[0]
1207 # Ensure the filename contains no shell meta-characters or other badness
1208 if not re_taint_free.match(filename):
1209 return "%s: tainted filename" % (filename)
1211 # Invoke gpgv on the file
1212 status_read, status_write = os.pipe()
1213 cmd = "gpgv --status-fd %s --keyring /dev/null %s" % (status_write, filename)
1214 (_, status, _) = gpgv_get_status_output(cmd, status_read, status_write)
1216 # Process the status-fd output
1217 (keywords, internal_error) = process_gpgv_output(status)
1219 return internal_error
1221 if not keywords.has_key("NO_PUBKEY"):
1222 return "didn't find expected NO_PUBKEY in gpgv status-fd output"
1224 fingerprint = keywords["NO_PUBKEY"][0]
1225 # XXX - gpg sucks. You can't use --secret-keyring=/dev/null as
1226 # it'll try to create a lockfile in /dev. A better solution might
1227 # be a tempfile or something.
1228 cmd = "gpg --no-default-keyring --secret-keyring=%s --no-options" \
1229 % (Cnf["Dinstall::SigningKeyring"])
1230 cmd += " --keyring %s --keyserver %s --recv-key %s" \
1231 % (keyring, keyserver, fingerprint)
1232 (result, output) = commands.getstatusoutput(cmd)
1234 return "'%s' failed with exit code %s" % (cmd, result)
1238 ################################################################################
1240 def gpg_keyring_args(keyrings=None):
1242 keyrings = Cnf.ValueList("Dinstall::GPGKeyring")
1244 return " ".join(["--keyring %s" % x for x in keyrings])
1246 ################################################################################
1248 def check_signature (sig_filename, data_filename="", keyrings=None, autofetch=None):
1250 Check the signature of a file and return the fingerprint if the
1251 signature is valid or 'None' if it's not. The first argument is the
1252 filename whose signature should be checked. The second argument is a
1253 reject function and is called when an error is found. The reject()
1254 function must allow for two arguments: the first is the error message,
1255 the second is an optional prefix string. It's possible for reject()
1256 to be called more than once during an invocation of check_signature().
1257 The third argument is optional and is the name of the files the
1258 detached signature applies to. The fourth argument is optional and is
1259 a *list* of keyrings to use. 'autofetch' can either be None, True or
1260 False. If None, the default behaviour specified in the config will be
1266 # Ensure the filename contains no shell meta-characters or other badness
1267 if not re_taint_free.match(sig_filename):
1268 rejects.append("!!WARNING!! tainted signature filename: '%s'." % (sig_filename))
1269 return (None, rejects)
1271 if data_filename and not re_taint_free.match(data_filename):
1272 rejects.append("!!WARNING!! tainted data filename: '%s'." % (data_filename))
1273 return (None, rejects)
1276 keyrings = Cnf.ValueList("Dinstall::GPGKeyring")
1278 # Autofetch the signing key if that's enabled
1279 if autofetch == None:
1280 autofetch = Cnf.get("Dinstall::KeyAutoFetch")
1282 error_msg = retrieve_key(sig_filename)
1284 rejects.append(error_msg)
1285 return (None, rejects)
1287 # Build the command line
1288 status_read, status_write = os.pipe()
1289 cmd = "gpgv --status-fd %s %s %s %s" % (
1290 status_write, gpg_keyring_args(keyrings), sig_filename, data_filename)
1292 # Invoke gpgv on the file
1293 (output, status, exit_status) = gpgv_get_status_output(cmd, status_read, status_write)
1295 # Process the status-fd output
1296 (keywords, internal_error) = process_gpgv_output(status)
1298 # If we failed to parse the status-fd output, let's just whine and bail now
1300 rejects.append("internal error while performing signature check on %s." % (sig_filename))
1301 rejects.append(internal_error, "")
1302 rejects.append("Please report the above errors to the Archive maintainers by replying to this mail.", "")
1303 return (None, rejects)
1305 # Now check for obviously bad things in the processed output
1306 if keywords.has_key("KEYREVOKED"):
1307 rejects.append("The key used to sign %s has been revoked." % (sig_filename))
1308 if keywords.has_key("BADSIG"):
1309 rejects.append("bad signature on %s." % (sig_filename))
1310 if keywords.has_key("ERRSIG") and not keywords.has_key("NO_PUBKEY"):
1311 rejects.append("failed to check signature on %s." % (sig_filename))
1312 if keywords.has_key("NO_PUBKEY"):
1313 args = keywords["NO_PUBKEY"]
1316 rejects.append("The key (0x%s) used to sign %s wasn't found in the keyring(s)." % (key, sig_filename))
1317 if keywords.has_key("BADARMOR"):
1318 rejects.append("ASCII armour of signature was corrupt in %s." % (sig_filename))
1319 if keywords.has_key("NODATA"):
1320 rejects.append("no signature found in %s." % (sig_filename))
1321 if keywords.has_key("EXPKEYSIG"):
1322 args = keywords["EXPKEYSIG"]
1325 rejects.append("Signature made by expired key 0x%s" % (key))
1326 if keywords.has_key("KEYEXPIRED") and not keywords.has_key("GOODSIG"):
1327 args = keywords["KEYEXPIRED"]
1331 if timestamp.count("T") == 0:
1333 expiredate = time.strftime("%Y-%m-%d", time.gmtime(float(timestamp)))
1335 expiredate = "unknown (%s)" % (timestamp)
1337 expiredate = timestamp
1338 rejects.append("The key used to sign %s has expired on %s" % (sig_filename, expiredate))
1340 if len(rejects) > 0:
1341 return (None, rejects)
1343 # Next check gpgv exited with a zero return code
1345 rejects.append("gpgv failed while checking %s." % (sig_filename))
1347 rejects.append(prefix_multi_line_string(status, " [GPG status-fd output:] "), "")
1349 rejects.append(prefix_multi_line_string(output, " [GPG output:] "), "")
1350 return (None, rejects)
1352 # Sanity check the good stuff we expect
1353 if not keywords.has_key("VALIDSIG"):
1354 rejects.append("signature on %s does not appear to be valid [No VALIDSIG]." % (sig_filename))
1356 args = keywords["VALIDSIG"]
1358 rejects.append("internal error while checking signature on %s." % (sig_filename))
1360 fingerprint = args[0]
1361 if not keywords.has_key("GOODSIG"):
1362 rejects.append("signature on %s does not appear to be valid [No GOODSIG]." % (sig_filename))
1363 if not keywords.has_key("SIG_ID"):
1364 rejects.append("signature on %s does not appear to be valid [No SIG_ID]." % (sig_filename))
1366 # Finally ensure there's not something we don't recognise
1367 known_keywords = Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="",
1368 SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="",
1369 NODATA="",NOTATION_DATA="",NOTATION_NAME="",KEYEXPIRED="")
1371 for keyword in keywords.keys():
1372 if not known_keywords.has_key(keyword):
1373 rejects.append("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], sig_filename))
1375 if len(rejects) > 0:
1376 return (None, rejects)
1378 return (fingerprint, [])
1380 ################################################################################
1382 def gpg_get_key_addresses(fingerprint):
1383 """retreive email addresses from gpg key uids for a given fingerprint"""
1384 addresses = key_uid_email_cache.get(fingerprint)
1385 if addresses != None:
1388 cmd = "gpg --no-default-keyring %s --fingerprint %s" \
1389 % (gpg_keyring_args(), fingerprint)
1390 (result, output) = commands.getstatusoutput(cmd)
1392 for l in output.split('\n'):
1393 m = re_gpg_uid.match(l)
1395 addresses.add(m.group(1))
1396 key_uid_email_cache[fingerprint] = addresses
1399 ################################################################################
1401 # Inspired(tm) by http://www.zopelabs.com/cookbook/1022242603
1403 def wrap(paragraph, max_length, prefix=""):
1407 words = paragraph.split()
1410 word_size = len(word)
1411 if word_size > max_length:
1413 s += line + '\n' + prefix
1414 s += word + '\n' + prefix
1417 new_length = len(line) + word_size + 1
1418 if new_length > max_length:
1419 s += line + '\n' + prefix
1432 ################################################################################
1434 def clean_symlink (src, dest, root):
1436 Relativize an absolute symlink from 'src' -> 'dest' relative to 'root'.
1439 src = src.replace(root, '', 1)
1440 dest = dest.replace(root, '', 1)
1441 dest = os.path.dirname(dest)
1442 new_src = '../' * len(dest.split('/'))
1443 return new_src + src
1445 ################################################################################
1447 def temp_filename(directory=None, prefix="dak", suffix=""):
1449 Return a secure and unique filename by pre-creating it.
1450 If 'directory' is non-null, it will be the directory the file is pre-created in.
1451 If 'prefix' is non-null, the filename will be prefixed with it, default is dak.
1452 If 'suffix' is non-null, the filename will end with it.
1454 Returns a pair (fd, name).
1457 return tempfile.mkstemp(suffix, prefix, directory)
1459 ################################################################################
1461 def temp_dirname(parent=None, prefix="dak", suffix=""):
1463 Return a secure and unique directory by pre-creating it.
1464 If 'parent' is non-null, it will be the directory the directory is pre-created in.
1465 If 'prefix' is non-null, the filename will be prefixed with it, default is dak.
1466 If 'suffix' is non-null, the filename will end with it.
1468 Returns a pathname to the new directory
1471 return tempfile.mkdtemp(suffix, prefix, parent)
1473 ################################################################################
1475 def is_email_alias(email):
1476 """ checks if the user part of the email is listed in the alias file """
1478 if alias_cache == None:
1479 aliasfn = which_alias_file()
1482 for l in open(aliasfn):
1483 alias_cache.add(l.split(':')[0])
1484 uid = email.split('@')[0]
1485 return uid in alias_cache
1487 ################################################################################
1489 def get_changes_files(from_dir):
1491 Takes a directory and lists all .changes files in it (as well as chdir'ing
1492 to the directory; this is due to broken behaviour on the part of p-u/p-a
1493 when you're not in the right place)
1495 Returns a list of filenames
1498 # Much of the rest of p-u/p-a depends on being in the right place
1500 changes_files = [x for x in os.listdir(from_dir) if x.endswith('.changes')]
1502 fubar("Failed to read list from directory %s (%s)" % (from_dir, e))
1504 return changes_files
1506 ################################################################################
1510 Cnf = apt_pkg.newConfiguration()
1511 apt_pkg.ReadConfigFileISC(Cnf,default_config)
1513 if which_conf_file() != default_config:
1514 apt_pkg.ReadConfigFileISC(Cnf,which_conf_file())