1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
3 # This program is free software; you can redistribute it and/or modify
4 # it under the terms of the GNU General Public License as published by
5 # the Free Software Foundation; either version 2 of the License, or
6 # (at your option) any later version.
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
13 # You should have received a copy of the GNU General Public License along
14 # with this program; if not, write to the Free Software Foundation, Inc.,
15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17 """module to handle uploads not yet installed to the archive
19 This module provides classes to handle uploads not yet installed to the
20 archive. Central is the L{Changes} class which represents a changes file.
21 It provides methods to access the included binary and source packages.
29 from daklib.gpg import SignedFile
30 from daklib.regexes import *
31 import daklib.packagelist
33 class UploadException(Exception):
36 class InvalidChangesException(UploadException):
39 class InvalidBinaryException(UploadException):
42 class InvalidSourceException(UploadException):
45 class InvalidHashException(UploadException):
46 def __init__(self, filename, hash_name, expected, actual):
47 self.filename = filename
48 self.hash_name = hash_name
49 self.expected = expected
52 return ("Invalid {0} hash for {1}:\n"
53 "According to the control file the {0} hash should be {2},\n"
56 "If you did not include {1} in you upload, a different version\n"
57 "might already be known to the archive software.") \
58 .format(self.hash_name, self.filename, self.expected, self.actual)
60 class InvalidFilenameException(UploadException):
61 def __init__(self, filename):
62 self.filename = filename
64 return "Invalid filename '{0}'.".format(self.filename)
66 class HashedFile(object):
67 """file with checksums
69 def __init__(self, filename, size, md5sum, sha1sum, sha256sum, section=None, priority=None):
70 self.filename = filename
81 """MD5 hash in hexdigits
85 self.sha1sum = sha1sum
86 """SHA1 hash in hexdigits
90 self.sha256sum = sha256sum
91 """SHA256 hash in hexdigits
95 self.section = section
100 self.priority = priority
101 """priority or C{None}
102 @type: str of C{None}
106 def from_file(cls, directory, filename, section=None, priority=None):
107 """create with values for an existing file
109 Create a C{HashedFile} object that refers to an already existing file.
112 @param directory: directory the file is located in
115 @param filename: filename
117 @type section: str or C{None}
118 @param section: optional section as given in .changes files
120 @type priority: str or C{None}
121 @param priority: optional priority as given in .changes files
123 @rtype: L{HashedFile}
124 @return: C{HashedFile} object for the given file
126 path = os.path.join(directory, filename)
127 size = os.stat(path).st_size
128 with open(path, 'r') as fh:
129 hashes = apt_pkg.Hashes(fh)
130 return cls(filename, size, hashes.md5, hashes.sha1, hashes.sha256, section, priority)
132 def check(self, directory):
135 Check if size and hashes match the expected value.
138 @param directory: directory the file is located in
140 @raise InvalidHashException: hash mismatch
142 path = os.path.join(directory, self.filename)
144 size = os.stat(path).st_size
145 if size != self.size:
146 raise InvalidHashException(self.filename, 'size', self.size, size)
148 with open(path) as fh:
149 hashes = apt_pkg.Hashes(fh)
151 if hashes.md5 != self.md5sum:
152 raise InvalidHashException(self.filename, 'md5sum', self.md5sum, hashes.md5)
154 if hashes.sha1 != self.sha1sum:
155 raise InvalidHashException(self.filename, 'sha1sum', self.sha1sum, hashes.sha1)
157 if hashes.sha256 != self.sha256sum:
158 raise InvalidHashException(self.filename, 'sha256sum', self.sha256sum, hashes.sha256)
160 def parse_file_list(control, has_priority_and_section):
161 """Parse Files and Checksums-* fields
163 @type control: dict-like
164 @param control: control file to take fields from
166 @type has_priority_and_section: bool
167 @param has_priority_and_section: Files field include section and priority
170 @raise InvalidChangesException: missing fields or other grave errors
173 @return: dict mapping filenames to L{daklib.upload.HashedFile} objects
177 for line in control.get("Files", "").split('\n'):
181 if has_priority_and_section:
182 (md5sum, size, section, priority, filename) = line.split()
183 entry = dict(md5sum=md5sum, size=long(size), section=section, priority=priority, filename=filename)
185 (md5sum, size, filename) = line.split()
186 entry = dict(md5sum=md5sum, size=long(size), filename=filename)
188 entries[filename] = entry
190 for line in control.get("Checksums-Sha1", "").split('\n'):
193 (sha1sum, size, filename) = line.split()
194 entry = entries.get(filename, None)
196 raise InvalidChangesException('{0} is listed in Checksums-Sha1, but not in Files.'.format(filename))
197 if entry is not None and entry.get('size', None) != long(size):
198 raise InvalidChangesException('Size for {0} in Files and Checksum-Sha1 fields differ.'.format(filename))
199 entry['sha1sum'] = sha1sum
201 for line in control.get("Checksums-Sha256", "").split('\n'):
204 (sha256sum, size, filename) = line.split()
205 entry = entries.get(filename, None)
207 raise InvalidChangesException('{0} is listed in Checksums-Sha256, but not in Files.'.format(filename))
208 if entry is not None and entry.get('size', None) != long(size):
209 raise InvalidChangesException('Size for {0} in Files and Checksum-Sha256 fields differ.'.format(filename))
210 entry['sha256sum'] = sha256sum
213 for entry in entries.itervalues():
214 filename = entry['filename']
215 if 'size' not in entry:
216 raise InvalidChangesException('No size for {0}.'.format(filename))
217 if 'md5sum' not in entry:
218 raise InvalidChangesException('No md5sum for {0}.'.format(filename))
219 if 'sha1sum' not in entry:
220 raise InvalidChangesException('No sha1sum for {0}.'.format(filename))
221 if 'sha256sum' not in entry:
222 raise InvalidChangesException('No sha256sum for {0}.'.format(filename))
223 if not re_file_safe.match(filename):
224 raise InvalidChangesException("{0}: References file with unsafe filename {1}.".format(self.filename, filename))
225 f = files[filename] = HashedFile(**entry)
229 class Changes(object):
230 """Representation of a .changes file
232 def __init__(self, directory, filename, keyrings, require_signature=True):
233 if not re_file_safe.match(filename):
234 raise InvalidChangesException('{0}: unsafe filename'.format(filename))
236 self.directory = directory
237 """directory the .changes is located in
241 self.filename = filename
242 """name of the .changes file
246 data = open(self.path).read()
247 self._signed_file = SignedFile(data, keyrings, require_signature)
248 self.changes = apt_pkg.TagSection(self._signed_file.contents)
249 """dict to access fields of the .changes file
253 self._binaries = None
256 self._keyrings = keyrings
257 self._require_signature = require_signature
261 """path to the .changes file
264 return os.path.join(self.directory, self.filename)
267 def primary_fingerprint(self):
268 """fingerprint of the key used for signing the .changes file
271 return self._signed_file.primary_fingerprint
274 def valid_signature(self):
275 """C{True} if the .changes has a valid signature
278 return self._signed_file.valid
281 def signature_timestamp(self):
282 return self._signed_file.signature_timestamp
285 def contents_sha1(self):
286 return self._signed_file.contents_sha1
289 def architectures(self):
290 """list of architectures included in the upload
293 return self.changes.get('Architecture', '').split()
296 def distributions(self):
297 """list of target distributions for the upload
300 return self.changes['Distribution'].split()
304 """included source or C{None}
305 @type: L{daklib.upload.Source} or C{None}
307 if self._source is None:
309 for f in self.files.itervalues():
310 if re_file_dsc.match(f.filename) or re_file_source.match(f.filename):
311 source_files.append(f)
312 if len(source_files) > 0:
313 self._source = Source(self.directory, source_files, self._keyrings, self._require_signature)
318 """C{True} if the upload includes source
321 return "source" in self.architectures
324 def source_name(self):
325 """source package name
328 return re_field_source.match(self.changes['Source']).group('package')
332 """included binary packages
333 @type: list of L{daklib.upload.Binary}
335 if self._binaries is None:
337 for f in self.files.itervalues():
338 if re_file_binary.match(f.filename):
339 binaries.append(Binary(self.directory, f))
340 self._binaries = binaries
341 return self._binaries
344 def byhand_files(self):
345 """included byhand files
346 @type: list of L{daklib.upload.HashedFile}
350 for f in self.files.itervalues():
351 if re_file_dsc.match(f.filename) or re_file_source.match(f.filename) or re_file_binary.match(f.filename):
353 if f.section != 'byhand' and f.section[:4] != 'raw-':
354 raise InvalidChangesException("{0}: {1} looks like a byhand package, but is in section {2}".format(self.filename, f.filename, f.section))
360 def binary_names(self):
361 """names of included binary packages
364 return self.changes['Binary'].split()
367 def closed_bugs(self):
368 """bugs closed by this upload
371 return self.changes.get('Closes', '').split()
375 """dict mapping filenames to L{daklib.upload.HashedFile} objects
378 if self._files is None:
379 self._files = parse_file_list(self.changes, True)
384 """total size of files included in this upload in bytes
388 for f in self.files.itervalues():
392 def __cmp__(self, other):
393 """compare two changes files
395 We sort by source name and version first. If these are identical,
396 we sort changes that include source before those without source (so
397 that sourceful uploads get processed first), and finally fall back
398 to the filename (this should really never happen).
401 @return: n where n < 0 if self < other, n = 0 if self == other, n > 0 if self > other
403 ret = cmp(self.changes.get('Source'), other.changes.get('Source'))
407 ret = apt_pkg.version_compare(self.changes.get('Version', ''), other.changes.get('Version', ''))
410 # sort changes with source before changes without source
411 if 'source' in self.architectures and 'source' not in other.architectures:
413 elif 'source' not in self.architectures and 'source' in other.architectures:
419 # fall back to filename
420 ret = cmp(self.filename, other.filename)
424 class Binary(object):
425 """Representation of a binary package
427 def __init__(self, directory, hashed_file):
428 self.hashed_file = hashed_file
429 """file object for the .deb
433 path = os.path.join(directory, hashed_file.filename)
434 data = apt_inst.DebFile(path).control.extractdata("control")
436 self.control = apt_pkg.TagSection(data)
437 """dict to access fields in DEBIAN/control
442 def from_file(cls, directory, filename):
443 hashed_file = HashedFile.from_file(directory, filename)
444 return cls(directory, hashed_file)
448 """get tuple with source package name and version
451 source = self.control.get("Source", None)
453 return (self.control["Package"], self.control["Version"])
454 match = re_field_source.match(source)
456 raise InvalidBinaryException('{0}: Invalid Source field.'.format(self.hashed_file.filename))
457 version = match.group('version')
459 version = self.control['Version']
460 return (match.group('package'), version)
464 return self.control['Package']
468 """package type ('deb' or 'udeb')
471 match = re_file_binary.match(self.hashed_file.filename)
473 raise InvalidBinaryException('{0}: Does not match re_file_binary'.format(self.hashed_file.filename))
474 return match.group('type')
481 fields = self.control['Section'].split('/')
486 class Source(object):
487 """Representation of a source package
489 def __init__(self, directory, hashed_files, keyrings, require_signature=True):
490 self.hashed_files = hashed_files
491 """list of source files (including the .dsc itself)
492 @type: list of L{HashedFile}
495 self._dsc_file = None
496 for f in hashed_files:
497 if re_file_dsc.match(f.filename):
498 if self._dsc_file is not None:
499 raise InvalidSourceException("Multiple .dsc found ({0} and {1})".format(self._dsc_file.filename, f.filename))
503 # make sure the hash for the dsc is valid before we use it
504 self._dsc_file.check(directory)
506 dsc_file_path = os.path.join(directory, self._dsc_file.filename)
507 data = open(dsc_file_path, 'r').read()
508 self._signed_file = SignedFile(data, keyrings, require_signature)
509 self.dsc = apt_pkg.TagSection(self._signed_file.contents)
510 """dict to access fields in the .dsc file
514 self.package_list = daklib.packagelist.PackageList(self.dsc)
515 """Information about packages built by the source.
516 @type: daklib.packagelist.PackageList
522 def from_file(cls, directory, filename, keyrings, require_signature=True):
523 hashed_file = HashedFile.from_file(directory, filename)
524 return cls(directory, [hashed_file], keyrings, require_signature)
528 """dict mapping filenames to L{HashedFile} objects for additional source files
530 This list does not include the .dsc itself.
534 if self._files is None:
535 self._files = parse_file_list(self.dsc, False)
539 def primary_fingerprint(self):
540 """fingerprint of the key used to sign the .dsc
543 return self._signed_file.primary_fingerprint
546 def valid_signature(self):
547 """C{True} if the .dsc has a valid signature
550 return self._signed_file.valid
554 """guessed component name
556 Might be wrong. Don't rely on this.
560 if 'Section' not in self.dsc:
562 fields = self.dsc['Section'].split('/')
569 """filename of .dsc file
572 return self._dsc_file.filename