1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
3 # This program is free software; you can redistribute it and/or modify
4 # it under the terms of the GNU General Public License as published by
5 # the Free Software Foundation; either version 2 of the License, or
6 # (at your option) any later version.
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
13 # You should have received a copy of the GNU General Public License along
14 # with this program; if not, write to the Free Software Foundation, Inc.,
15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17 """module to manipulate the archive
19 This module provides classes to manipulate the archive.
22 from daklib.dbconn import *
23 import daklib.checks as checks
24 from daklib.config import Config
25 import daklib.upload as upload
26 import daklib.utils as utils
27 from daklib.fstransactions import FilesystemTransaction
28 from daklib.regexes import re_changelog_versions, re_bin_only_nmu
31 from datetime import datetime
35 from sqlalchemy.orm.exc import NoResultFound
40 class ArchiveException(Exception):
43 class HashMismatchException(ArchiveException):
46 class ArchiveTransaction(object):
47 """manipulate the archive in a transaction
50 self.fs = FilesystemTransaction()
51 self.session = DBConn().session()
53 def get_file(self, hashed_file, source_name, check_hashes=True):
54 """Look for file C{hashed_file} in database
56 @type hashed_file: L{daklib.upload.HashedFile}
57 @param hashed_file: file to look for in the database
59 @type source_name: str
60 @param source_name: source package name
62 @type check_hashes: bool
63 @param check_hashes: check size and hashes match
65 @raise KeyError: file was not found in the database
66 @raise HashMismatchException: hash mismatch
68 @rtype: L{daklib.dbconn.PoolFile}
69 @return: database entry for the file
71 poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
73 poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
74 if check_hashes and (poolfile.filesize != hashed_file.size
75 or poolfile.md5sum != hashed_file.md5sum
76 or poolfile.sha1sum != hashed_file.sha1sum
77 or poolfile.sha256sum != hashed_file.sha256sum):
78 raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
81 raise KeyError('{0} not found in database.'.format(poolname))
83 def _install_file(self, directory, hashed_file, archive, component, source_name):
86 Will not give an error when the file is already present.
88 @rtype: L{daklib.dbconn.PoolFile}
89 @return: batabase object for the new file
91 session = self.session
93 poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
95 poolfile = self.get_file(hashed_file, source_name)
97 poolfile = PoolFile(filename=poolname, filesize=hashed_file.size)
98 poolfile.md5sum = hashed_file.md5sum
99 poolfile.sha1sum = hashed_file.sha1sum
100 poolfile.sha256sum = hashed_file.sha256sum
101 session.add(poolfile)
105 session.query(ArchiveFile).filter_by(archive=archive, component=component, file=poolfile).one()
106 except NoResultFound:
107 archive_file = ArchiveFile(archive, component, poolfile)
108 session.add(archive_file)
111 path = os.path.join(archive.path, 'pool', component.component_name, poolname)
112 hashed_file_path = os.path.join(directory, hashed_file.filename)
113 self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
117 def install_binary(self, directory, binary, suite, component, allow_tainted=False, fingerprint=None, source_suites=None, extra_source_archives=None):
118 """Install a binary package
121 @param directory: directory the binary package is located in
123 @type binary: L{daklib.upload.Binary}
124 @param binary: binary package to install
126 @type suite: L{daklib.dbconn.Suite}
127 @param suite: target suite
129 @type component: L{daklib.dbconn.Component}
130 @param component: target component
132 @type allow_tainted: bool
133 @param allow_tainted: allow to copy additional files from tainted archives
135 @type fingerprint: L{daklib.dbconn.Fingerprint}
136 @param fingerprint: optional fingerprint
138 @type source_suites: SQLAlchemy subquery for C{daklib.dbconn.Suite} or C{True}
139 @param source_suites: suites to copy the source from if they are not
140 in C{suite} or C{True} to allow copying from any
143 @type extra_source_archives: list of L{daklib.dbconn.Archive}
144 @param extra_source_archives: extra archives to copy Built-Using sources from
146 @rtype: L{daklib.dbconn.DBBinary}
147 @return: databse object for the new package
149 session = self.session
150 control = binary.control
151 maintainer = get_or_set_maintainer(control['Maintainer'], session)
152 architecture = get_architecture(control['Architecture'], session)
154 (source_name, source_version) = binary.source
155 source_query = session.query(DBSource).filter_by(source=source_name, version=source_version)
156 source = source_query.filter(DBSource.suites.contains(suite)).first()
158 if source_suites != True:
159 source_query = source_query.join(DBSource.suites) \
160 .filter(Suite.suite_id == source_suites.c.id)
161 source = source_query.first()
163 raise ArchiveException('{0}: trying to install to {1}, but could not find source'.format(binary.hashed_file.filename, suite.suite_name))
164 self.copy_source(source, suite, component)
166 db_file = self._install_file(directory, binary.hashed_file, suite.archive, component, source_name)
169 package=control['Package'],
170 version=control['Version'],
171 architecture=architecture,
175 maintainer=maintainer,
177 binarytype=binary.type,
178 fingerprint=fingerprint,
182 db_binary = session.query(DBBinary).filter_by(**unique).one()
183 for key, value in rest.iteritems():
184 if getattr(db_binary, key) != value:
185 raise ArchiveException('{0}: Does not match binary in database.'.format(binary.hashed_file.filename))
186 except NoResultFound:
187 db_binary = DBBinary(**unique)
188 for key, value in rest.iteritems():
189 setattr(db_binary, key, value)
190 session.add(db_binary)
192 import_metadata_into_db(db_binary, session)
194 self._add_built_using(db_binary, binary.hashed_file.filename, control, suite, extra_archives=extra_source_archives)
196 if suite not in db_binary.suites:
197 db_binary.suites.append(suite)
203 def _ensure_extra_source_exists(self, filename, source, archive, extra_archives=None):
204 """ensure source exists in the given archive
206 This is intended to be used to check that Built-Using sources exist.
209 @param filename: filename to use in error messages
211 @type source: L{daklib.dbconn.DBSource}
212 @param source: source to look for
214 @type archive: L{daklib.dbconn.Archive}
215 @param archive: archive to look in
217 @type extra_archives: list of L{daklib.dbconn.Archive}
218 @param extra_archives: list of archives to copy the source package from
219 if it is not yet present in C{archive}
221 session = self.session
222 db_file = session.query(ArchiveFile).filter_by(file=source.poolfile, archive=archive).first()
223 if db_file is not None:
226 # Try to copy file from one extra archive
227 if extra_archives is None:
229 db_file = session.query(ArchiveFile).filter_by(file=source.poolfile).filter(ArchiveFile.archive_id.in_([ a.archive_id for a in extra_archives])).first()
231 raise ArchiveException('{0}: Built-Using refers to package {1} (= {2}) not in target archive {3}.'.format(filename, source.source, source.version, archive.archive_name))
233 source_archive = db_file.archive
234 for dsc_file in source.srcfiles:
235 af = session.query(ArchiveFile).filter_by(file=dsc_file.poolfile, archive=source_archive, component=db_file.component).one()
236 # We were given an explicit list of archives so it is okay to copy from tainted archives.
237 self._copy_file(af.file, archive, db_file.component, allow_tainted=True)
239 def _add_built_using(self, db_binary, filename, control, suite, extra_archives=None):
240 """Add Built-Using sources to C{db_binary.extra_sources}
242 session = self.session
243 built_using = control.get('Built-Using', None)
245 if built_using is not None:
246 for dep in apt_pkg.parse_depends(built_using):
247 assert len(dep) == 1, 'Alternatives are not allowed in Built-Using field'
248 bu_source_name, bu_source_version, comp = dep[0]
249 assert comp == '=', 'Built-Using must contain strict dependencies'
251 bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first()
252 if bu_source is None:
253 raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version))
255 self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives)
257 db_binary.extra_sources.append(bu_source)
259 def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None):
260 """Install a source package
263 @param directory: directory the source package is located in
265 @type source: L{daklib.upload.Source}
266 @param source: source package to install
268 @type suite: L{daklib.dbconn.Suite}
269 @param suite: target suite
271 @type component: L{daklib.dbconn.Component}
272 @param component: target component
274 @type changed_by: L{daklib.dbconn.Maintainer}
275 @param changed_by: person who prepared this version of the package
277 @type allow_tainted: bool
278 @param allow_tainted: allow to copy additional files from tainted archives
280 @type fingerprint: L{daklib.dbconn.Fingerprint}
281 @param fingerprint: optional fingerprint
283 @rtype: L{daklib.dbconn.DBSource}
284 @return: database object for the new source
286 session = self.session
287 archive = suite.archive
289 maintainer = get_or_set_maintainer(control['Maintainer'], session)
290 source_name = control['Source']
292 ### Add source package to database
294 # We need to install the .dsc first as the DBSource object refers to it.
295 db_file_dsc = self._install_file(directory, source._dsc_file, archive, component, source_name)
299 version=control['Version'],
302 maintainer=maintainer,
303 changedby=changed_by,
304 #install_date=datetime.now().date(),
305 poolfile=db_file_dsc,
306 fingerprint=fingerprint,
307 dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'),
312 db_source = session.query(DBSource).filter_by(**unique).one()
313 for key, value in rest.iteritems():
314 if getattr(db_source, key) != value:
315 raise ArchiveException('{0}: Does not match source in database.'.format(source._dsc_file.filename))
316 except NoResultFound:
318 db_source = DBSource(**unique)
319 for key, value in rest.iteritems():
320 setattr(db_source, key, value)
321 # XXX: set as default in postgres?
322 db_source.install_date = datetime.now().date()
323 session.add(db_source)
326 # Add .dsc file. Other files will be added later.
327 db_dsc_file = DSCFile()
328 db_dsc_file.source = db_source
329 db_dsc_file.poolfile = db_file_dsc
330 session.add(db_dsc_file)
333 if suite in db_source.suites:
336 db_source.suites.append(suite)
339 for f in db_source.srcfiles:
340 self._copy_file(f.poolfile, archive, component, allow_tainted=allow_tainted)
343 ### Now add remaining files and copy them to the archive.
345 for hashed_file in source.files.itervalues():
346 hashed_file_path = os.path.join(directory, hashed_file.filename)
347 if os.path.exists(hashed_file_path):
348 db_file = self._install_file(directory, hashed_file, archive, component, source_name)
351 db_file = self.get_file(hashed_file, source_name)
352 self._copy_file(db_file, archive, component, allow_tainted=allow_tainted)
354 db_dsc_file = DSCFile()
355 db_dsc_file.source = db_source
356 db_dsc_file.poolfile = db_file
357 session.add(db_dsc_file)
361 # Importing is safe as we only arrive here when we did not find the source already installed earlier.
362 import_metadata_into_db(db_source, session)
364 # Uploaders are the maintainer and co-maintainers from the Uploaders field
365 db_source.uploaders.append(maintainer)
366 if 'Uploaders' in control:
367 from daklib.textutils import split_uploaders
368 for u in split_uploaders(control['Uploaders']):
369 db_source.uploaders.append(get_or_set_maintainer(u, session))
374 def _copy_file(self, db_file, archive, component, allow_tainted=False):
375 """Copy a file to the given archive and component
377 @type db_file: L{daklib.dbconn.PoolFile}
378 @param db_file: file to copy
380 @type archive: L{daklib.dbconn.Archive}
381 @param archive: target archive
383 @type component: L{daklib.dbconn.Archive}
384 @param component: target component
386 @type allow_tainted: bool
387 @param allow_tainted: allow to copy from tainted archives (such as NEW)
389 session = self.session
391 if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None:
392 query = session.query(ArchiveFile).filter_by(file=db_file, component=component)
393 if not allow_tainted:
394 query = query.join(Archive).filter(Archive.tainted == False)
396 source_af = query.first()
397 if source_af is None:
398 raise ArchiveException('cp: Could not find {0} in component {1} in any archive.'.format(db_file.filename, component.component_name))
399 target_af = ArchiveFile(archive, component, db_file)
400 session.add(target_af)
402 self.fs.copy(source_af.path, target_af.path, link=False, mode=archive.mode)
404 def copy_binary(self, db_binary, suite, component, allow_tainted=False, extra_archives=None):
405 """Copy a binary package to the given suite and component
407 @type db_binary: L{daklib.dbconn.DBBinary}
408 @param db_binary: binary to copy
410 @type suite: L{daklib.dbconn.Suite}
411 @param suite: target suite
413 @type component: L{daklib.dbconn.Component}
414 @param component: target component
416 @type allow_tainted: bool
417 @param allow_tainted: allow to copy from tainted archives (such as NEW)
419 @type extra_archives: list of L{daklib.dbconn.Archive}
420 @param extra_archives: extra archives to copy Built-Using sources from
422 session = self.session
423 archive = suite.archive
427 filename = db_binary.poolfile.filename
429 # make sure source is present in target archive
430 db_source = db_binary.source
431 if session.query(ArchiveFile).filter_by(archive=archive, file=db_source.poolfile).first() is None:
432 raise ArchiveException('{0}: cannot copy to {1}: source is not present in target archive'.format(filename, suite.suite_name))
434 # make sure built-using packages are present in target archive
435 for db_source in db_binary.extra_sources:
436 self._ensure_extra_source_exists(filename, db_source, archive, extra_archives=extra_archives)
439 db_file = db_binary.poolfile
440 self._copy_file(db_file, suite.archive, component, allow_tainted=allow_tainted)
441 if suite not in db_binary.suites:
442 db_binary.suites.append(suite)
445 def copy_source(self, db_source, suite, component, allow_tainted=False):
446 """Copy a source package to the given suite and component
448 @type db_source: L{daklib.dbconn.DBSource}
449 @param db_source: source to copy
451 @type suite: L{daklib.dbconn.Suite}
452 @param suite: target suite
454 @type component: L{daklib.dbconn.Component}
455 @param component: target component
457 @type allow_tainted: bool
458 @param allow_tainted: allow to copy from tainted archives (such as NEW)
460 archive = suite.archive
463 for db_dsc_file in db_source.srcfiles:
464 self._copy_file(db_dsc_file.poolfile, archive, component, allow_tainted=allow_tainted)
465 if suite not in db_source.suites:
466 db_source.suites.append(suite)
469 def remove_file(self, db_file, archive, component):
470 """Remove a file from a given archive and component
472 @type db_file: L{daklib.dbconn.PoolFile}
473 @param db_file: file to remove
475 @type archive: L{daklib.dbconn.Archive}
476 @param archive: archive to remove the file from
478 @type component: L{daklib.dbconn.Component}
479 @param component: component to remove the file from
481 af = self.session.query(ArchiveFile).filter_by(file=db_file, archive=archive, component=component)
482 self.fs.unlink(af.path)
483 self.session.delete(af)
485 def remove_binary(self, binary, suite):
486 """Remove a binary from a given suite and component
488 @type binary: L{daklib.dbconn.DBBinary}
489 @param binary: binary to remove
491 @type suite: L{daklib.dbconn.Suite}
492 @param suite: suite to remove the package from
494 binary.suites.remove(suite)
497 def remove_source(self, source, suite):
498 """Remove a source from a given suite and component
500 @type source: L{daklib.dbconn.DBSource}
501 @param source: source to remove
503 @type suite: L{daklib.dbconn.Suite}
504 @param suite: suite to remove the package from
506 @raise ArchiveException: source package is still referenced by other
507 binaries in the suite
509 session = self.session
511 query = session.query(DBBinary).filter_by(source=source) \
512 .filter(DBBinary.suites.contains(suite))
513 if query.first() is not None:
514 raise ArchiveException('src:{0} is still used by binaries in suite {1}'.format(source.source, suite.suite_name))
516 source.suites.remove(suite)
522 self.session.commit()
525 self.session.rollback()
529 """rollback changes"""
530 self.session.rollback()
536 def __exit__(self, type, value, traceback):
543 class ArchiveUpload(object):
546 This class can be used in a with-statement::
548 with ArchiveUpload(...) as upload:
551 Doing so will automatically run any required cleanup and also rollback the
552 transaction if it was not committed.
554 def __init__(self, directory, changes, keyrings):
555 self.transaction = ArchiveTransaction()
556 """transaction used to handle the upload
557 @type: L{daklib.archive.ArchiveTransaction}
560 self.session = self.transaction.session
561 """database session"""
563 self.original_directory = directory
564 self.original_changes = changes
568 @type: L{daklib.upload.Changes}
571 self.directory = None
572 """directory with temporary copy of files. set by C{prepare}
576 self.keyrings = keyrings
578 self.fingerprint = self.session.query(Fingerprint).filter_by(fingerprint=changes.primary_fingerprint).one()
579 """fingerprint of the key used to sign the upload
580 @type: L{daklib.dbconn.Fingerprint}
583 self.reject_reasons = []
584 """reasons why the upload cannot by accepted
594 self.final_suites = None
597 """upload is NEW. set by C{check}
601 self._checked = False
602 """checks passes. set by C{check}
606 self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one()
607 self._new = self._new_queue.suite
609 def warn(self, message):
610 """add a warning message
612 Adds a warning message that can later be seen in C{self.warnings}
614 @type message: string
615 @param message: warning message
617 self.warnings.append(message)
620 """prepare upload for further processing
622 This copies the files involved to a temporary directory. If you use
623 this method directly, you have to remove the directory given by the
624 C{directory} attribute later on your own.
626 Instead of using the method directly, you can also use a with-statement::
628 with ArchiveUpload(...) as upload:
631 This will automatically handle any required cleanup.
633 assert self.directory is None
634 assert self.original_changes.valid_signature
637 session = self.transaction.session
639 self.directory = utils.temp_dirname(parent=cnf.get('Dir::TempPath'),
640 mode=0o2750, group=cnf.unprivgroup)
641 with FilesystemTransaction() as fs:
642 src = os.path.join(self.original_directory, self.original_changes.filename)
643 dst = os.path.join(self.directory, self.original_changes.filename)
644 fs.copy(src, dst, mode=0o640)
646 self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings)
648 for f in self.changes.files.itervalues():
649 src = os.path.join(self.original_directory, f.filename)
650 dst = os.path.join(self.directory, f.filename)
651 if not os.path.exists(src):
653 fs.copy(src, dst, mode=0o640)
655 source = self.changes.source
656 if source is not None:
657 for f in source.files.itervalues():
658 src = os.path.join(self.original_directory, f.filename)
659 dst = os.path.join(self.directory, f.filename)
660 if not os.path.exists(dst):
662 db_file = self.transaction.get_file(f, source.dsc['Source'], check_hashes=False)
663 db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first()
664 fs.copy(db_archive_file.path, dst, symlink=True)
666 # Ignore if get_file could not find it. Upload will
667 # probably be rejected later.
670 def unpacked_source(self):
671 """Path to unpacked source
673 Get path to the unpacked source. This method does unpack the source
674 into a temporary directory under C{self.directory} if it has not
675 been done so already.
677 @rtype: str or C{None}
678 @return: string giving the path to the unpacked source directory
679 or C{None} if no source was included in the upload.
681 assert self.directory is not None
683 source = self.changes.source
686 dsc_path = os.path.join(self.directory, source._dsc_file.filename)
688 sourcedir = os.path.join(self.directory, 'source')
689 if not os.path.exists(sourcedir):
690 devnull = open('/dev/null', 'w')
691 subprocess.check_call(["dpkg-source", "--no-copy", "--no-check", "-x", dsc_path, sourcedir], shell=False, stdout=devnull)
692 if not os.path.isdir(sourcedir):
693 raise Exception("{0} is not a directory after extracting source package".format(sourcedir))
696 def _map_suite(self, suite_name):
697 for rule in Config().value_list("SuiteMappings"):
698 fields = rule.split()
700 if rtype == "map" or rtype == "silent-map":
701 (src, dst) = fields[1:3]
702 if src == suite_name:
704 if rtype != "silent-map":
705 self.warnings.append('Mapping {0} to {1}.'.format(src, dst))
706 elif rtype == "ignore":
708 if suite_name == ignored:
709 self.warnings.append('Ignoring target suite {0}.'.format(ignored))
711 elif rtype == "reject":
713 if suite_name == rejected:
714 self.reject_reasons.append('Uploads to {0} are not accepted.'.format(suite))
715 ## XXX: propup-version and map-unreleased not yet implemented
718 def _mapped_suites(self):
719 """Get target suites after mappings
721 @rtype: list of L{daklib.dbconn.Suite}
722 @return: list giving the mapped target suites of this upload
724 session = self.session
727 for dist in self.changes.distributions:
728 suite_name = self._map_suite(dist)
729 if suite_name is not None:
730 suite_names.append(suite_name)
732 suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
735 def _check_new(self, suite):
736 """Check if upload is NEW
738 An upload is NEW if it has binary or source packages that do not have
739 an override in C{suite} OR if it references files ONLY in a tainted
740 archive (eg. when it references files in NEW).
743 @return: C{True} if the upload is NEW, C{False} otherwise
745 session = self.session
748 # Check for missing overrides
749 for b in self.changes.binaries:
750 override = self._binary_override(suite, b)
752 self.warnings.append('binary:{0} is NEW.'.format(b.control['Package']))
755 if self.changes.source is not None:
756 override = self._source_override(suite, self.changes.source)
758 self.warnings.append('source:{0} is NEW.'.format(self.changes.source.dsc['Source']))
761 # Check if we reference a file only in a tainted archive
762 files = self.changes.files.values()
763 if self.changes.source is not None:
764 files.extend(self.changes.source.files.values())
766 query = session.query(ArchiveFile).join(PoolFile).filter(PoolFile.sha1sum == f.sha1sum)
767 query_untainted = query.join(Archive).filter(Archive.tainted == False)
769 in_archive = (query.first() is not None)
770 in_untainted_archive = (query_untainted.first() is not None)
772 if in_archive and not in_untainted_archive:
773 self.warnings.append('{0} is only available in NEW.'.format(f.filename))
778 def _final_suites(self):
779 session = self.session
781 mapped_suites = self._mapped_suites()
784 for suite in mapped_suites:
785 overridesuite = suite
786 if suite.overridesuite is not None:
787 overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
788 if self._check_new(overridesuite):
790 final_suites.add(suite)
794 def _binary_override(self, suite, binary):
795 """Get override entry for a binary
797 @type suite: L{daklib.dbconn.Suite}
798 @param suite: suite to get override for
800 @type binary: L{daklib.upload.Binary}
801 @param binary: binary to get override for
803 @rtype: L{daklib.dbconn.Override} or C{None}
804 @return: override for the given binary or C{None}
806 if suite.overridesuite is not None:
807 suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
809 mapped_component = get_mapped_component(binary.component)
810 if mapped_component is None:
813 query = self.session.query(Override).filter_by(suite=suite, package=binary.control['Package']) \
814 .join(Component).filter(Component.component_name == mapped_component.component_name) \
815 .join(OverrideType).filter(OverrideType.overridetype == binary.type)
819 except NoResultFound:
822 def _source_override(self, suite, source):
823 """Get override entry for a source
825 @type suite: L{daklib.dbconn.Suite}
826 @param suite: suite to get override for
828 @type source: L{daklib.upload.Source}
829 @param source: source to get override for
831 @rtype: L{daklib.dbconn.Override} or C{None}
832 @return: override for the given source or C{None}
834 if suite.overridesuite is not None:
835 suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
837 # XXX: component for source?
838 query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \
839 .join(OverrideType).filter(OverrideType.overridetype == 'dsc')
843 except NoResultFound:
846 def _binary_component(self, suite, binary, only_overrides=True):
847 """get component for a binary
849 By default this will only look at overrides to get the right component;
850 if C{only_overrides} is C{False} this method will also look at the
853 @type suite: L{daklib.dbconn.Suite}
855 @type binary: L{daklib.upload.Binary}
857 @type only_overrides: bool
858 @param only_overrides: only use overrides to get the right component
860 @rtype: L{daklib.dbconn.Component} or C{None}
862 override = self._binary_override(suite, binary)
863 if override is not None:
864 return override.component
867 return get_mapped_component(binary.component, self.session)
869 def check(self, force=False):
870 """run checks against the upload
873 @param force: ignore failing forcable checks
876 @return: C{True} if all checks passed, C{False} otherwise
878 # XXX: needs to be better structured.
879 assert self.changes.valid_signature
882 # Validate signatures and hashes before we do any real work:
884 checks.SignatureCheck,
887 checks.ExternalHashesCheck,
890 checks.BinaryTimestampCheck,
891 checks.SingleDistributionCheck,
895 final_suites = self._final_suites()
896 if len(final_suites) == 0:
897 self.reject_reasons.append('No target suite found. Please check your target distribution and that you uploaded to the right archive.')
900 self.final_suites = final_suites
903 checks.TransitionCheck,
905 checks.NoSourceOnlyCheck,
912 checks.SourceFormatCheck,
913 checks.SuiteArchitectureCheck,
916 for suite in final_suites:
917 chk().per_suite_check(self, suite)
919 if len(self.reject_reasons) != 0:
924 except checks.Reject as e:
925 self.reject_reasons.append(unicode(e))
926 except Exception as e:
927 self.reject_reasons.append("Processing raised an exception: {0}.\n{1}".format(e, traceback.format_exc()))
930 def _install_to_suite(self, suite, source_component_func, binary_component_func, source_suites=None, extra_source_archives=None):
931 """Install upload to the given suite
933 @type suite: L{daklib.dbconn.Suite}
934 @param suite: suite to install the package into. This is the real suite,
935 ie. after any redirection to NEW or a policy queue
937 @param source_component_func: function to get the L{daklib.dbconn.Component}
938 for a L{daklib.upload.Source} object
940 @param binary_component_func: function to get the L{daklib.dbconn.Component}
941 for a L{daklib.upload.Binary} object
943 @param source_suites: see L{daklib.archive.ArchiveTransaction.install_binary}
945 @param extra_source_archives: see L{daklib.archive.ArchiveTransaction.install_binary}
947 @return: tuple with two elements. The first is a L{daklib.dbconn.DBSource}
948 object for the install source or C{None} if no source was
949 included. The second is a list of L{daklib.dbconn.DBBinary}
950 objects for the installed binary packages.
952 # XXX: move this function to ArchiveTransaction?
954 control = self.changes.changes
955 changed_by = get_or_set_maintainer(control.get('Changed-By', control['Maintainer']), self.session)
957 if source_suites is None:
958 source_suites = self.session.query(Suite).join((VersionCheck, VersionCheck.reference_id == Suite.suite_id)).filter(VersionCheck.check == 'Enhances').filter(VersionCheck.suite == suite).subquery()
960 source = self.changes.source
961 if source is not None:
962 component = source_component_func(source)
963 db_source = self.transaction.install_source(self.directory, source, suite, component, changed_by, fingerprint=self.fingerprint)
968 for binary in self.changes.binaries:
969 component = binary_component_func(binary)
970 db_binary = self.transaction.install_binary(self.directory, binary, suite, component, fingerprint=self.fingerprint, source_suites=source_suites, extra_source_archives=extra_source_archives)
971 db_binaries.append(db_binary)
973 if suite.copychanges:
974 src = os.path.join(self.directory, self.changes.filename)
975 dst = os.path.join(suite.archive.path, 'dists', suite.suite_name, self.changes.filename)
976 self.transaction.fs.copy(src, dst, mode=suite.archive.mode)
978 return (db_source, db_binaries)
980 def _install_changes(self):
981 assert self.changes.valid_signature
982 control = self.changes.changes
983 session = self.transaction.session
987 # Only add changelog for sourceful uploads and binNMUs
988 if 'source' in self.changes.architectures or re_bin_only_nmu.search(control['Version']):
989 query = 'INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id'
990 changelog_id = session.execute(query, {'changelog': control['Changes']}).scalar()
991 assert changelog_id is not None
993 db_changes = DBChange()
994 db_changes.changesname = self.changes.filename
995 db_changes.source = control['Source']
996 db_changes.binaries = control.get('Binary', None)
997 db_changes.architecture = control['Architecture']
998 db_changes.version = control['Version']
999 db_changes.distribution = control['Distribution']
1000 db_changes.urgency = control['Urgency']
1001 db_changes.maintainer = control['Maintainer']
1002 db_changes.changedby = control.get('Changed-By', control['Maintainer'])
1003 db_changes.date = control['Date']
1004 db_changes.fingerprint = self.fingerprint.fingerprint
1005 db_changes.changelog_id = changelog_id
1006 db_changes.closes = self.changes.closed_bugs
1009 self.transaction.session.add(db_changes)
1010 self.transaction.session.flush()
1011 except sqlalchemy.exc.IntegrityError:
1012 raise ArchiveException('{0} is already known.'.format(self.changes.filename))
1016 def _install_policy(self, policy_queue, target_suite, db_changes, db_source, db_binaries):
1017 u = PolicyQueueUpload()
1018 u.policy_queue = policy_queue
1019 u.target_suite = target_suite
1020 u.changes = db_changes
1021 u.source = db_source
1022 u.binaries = db_binaries
1023 self.transaction.session.add(u)
1024 self.transaction.session.flush()
1026 dst = os.path.join(policy_queue.path, self.changes.filename)
1027 self.transaction.fs.copy(self.changes.path, dst, mode=policy_queue.change_perms)
1031 def try_autobyhand(self):
1034 Try to handle byhand packages automatically.
1036 @rtype: list of L{daklib.upload.HashedFile}
1037 @return: list of remaining byhand files
1039 assert len(self.reject_reasons) == 0
1040 assert self.changes.valid_signature
1041 assert self.final_suites is not None
1042 assert self._checked
1044 byhand = self.changes.byhand_files
1045 if len(byhand) == 0:
1048 suites = list(self.final_suites)
1049 assert len(suites) == 1, "BYHAND uploads must be to a single suite"
1053 control = self.changes.changes
1054 automatic_byhand_packages = cnf.subtree("AutomaticByHandPackages")
1058 parts = f.filename.split('_', 2)
1060 print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
1064 package, version, archext = parts
1065 arch, ext = archext.split('.', 1)
1068 rule = automatic_byhand_packages.subtree(package)
1073 if rule['Source'] != self.changes.source_name or rule['Section'] != f.section or rule['Extension'] != ext:
1077 script = rule['Script']
1078 retcode = subprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False)
1080 print "W: error processing {0}.".format(f.filename)
1083 return len(remaining) == 0
1085 def _install_byhand(self, policy_queue_upload, hashed_file):
1086 """install byhand file
1088 @type policy_queue_upload: L{daklib.dbconn.PolicyQueueUpload}
1090 @type hashed_file: L{daklib.upload.HashedFile}
1092 fs = self.transaction.fs
1093 session = self.transaction.session
1094 policy_queue = policy_queue_upload.policy_queue
1096 byhand_file = PolicyQueueByhandFile()
1097 byhand_file.upload = policy_queue_upload
1098 byhand_file.filename = hashed_file.filename
1099 session.add(byhand_file)
1102 src = os.path.join(self.directory, hashed_file.filename)
1103 dst = os.path.join(policy_queue.path, hashed_file.filename)
1104 fs.copy(src, dst, mode=policy_queue.change_perms)
1108 def _do_bts_versiontracking(self):
1110 fs = self.transaction.fs
1112 btsdir = cnf.get('Dir::BTSVersionTrack')
1113 if btsdir is None or btsdir == '':
1116 base = os.path.join(btsdir, self.changes.filename[:-8])
1119 sourcedir = self.unpacked_source()
1120 if sourcedir is not None:
1121 fh = open(os.path.join(sourcedir, 'debian', 'changelog'), 'r')
1122 versions = fs.create("{0}.versions".format(base), mode=0o644)
1123 for line in fh.readlines():
1124 if re_changelog_versions.match(line):
1125 versions.write(line)
1129 # binary -> source mapping
1130 debinfo = fs.create("{0}.debinfo".format(base), mode=0o644)
1131 for binary in self.changes.binaries:
1132 control = binary.control
1133 source_package, source_version = binary.source
1134 line = " ".join([control['Package'], control['Version'], control['Architecture'], source_package, source_version])
1135 print >>debinfo, line
1138 def _policy_queue(self, suite):
1139 if suite.policy_queue is not None:
1140 return suite.policy_queue
1146 Install upload to a suite or policy queue. This method does B{not}
1147 handle uploads to NEW.
1149 You need to have called the C{check} method before calling this method.
1151 assert len(self.reject_reasons) == 0
1152 assert self.changes.valid_signature
1153 assert self.final_suites is not None
1154 assert self._checked
1157 db_changes = self._install_changes()
1159 for suite in self.final_suites:
1160 overridesuite = suite
1161 if suite.overridesuite is not None:
1162 overridesuite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
1164 policy_queue = self._policy_queue(suite)
1166 redirected_suite = suite
1167 if policy_queue is not None:
1168 redirected_suite = policy_queue.suite
1170 # source can be in the suite we install to or any suite we enhance
1171 source_suite_ids = set([suite.suite_id, redirected_suite.suite_id])
1172 for enhanced_suite_id, in self.session.query(VersionCheck.reference_id) \
1173 .filter(VersionCheck.suite_id.in_(source_suite_ids)) \
1174 .filter(VersionCheck.check == 'Enhances'):
1175 source_suite_ids.add(enhanced_suite_id)
1177 source_suites = self.session.query(Suite).filter(Suite.suite_id.in_(source_suite_ids)).subquery()
1179 source_component_func = lambda source: self._source_override(overridesuite, source).component
1180 binary_component_func = lambda binary: self._binary_component(overridesuite, binary)
1182 (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])
1184 if policy_queue is not None:
1185 self._install_policy(policy_queue, suite, db_changes, db_source, db_binaries)
1187 # copy to build queues
1188 if policy_queue is None or policy_queue.send_to_build_queues:
1189 for build_queue in suite.copy_queues:
1190 self._install_to_suite(build_queue.suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])
1192 self._do_bts_versiontracking()
1194 def install_to_new(self):
1195 """install upload to NEW
1197 Install upload to NEW. This method does B{not} handle regular uploads
1198 to suites or policy queues.
1200 You need to have called the C{check} method before calling this method.
1202 # Uploads to NEW are special as we don't have overrides.
1203 assert len(self.reject_reasons) == 0
1204 assert self.changes.valid_signature
1205 assert self.final_suites is not None
1207 source = self.changes.source
1208 binaries = self.changes.binaries
1209 byhand = self.changes.byhand_files
1211 # we need a suite to guess components
1212 suites = list(self.final_suites)
1213 assert len(suites) == 1, "NEW uploads must be to a single suite"
1216 # decide which NEW queue to use
1217 if suite.new_queue is None:
1218 new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one()
1220 new_queue = suite.new_queue
1222 # There is only one global BYHAND queue
1223 new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one()
1224 new_suite = new_queue.suite
1227 def binary_component_func(binary):
1228 return self._binary_component(suite, binary, only_overrides=False)
1230 # guess source component
1231 # XXX: should be moved into an extra method
1232 binary_component_names = set()
1233 for binary in binaries:
1234 component = binary_component_func(binary)
1235 binary_component_names.add(component.component_name)
1236 source_component_name = None
1237 for c in self.session.query(Component).order_by(Component.component_id):
1238 guess = c.component_name
1239 if guess in binary_component_names:
1240 source_component_name = guess
1242 if source_component_name is None:
1243 source_component = self.session.query(Component).order_by(Component.component_id).first()
1245 source_component = self.session.query(Component).filter_by(component_name=source_component_name).one()
1246 source_component_func = lambda source: source_component
1248 db_changes = self._install_changes()
1249 (db_source, db_binaries) = self._install_to_suite(new_suite, source_component_func, binary_component_func, source_suites=True, extra_source_archives=[suite.archive])
1250 policy_upload = self._install_policy(new_queue, suite, db_changes, db_source, db_binaries)
1253 self._install_byhand(policy_upload, f)
1255 self._do_bts_versiontracking()
1258 """commit changes"""
1259 self.transaction.commit()
1262 """rollback changes"""
1263 self.transaction.rollback()
1265 def __enter__(self):
1269 def __exit__(self, type, value, traceback):
1270 if self.directory is not None:
1271 shutil.rmtree(self.directory)
1272 self.directory = None
1274 self.transaction.rollback()