1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
3 # This program is free software; you can redistribute it and/or modify
4 # it under the terms of the GNU General Public License as published by
5 # the Free Software Foundation; either version 2 of the License, or
6 # (at your option) any later version.
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
13 # You should have received a copy of the GNU General Public License along
14 # with this program; if not, write to the Free Software Foundation, Inc.,
15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17 """module to manipulate the archive
19 This module provides classes to manipulate the archive.
23 import daklib.checks as checks
24 from daklib.config import Config
25 import daklib.upload as upload
26 import daklib.utils as utils
27 from .fstransactions import FilesystemTransaction
28 from .regexes import re_changelog_versions, re_bin_only_nmu
31 from datetime import datetime
35 from sqlalchemy.orm.exc import NoResultFound
39 class ArchiveException(Exception):
42 class HashMismatchException(ArchiveException):
45 class ArchiveTransaction(object):
46 """manipulate the archive in a transaction
49 self.fs = FilesystemTransaction()
50 self.session = DBConn().session()
52 def get_file(self, hashed_file, source_name):
53 """Look for file `hashed_file` in database
56 hashed_file (daklib.upload.HashedFile): file to look for in the database
59 KeyError: file was not found in the database
60 HashMismatchException: hash mismatch
63 `daklib.dbconn.PoolFile` object for the database
65 poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
67 poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
68 if poolfile.filesize != hashed_file.size or poolfile.md5sum != hashed_file.md5sum or poolfile.sha1sum != hashed_file.sha1sum or poolfile.sha256sum != hashed_file.sha256sum:
69 raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
72 raise KeyError('{0} not found in database.'.format(poolname))
74 def _install_file(self, directory, hashed_file, archive, component, source_name):
77 Will not give an error when the file is already present.
80 `daklib.dbconn.PoolFile` object for the new file
82 session = self.session
84 poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
86 poolfile = self.get_file(hashed_file, source_name)
88 poolfile = PoolFile(filename=poolname, filesize=hashed_file.size)
89 poolfile.md5sum = hashed_file.md5sum
90 poolfile.sha1sum = hashed_file.sha1sum
91 poolfile.sha256sum = hashed_file.sha256sum
96 session.query(ArchiveFile).filter_by(archive=archive, component=component, file=poolfile).one()
98 archive_file = ArchiveFile(archive, component, poolfile)
99 session.add(archive_file)
102 path = os.path.join(archive.path, 'pool', component.component_name, poolname)
103 hashed_file_path = os.path.join(directory, hashed_file.filename)
104 self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
108 def install_binary(self, directory, binary, suite, component, allow_tainted=False, fingerprint=None, source_suites=None, extra_source_archives=None):
109 """Install a binary package
112 directory (str): directory the binary package is located in
113 binary (daklib.upload.Binary): binary package to install
114 suite (daklib.dbconn.Suite): target suite
115 component (daklib.dbconn.Component): target component
118 allow_tainted (bool): allow to copy additional files from tainted archives
119 fingerprint (daklib.dbconn.Fingerprint): optional fingerprint
120 source_suites (list of daklib.dbconn.Suite or True): suites to copy
121 the source from if they are not in `suite` or True to allow
122 copying from any suite.
123 This can also be a SQLAlchemy (sub)query object.
124 extra_source_archives (list of daklib.dbconn.Archive): extra archives to copy Built-Using sources from
127 `daklib.dbconn.DBBinary` object for the new package
129 session = self.session
130 control = binary.control
131 maintainer = get_or_set_maintainer(control['Maintainer'], session)
132 architecture = get_architecture(control['Architecture'], session)
134 (source_name, source_version) = binary.source
135 source_query = session.query(DBSource).filter_by(source=source_name, version=source_version)
136 source = source_query.filter(DBSource.suites.contains(suite)).first()
138 if source_suites != True:
139 source_query = source_query.join(DBSource.suites) \
140 .filter(Suite.suite_id == source_suites.c.id)
141 source = source_query.first()
143 raise ArchiveException('{0}: trying to install to {1}, but could not find source'.format(binary.hashed_file.filename, suite.suite_name))
144 self.copy_source(source, suite, component)
146 db_file = self._install_file(directory, binary.hashed_file, suite.archive, component, source_name)
149 package=control['Package'],
150 version=control['Version'],
151 architecture=architecture,
155 maintainer=maintainer,
157 binarytype=binary.type,
158 fingerprint=fingerprint,
162 db_binary = session.query(DBBinary).filter_by(**unique).one()
163 for key, value in rest.iteritems():
164 if getattr(db_binary, key) != value:
165 raise ArchiveException('{0}: Does not match binary in database.'.format(binary.hashed_file.filename))
166 except NoResultFound:
167 db_binary = DBBinary(**unique)
168 for key, value in rest.iteritems():
169 setattr(db_binary, key, value)
170 session.add(db_binary)
172 import_metadata_into_db(db_binary, session)
174 self._add_built_using(db_binary, binary.hashed_file.filename, control, suite, extra_archives=extra_source_archives)
176 if suite not in db_binary.suites:
177 db_binary.suites.append(suite)
183 def _ensure_extra_source_exists(self, filename, source, archive, extra_archives=None):
184 """ensure source exists in the given archive
186 This is intended to be used to check that Built-Using sources exist.
189 filename (str): filename to use in error messages
190 source (daklib.dbconn.DBSource): source to look for
191 archive (daklib.dbconn.Archive): archive to look in
194 extra_archives (list of daklib.dbconn.Archive): list of archives to copy
195 the source package from if it is not yet present in `archive`
197 session = self.session
198 db_file = session.query(ArchiveFile).filter_by(file=source.poolfile, archive=archive).first()
199 if db_file is not None:
202 # Try to copy file from one extra archive
203 if extra_archives is None:
205 db_file = session.query(ArchiveFile).filter_by(file=source.poolfile).filter(ArchiveFile.archive_id.in_([ a.archive_id for a in extra_archives])).first()
207 raise ArchiveException('{0}: Built-Using refers to package {1} (= {2}) not in target archive {3}.'.format(filename, source.source, source.version, archive.archive_name))
209 source_archive = db_file.archive
210 for dsc_file in source.srcfiles:
211 af = session.query(ArchiveFile).filter_by(file=dsc_file.poolfile, archive=source_archive, component=db_file.component).one()
212 # We were given an explicit list of archives so it is okay to copy from tainted archives.
213 self._copy_file(af.file, archive, db_file.component, allow_tainted=True)
215 def _add_built_using(self, db_binary, filename, control, suite, extra_archives=None):
216 """Add Built-Using sources to `db_binary.extra_sources`
218 session = self.session
219 built_using = control.get('Built-Using', None)
221 if built_using is not None:
222 for dep in apt_pkg.parse_depends(built_using):
223 assert len(dep) == 1, 'Alternatives are not allowed in Built-Using field'
224 bu_source_name, bu_source_version, comp = dep[0]
225 assert comp == '=', 'Built-Using must contain strict dependencies'
227 bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first()
228 if bu_source is None:
229 raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version))
231 self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives)
233 db_binary.extra_sources.append(bu_source)
235 def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None):
236 """Install a source package
239 directory (str): directory the source package is located in
240 source (daklib.upload.Source): source package to install
241 suite (daklib.dbconn.Suite): target suite
242 component (daklib.dbconn.Component): target component
243 changed_by (daklib.dbconn.Maintainer): person who prepared this version of the package
246 allow_tainted (bool): allow to copy additional files from tainted archives
247 fingerprint (daklib.dbconn.Fingerprint): optional fingerprint
250 `daklib.dbconn.DBSource` object for the new source
252 session = self.session
253 archive = suite.archive
255 maintainer = get_or_set_maintainer(control['Maintainer'], session)
256 source_name = control['Source']
258 ### Add source package to database
260 # We need to install the .dsc first as the DBSource object refers to it.
261 db_file_dsc = self._install_file(directory, source._dsc_file, archive, component, source_name)
265 version=control['Version'],
268 maintainer=maintainer,
269 changedby=changed_by,
270 #install_date=datetime.now().date(),
271 poolfile=db_file_dsc,
272 fingerprint=fingerprint,
273 dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'),
278 db_source = session.query(DBSource).filter_by(**unique).one()
279 for key, value in rest.iteritems():
280 if getattr(db_source, key) != value:
281 raise ArchiveException('{0}: Does not match source in database.'.format(source._dsc_file.filename))
282 except NoResultFound:
284 db_source = DBSource(**unique)
285 for key, value in rest.iteritems():
286 setattr(db_source, key, value)
287 # XXX: set as default in postgres?
288 db_source.install_date = datetime.now().date()
289 session.add(db_source)
292 # Add .dsc file. Other files will be added later.
293 db_dsc_file = DSCFile()
294 db_dsc_file.source = db_source
295 db_dsc_file.poolfile = db_file_dsc
296 session.add(db_dsc_file)
299 if suite in db_source.suites:
302 db_source.suites.append(suite)
307 ### Now add remaining files and copy them to the archive.
309 for hashed_file in source.files.itervalues():
310 hashed_file_path = os.path.join(directory, hashed_file.filename)
311 if os.path.exists(hashed_file_path):
312 db_file = self._install_file(directory, hashed_file, archive, component, source_name)
315 db_file = self.get_file(hashed_file, source_name)
316 self._copy_file(db_file, archive, component, allow_tainted=allow_tainted)
318 db_dsc_file = DSCFile()
319 db_dsc_file.source = db_source
320 db_dsc_file.poolfile = db_file
321 session.add(db_dsc_file)
325 # Importing is safe as we only arrive here when we did not find the source already installed earlier.
326 import_metadata_into_db(db_source, session)
328 # Uploaders are the maintainer and co-maintainers from the Uploaders field
329 db_source.uploaders.append(maintainer)
330 if 'Uploaders' in control:
331 def split_uploaders(field):
333 for u in re.sub(">[ ]*,", ">\t", field).split("\t"):
336 for u in split_uploaders(control['Uploaders']):
337 db_source.uploaders.append(get_or_set_maintainer(u, session))
342 def _copy_file(self, db_file, archive, component, allow_tainted=False):
343 """Copy a file to the given archive and component
346 db_file (daklib.dbconn.PoolFile): file to copy
347 archive (daklib.dbconn.Archive): target archive
348 component (daklib.dbconn.Component): target component
351 allow_tainted (bool): allow to copy from tainted archives (such as NEW)
353 session = self.session
355 if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None:
356 query = session.query(ArchiveFile).filter_by(file=db_file, component=component)
357 if not allow_tainted:
358 query = query.join(Archive).filter(Archive.tainted == False)
360 source_af = query.first()
361 if source_af is None:
362 raise ArchiveException('cp: Could not find {0} in component {1} in any archive.'.format(db_file.filename, component.component_name))
363 target_af = ArchiveFile(archive, component, db_file)
364 session.add(target_af)
366 self.fs.copy(source_af.path, target_af.path, link=False, mode=archive.mode)
368 def copy_binary(self, db_binary, suite, component, allow_tainted=False, extra_archives=None):
369 """Copy a binary package to the given suite and component
372 db_binary (daklib.dbconn.DBBinary): binary to copy
373 suite (daklib.dbconn.Suite): target suite
374 component (daklib.dbconn.Component): target component
377 allow_tainted (bool): allow to copy from tainted archives (such as NEW)
378 extra_archives (list of daklib.dbconn.Archive): extra archives to copy Built-Using sources from
380 session = self.session
381 archive = suite.archive
385 filename = db_binary.poolfile.filename
387 # make sure source is present in target archive
388 db_source = db_binary.source
389 if session.query(ArchiveFile).filter_by(archive=archive, file=db_source.poolfile).first() is None:
390 raise ArchiveException('{0}: cannot copy to {1}: source is not present in target archive'.format(filename, suite.suite_name))
392 # make sure built-using packages are present in target archive
393 for db_source in db_binary.extra_sources:
394 self._ensure_extra_source_exists(filename, db_source, archive, extra_archives=extra_archives)
397 db_file = db_binary.poolfile
398 self._copy_file(db_file, suite.archive, component, allow_tainted=allow_tainted)
399 if suite not in db_binary.suites:
400 db_binary.suites.append(suite)
403 def copy_source(self, db_source, suite, component, allow_tainted=False):
404 """Copy a source package to the given suite and component
407 db_source (daklib.dbconn.DBSource): source to copy
408 suite (daklib.dbconn.Suite): target suite
409 component (daklib.dbconn.Component): target component
412 allow_tainted (bool): allow to copy from tainted archives (such as NEW)
414 archive = suite.archive
417 for db_dsc_file in db_source.srcfiles:
418 self._copy_file(db_dsc_file.poolfile, archive, component, allow_tainted=allow_tainted)
419 if suite not in db_source.suites:
420 db_source.suites.append(suite)
423 def remove_file(self, db_file, archive, component):
424 """Remove a file from a given archive and component
427 db_file (daklib.dbconn.PoolFile): file to remove
428 archive (daklib.dbconn.Archive): archive to remove the file from
429 component (daklib.dbconn.Component): component to remove the file from
431 af = self.session.query(ArchiveFile).filter_by(file=db_file, archive=archive, component=component)
432 self.fs.unlink(af.path)
433 self.session.delete(af)
435 def remove_binary(self, binary, suite):
436 """Remove a binary from a given suite and component
439 binary (daklib.dbconn.DBBinary): binary to remove
440 suite (daklib.dbconn.Suite): suite to remove the package from
442 binary.suites.remove(suite)
445 def remove_source(self, source, suite):
446 """Remove a source from a given suite and component
449 ArchiveException: source package is still referenced by other
450 binaries in the suite
453 binary (daklib.dbconn.DBSource): source to remove
454 suite (daklib.dbconn.Suite): suite to remove the package from
456 session = self.session
458 query = session.query(DBBinary).filter_by(source=source) \
459 .filter(DBBinary.suites.contains(suite))
460 if query.first() is not None:
461 raise ArchiveException('src:{0} is still used by binaries in suite {1}'.format(source.source, suite.suite_name))
463 source.suites.remove(suite)
469 self.session.commit()
472 self.session.rollback()
476 """rollback changes"""
477 self.session.rollback()
483 def __exit__(self, type, value, traceback):
490 class ArchiveUpload(object):
493 This class can be used in a with-statement:
495 with ArchiveUpload(...) as upload:
498 Doing so will automatically run any required cleanup and also rollback the
499 transaction if it was not committed.
502 changes (daklib.upload.Changes): upload to process
503 directory (str): directory with temporary copy of files. set by `prepare`
504 fingerprint (daklib.dbconn.Fingerprint): fingerprint used to sign the upload
505 new (bool): upload is NEW. set by `check`
506 reject_reasons (list of str): reasons why the upload cannot be accepted
507 session: database session
508 transaction (daklib.archive.ArchiveTransaction): transaction used to handle the upload
509 warnings (list of str): warnings (NOT USED YET)
511 def __init__(self, directory, changes, keyrings):
512 self.transaction = ArchiveTransaction()
513 self.session = self.transaction.session
515 self.original_directory = directory
516 self.original_changes = changes
518 self.directory = None
519 self.keyrings = keyrings
521 self.fingerprint = self.session.query(Fingerprint).filter_by(fingerprint=changes.primary_fingerprint).one()
523 self.reject_reasons = []
525 self.final_suites = None
528 self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one()
529 self._new = self._new_queue.suite
532 """prepare upload for further processing
534 This copies the files involved to a temporary directory. If you use
535 this method directly, you have to remove the directory given by the
536 `directory` attribute later on your own.
538 Instead of using the method directly, you can also use a with-statement:
540 with ArchiveUpload(...) as upload:
543 This will automatically handle any required cleanup.
545 assert self.directory is None
546 assert self.original_changes.valid_signature
549 session = self.transaction.session
551 self.directory = tempfile.mkdtemp(dir=cnf.get('Dir::TempPath'))
552 with FilesystemTransaction() as fs:
553 src = os.path.join(self.original_directory, self.original_changes.filename)
554 dst = os.path.join(self.directory, self.original_changes.filename)
557 self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings)
559 for f in self.changes.files.itervalues():
560 src = os.path.join(self.original_directory, f.filename)
561 dst = os.path.join(self.directory, f.filename)
564 source = self.changes.source
565 if source is not None:
566 for f in source.files.itervalues():
567 src = os.path.join(self.original_directory, f.filename)
568 dst = os.path.join(self.directory, f.filename)
569 if f.filename not in self.changes.files:
570 db_file = self.transaction.get_file(f, source.dsc['Source'])
571 db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first()
572 fs.copy(db_archive_file.path, dst, symlink=True)
574 def unpacked_source(self):
575 """Path to unpacked source
577 Get path to the unpacked source. This method does unpack the source
578 into a temporary directory under `self.directory` if it has not
579 been done so already.
582 String giving the path to the unpacked source directory
583 or None if no source was included in the upload.
585 assert self.directory is not None
587 source = self.changes.source
590 dsc_path = os.path.join(self.directory, source._dsc_file.filename)
592 sourcedir = os.path.join(self.directory, 'source')
593 if not os.path.exists(sourcedir):
594 subprocess.check_call(["dpkg-source", "--no-copy", "-x", dsc_path, sourcedir], shell=False)
595 if not os.path.isdir(sourcedir):
596 raise Exception("{0} is not a directory after extracting source package".format(sourcedir))
599 def _map_suite(self, suite_name):
600 for rule in Config().value_list("SuiteMappings"):
601 fields = rule.split()
603 if rtype == "map" or rtype == "silent-map":
604 (src, dst) = fields[1:3]
605 if src == suite_name:
607 if rtype != "silent-map":
608 self.warnings.append('Mapping {0} to {0}.'.format(src, dst))
609 elif rtype == "ignore":
611 if suite_name == ignored:
612 self.warnings.append('Ignoring target suite {0}.'.format(ignored))
614 elif rtype == "reject":
616 if suite_name == rejected:
617 self.reject_reasons.append('Uploads to {0} are not accepted.'.format(suite))
618 ## XXX: propup-version and map-unreleased not yet implemented
621 def _mapped_suites(self):
622 """Get target suites after mappings
625 list of daklib.dbconn.Suite giving the mapped target suites of this upload
627 session = self.session
630 for dist in self.changes.distributions:
631 suite_name = self._map_suite(dist)
632 if suite_name is not None:
633 suite_names.append(suite_name)
635 suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
638 def _mapped_component(self, component_name):
639 """get component after mappings
641 Evaluate component mappings from ComponentMappings in dak.conf for the
642 given component name.
644 NOTE: ansgar wants to get rid of this. It's currently only used for
648 component_name (str): component name
651 `daklib.dbconn.Component` object
654 for m in cnf.value_list("ComponentMappings"):
655 (src, dst) = m.split()
656 if component_name == src:
658 component = self.session.query(Component).filter_by(component_name=component_name).one()
661 def _check_new(self, suite):
662 """Check if upload is NEW
664 An upload is NEW if it has binary or source packages that do not have
665 an override in `suite` OR if it references files ONLY in a tainted
666 archive (eg. when it references files in NEW).
669 True if the upload is NEW, False otherwise
671 session = self.session
673 # Check for missing overrides
674 for b in self.changes.binaries:
675 override = self._binary_override(suite, b)
679 if self.changes.source is not None:
680 override = self._source_override(suite, self.changes.source)
684 # Check if we reference a file only in a tainted archive
685 files = self.changes.files.values()
686 if self.changes.source is not None:
687 files.extend(self.changes.source.files.values())
689 query = session.query(ArchiveFile).join(PoolFile).filter(PoolFile.sha1sum == f.sha1sum)
690 query_untainted = query.join(Archive).filter(Archive.tainted == False)
692 in_archive = (query.first() is not None)
693 in_untainted_archive = (query_untainted.first() is not None)
695 if in_archive and not in_untainted_archive:
698 def _final_suites(self):
699 session = self.session
701 mapped_suites = self._mapped_suites()
704 for suite in mapped_suites:
705 overridesuite = suite
706 if suite.overridesuite is not None:
707 overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
708 if self._check_new(overridesuite):
710 final_suites.add(suite)
714 def _binary_override(self, suite, binary):
715 """Get override entry for a binary
718 suite (daklib.dbconn.Suite)
719 binary (daklib.upload.Binary)
722 daklib.dbconn.Override or None
724 if suite.overridesuite is not None:
725 suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
727 query = self.session.query(Override).filter_by(suite=suite, package=binary.control['Package']) \
728 .join(Component).filter(Component.component_name == binary.component) \
729 .join(OverrideType).filter(OverrideType.overridetype == binary.type)
733 except NoResultFound:
736 def _source_override(self, suite, source):
737 """Get override entry for a source
740 suite (daklib.dbconn.Suite)
741 source (daklib.upload.Source)
744 daklib.dbconn.Override or None
746 if suite.overridesuite is not None:
747 suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
749 # XXX: component for source?
750 query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \
751 .join(OverrideType).filter(OverrideType.overridetype == 'dsc')
755 except NoResultFound:
758 def _binary_component(self, suite, binary, only_overrides=True):
759 """get component for a binary
761 By default this will only look at overrides to get the right component;
762 if `only_overrides` is False this method will also look at the Section field.
765 suite (daklib.dbconn.Suite)
766 binary (daklib.upload.Binary)
769 only_overrides (bool): only use overrides to get the right component.
773 `daklib.dbconn.Component` object or None
775 override = self._binary_override(suite, binary)
776 if override is not None:
777 return override.component
780 return self._mapped_component(binary.component)
782 def check(self, force=False):
783 """run checks against the upload
786 force (bool): ignore failing forcable checks
789 True if all checks passed, False otherwise
791 # XXX: needs to be better structured.
792 assert self.changes.valid_signature
796 checks.SignatureCheck,
798 checks.TransitionCheck,
799 checks.UploadBlockCheck,
803 checks.BinaryTimestampCheck,
805 checks.SingleDistributionCheck,
806 checks.NoSourceOnlyCheck,
811 final_suites = self._final_suites()
812 if len(final_suites) == 0:
813 self.reject_reasons.append('Ended with no suite to install to.')
817 checks.SourceFormatCheck,
818 checks.SuiteArchitectureCheck,
821 for suite in final_suites:
822 chk().per_suite_check(self, suite)
824 if len(self.reject_reasons) != 0:
827 self.final_suites = final_suites
829 except checks.Reject as e:
830 self.reject_reasons.append(unicode(e))
831 except Exception as e:
832 self.reject_reasons.append("Processing raised an exception: {0}.\n{1}".format(e, traceback.format_exc()))
835 def _install_to_suite(self, suite, source_component_func, binary_component_func, source_suites=None, extra_source_archives=None):
836 """Install upload to the given suite
839 suite (daklib.dbconn.Suite): suite to install the package into.
840 This is the real suite, ie. after any redirection to NEW or a policy queue
841 source_component_func: function to get the `daklib.dbconn.Component`
842 for a `daklib.upload.Source` object
843 binary_component_func: function to get the `daklib.dbconn.Component`
844 for a `daklib.upload.Binary` object
847 source_suites: see `daklib.archive.ArchiveTransaction.install_binary`
848 extra_source_archives: see `daklib.archive.ArchiveTransaction.install_binary`
851 tuple with two elements. The first is a `daklib.dbconn.DBSource`
852 object for the install source or None if no source was included.
853 The second is a list of `daklib.dbconn.DBBinary` objects for the
854 installed binary packages.
856 # XXX: move this function to ArchiveTransaction?
858 control = self.changes.changes
859 changed_by = get_or_set_maintainer(control.get('Changed-By', control['Maintainer']), self.session)
861 if source_suites is None:
862 source_suites = self.session.query(Suite).join((VersionCheck, VersionCheck.reference_id == Suite.suite_id)).filter(VersionCheck.suite == suite).subquery()
864 source = self.changes.source
865 if source is not None:
866 component = source_component_func(source)
867 db_source = self.transaction.install_source(self.directory, source, suite, component, changed_by, fingerprint=self.fingerprint)
872 for binary in self.changes.binaries:
873 component = binary_component_func(binary)
874 db_binary = self.transaction.install_binary(self.directory, binary, suite, component, fingerprint=self.fingerprint, source_suites=source_suites, extra_source_archives=extra_source_archives)
875 db_binaries.append(db_binary)
877 if suite.copychanges:
878 src = os.path.join(self.directory, self.changes.filename)
879 dst = os.path.join(suite.archive.path, 'dists', suite.suite_name, self.changes.filename)
880 self.transaction.fs.copy(src, dst)
882 return (db_source, db_binaries)
884 def _install_changes(self):
885 assert self.changes.valid_signature
886 control = self.changes.changes
887 session = self.transaction.session
891 # Only add changelog for sourceful uploads and binNMUs
892 if 'source' in self.changes.architectures or re_bin_only_nmu.search(control['Version']):
893 query = 'INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id'
894 changelog_id = session.execute(query, {'changelog': control['Changes']}).scalar()
895 assert changelog_id is not None
897 db_changes = DBChange()
898 db_changes.changesname = self.changes.filename
899 db_changes.source = control['Source']
900 db_changes.binaries = control.get('Binary', None)
901 db_changes.architecture = control['Architecture']
902 db_changes.version = control['Version']
903 db_changes.distribution = control['Distribution']
904 db_changes.urgency = control['Urgency']
905 db_changes.maintainer = control['Maintainer']
906 db_changes.changedby = control.get('Changed-By', control['Maintainer'])
907 db_changes.date = control['Date']
908 db_changes.fingerprint = self.fingerprint.fingerprint
909 db_changes.changelog_id = changelog_id
910 db_changes.closes = self.changes.closed_bugs
912 self.transaction.session.add(db_changes)
913 self.transaction.session.flush()
917 def _install_policy(self, policy_queue, target_suite, db_changes, db_source, db_binaries):
918 u = PolicyQueueUpload()
919 u.policy_queue = policy_queue
920 u.target_suite = target_suite
921 u.changes = db_changes
923 u.binaries = db_binaries
924 self.transaction.session.add(u)
925 self.transaction.session.flush()
927 dst = os.path.join(policy_queue.path, self.changes.filename)
928 self.transaction.fs.copy(self.changes.path, dst)
932 def try_autobyhand(self):
935 Try to handle byhand packages automatically.
938 list of `daklib.upload.hashed_file` for the remaining byhand packages
940 assert len(self.reject_reasons) == 0
941 assert self.changes.valid_signature
942 assert self.final_suites is not None
944 byhand = self.changes.byhand_files
948 suites = list(self.final_suites)
949 assert len(suites) == 1, "BYHAND uploads must be to a single suite"
953 control = self.changes.changes
954 automatic_byhand_packages = cnf.subtree("AutomaticByHandPackages")
958 parts = f.filename.split('_', 2)
960 print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
964 package, version, archext = parts
965 arch, ext = archext.split('.', 1)
967 rule = automatic_byhand_packages.get(package)
972 if rule['Source'] != control['Source'] or rule['Section'] != f.section or rule['Extension'] != ext:
976 script = rule['Script']
977 retcode = subprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False)
979 print "W: error processing {0}.".format(f.filename)
982 return len(remaining) == 0
984 def _install_byhand(self, policy_queue_upload, hashed_file):
987 policy_queue_upload (daklib.dbconn.PolicyQueueUpload): XXX
988 hashed_file (daklib.upload.HashedFile): XXX
990 fs = self.transaction.fs
991 session = self.transaction.session
992 policy_queue = policy_queue_upload.policy_queue
994 byhand_file = PolicyQueueByhandFile()
995 byhand_file.upload = policy_queue_upload
996 byhand_file.filename = hashed_file.filename
997 session.add(byhand_file)
1000 src = os.path.join(self.directory, hashed_file.filename)
1001 dst = os.path.join(policy_queue.path, hashed_file.filename)
1006 def _do_bts_versiontracking(self):
1008 fs = self.transaction.fs
1010 btsdir = cnf.get('Dir::BTSVersionTrack')
1011 if btsdir is None or btsdir == '':
1014 base = os.path.join(btsdir, self.changes.filename[:-8])
1017 sourcedir = self.unpacked_source()
1018 if sourcedir is not None:
1019 fh = open(os.path.join(sourcedir, 'debian', 'changelog'), 'r')
1020 versions = fs.create("{0}.versions".format(base), mode=0o644)
1021 for line in fh.readlines():
1022 if re_changelog_versions.match(line):
1023 versions.write(line)
1027 # binary -> source mapping
1028 debinfo = fs.create("{0}.debinfo".format(base), mode=0o644)
1029 for binary in self.changes.binaries:
1030 control = binary.control
1031 source_package, source_version = binary.source
1032 line = " ".join([control['Package'], control['Version'], source_package, source_version])
1033 print >>debinfo, line
1036 def _policy_queue(self, suite):
1037 if suite.policy_queue is not None:
1038 return suite.policy_queue
1044 Install upload to a suite or policy queue. This method does *not*
1045 handle uploads to NEW.
1047 You need to have called the `check` method before calling this method.
1049 assert len(self.reject_reasons) == 0
1050 assert self.changes.valid_signature
1051 assert self.final_suites is not None
1054 db_changes = self._install_changes()
1056 for suite in self.final_suites:
1057 overridesuite = suite
1058 if suite.overridesuite is not None:
1059 overridesuite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
1061 policy_queue = self._policy_queue(suite)
1063 redirected_suite = suite
1064 if policy_queue is not None:
1065 redirected_suite = policy_queue.suite
1067 source_component_func = lambda source: self._source_override(overridesuite, source).component
1068 binary_component_func = lambda binary: self._binary_component(overridesuite, binary)
1070 (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, extra_source_archives=[suite.archive])
1072 if policy_queue is not None:
1073 self._install_policy(policy_queue, suite, db_changes, db_source, db_binaries)
1075 # copy to build queues
1076 if policy_queue is None or policy_queue.send_to_build_queues:
1077 for build_queue in suite.copy_queues:
1078 self._install_to_suite(build_queue.suite, source_component_func, binary_component_func, extra_source_archives=[suite.archive])
1080 self._do_bts_versiontracking()
1082 def install_to_new(self):
1083 """install upload to NEW
1085 Install upload to NEW. This method does *not* handle regular uploads
1086 to suites or policy queues.
1088 You need to have called the `check` method before calling this method.
1090 # Uploads to NEW are special as we don't have overrides.
1091 assert len(self.reject_reasons) == 0
1092 assert self.changes.valid_signature
1093 assert self.final_suites is not None
1095 source = self.changes.source
1096 binaries = self.changes.binaries
1097 byhand = self.changes.byhand_files
1099 new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one()
1101 new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one()
1102 new_suite = new_queue.suite
1104 # we need a suite to guess components
1105 suites = list(self.final_suites)
1106 assert len(suites) == 1, "NEW uploads must be to a single suite"
1109 def binary_component_func(binary):
1110 return self._binary_component(suite, binary, only_overrides=False)
1112 # guess source component
1113 # XXX: should be moved into an extra method
1114 binary_component_names = set()
1115 for binary in binaries:
1116 component = binary_component_func(binary)
1117 binary_component_names.add(component.component_name)
1118 source_component_name = None
1119 for c in self.session.query(Component).order_by(Component.component_id):
1120 guess = c.component_name
1121 if guess in binary_component_names:
1122 source_component_name = guess
1124 if source_component_name is None:
1125 raise Exception('Could not guess source component.')
1126 source_component = self.session.query(Component).filter_by(component_name=source_component_name).one()
1127 source_component_func = lambda source: source_component
1129 db_changes = self._install_changes()
1130 (db_source, db_binaries) = self._install_to_suite(new_suite, source_component_func, binary_component_func, source_suites=True, extra_source_archives=[suite.archive])
1131 policy_upload = self._install_policy(new_queue, suite, db_changes, db_source, db_binaries)
1134 self._install_byhand(policy_upload, f)
1136 self._do_bts_versiontracking()
1139 """commit changes"""
1140 self.transaction.commit()
1143 """rollback changes"""
1144 self.transaction.rollback()
1146 def __enter__(self):
1150 def __exit__(self, type, value, traceback):
1151 if self.directory is not None:
1152 shutil.rmtree(self.directory)
1153 self.directory = None
1155 self.transaction.rollback()