import daklib.utils as utils
from daklib.fstransactions import FilesystemTransaction
from daklib.regexes import re_changelog_versions, re_bin_only_nmu
+import daklib.daksubprocess
import apt_pkg
from datetime import datetime
import os
import shutil
-import subprocess
from sqlalchemy.orm.exc import NoResultFound
+import sqlalchemy.exc
import tempfile
import traceback
self.fs = FilesystemTransaction()
self.session = DBConn().session()
- def get_file(self, hashed_file, source_name):
+ def get_file(self, hashed_file, source_name, check_hashes=True):
"""Look for file C{hashed_file} in database
@type hashed_file: L{daklib.upload.HashedFile}
@param hashed_file: file to look for in the database
+ @type source_name: str
+ @param source_name: source package name
+
+ @type check_hashes: bool
+ @param check_hashes: check size and hashes match
+
@raise KeyError: file was not found in the database
@raise HashMismatchException: hash mismatch
poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
try:
poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
- if poolfile.filesize != hashed_file.size or poolfile.md5sum != hashed_file.md5sum or poolfile.sha1sum != hashed_file.sha1sum or poolfile.sha256sum != hashed_file.sha256sum:
+ if check_hashes and (poolfile.filesize != hashed_file.size
+ or poolfile.md5sum != hashed_file.md5sum
+ or poolfile.sha1sum != hashed_file.sha1sum
+ or poolfile.sha256sum != hashed_file.sha256sum):
raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
return poolfile
except NoResultFound:
Will not give an error when the file is already present.
@rtype: L{daklib.dbconn.PoolFile}
- @return: batabase object for the new file
+ @return: database object for the new file
"""
session = self.session
db_source.suites.append(suite)
if not created:
+ for f in db_source.srcfiles:
+ self._copy_file(f.poolfile, archive, component, allow_tainted=allow_tainted)
return db_source
### Now add remaining files and copy them to the archive.
# Uploaders are the maintainer and co-maintainers from the Uploaders field
db_source.uploaders.append(maintainer)
if 'Uploaders' in control:
- def split_uploaders(field):
- import re
- for u in re.sub(">[ ]*,", ">\t", field).split("\t"):
- yield u.strip()
-
+ from daklib.textutils import split_uploaders
for u in split_uploaders(control['Uploaders']):
db_source.uploaders.append(get_or_set_maintainer(u, session))
session.flush()
session = self.session
if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None:
- query = session.query(ArchiveFile).filter_by(file=db_file, component=component)
+ query = session.query(ArchiveFile).filter_by(file=db_file)
if not allow_tainted:
query = query.join(Archive).filter(Archive.tainted == False)
source_af = query.first()
if source_af is None:
- raise ArchiveException('cp: Could not find {0} in component {1} in any archive.'.format(db_file.filename, component.component_name))
+ raise ArchiveException('cp: Could not find {0} in any archive.'.format(db_file.filename))
target_af = ArchiveFile(archive, component, db_file)
session.add(target_af)
session.flush()
@type: bool
"""
+ self._checked = False
+ """checks passes. set by C{check}
+ @type: bool
+ """
+
self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one()
self._new = self._new_queue.suite
+ def warn(self, message):
+ """add a warning message
+
+ Adds a warning message that can later be seen in C{self.warnings}
+
+ @type message: string
+ @param message: warning message
+ """
+ self.warnings.append(message)
+
def prepare(self):
"""prepare upload for further processing
cnf = Config()
session = self.transaction.session
- self.directory = tempfile.mkdtemp(dir=cnf.get('Dir::TempPath'))
+ group = cnf.get('Dinstall::UnprivGroup') or None
+ self.directory = utils.temp_dirname(parent=cnf.get('Dir::TempPath'),
+ mode=0o2750, group=group)
with FilesystemTransaction() as fs:
src = os.path.join(self.original_directory, self.original_changes.filename)
dst = os.path.join(self.directory, self.original_changes.filename)
- fs.copy(src, dst)
+ fs.copy(src, dst, mode=0o640)
self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings)
dst = os.path.join(self.directory, f.filename)
if not os.path.exists(src):
continue
- fs.copy(src, dst)
+ fs.copy(src, dst, mode=0o640)
+
+ source = None
+ try:
+ source = self.changes.source
+ except Exception:
+ # Do not raise an exception here if the .dsc is invalid.
+ pass
- source = self.changes.source
if source is not None:
for f in source.files.itervalues():
src = os.path.join(self.original_directory, f.filename)
dst = os.path.join(self.directory, f.filename)
if not os.path.exists(dst):
try:
- db_file = self.transaction.get_file(f, source.dsc['Source'])
+ db_file = self.transaction.get_file(f, source.dsc['Source'], check_hashes=False)
db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first()
- fs.copy(db_archive_file.path, dst, symlink=True)
+ fs.copy(db_archive_file.path, dst, mode=0o640)
except KeyError:
# Ignore if get_file could not find it. Upload will
# probably be rejected later.
sourcedir = os.path.join(self.directory, 'source')
if not os.path.exists(sourcedir):
- subprocess.check_call(["dpkg-source", "--no-copy", "-x", dsc_path, sourcedir], shell=False)
+ devnull = open('/dev/null', 'w')
+ daklib.daksubprocess.check_call(["dpkg-source", "--no-copy", "--no-check", "-x", dsc_path, sourcedir], shell=False, stdout=devnull)
if not os.path.isdir(sourcedir):
raise Exception("{0} is not a directory after extracting source package".format(sourcedir))
return sourcedir
if src == suite_name:
suite_name = dst
if rtype != "silent-map":
- self.warnings.append('Mapping {0} to {0}.'.format(src, dst))
+ self.warnings.append('Mapping {0} to {1}.'.format(src, dst))
elif rtype == "ignore":
ignored = fields[1]
if suite_name == ignored:
elif rtype == "reject":
rejected = fields[1]
if suite_name == rejected:
- self.reject_reasons.append('Uploads to {0} are not accepted.'.format(suite))
+ raise checks.Reject('Uploads to {0} are not accepted.'.format(rejected))
## XXX: propup-version and map-unreleased not yet implemented
return suite_name
suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
return suites
+ def _check_new_binary_overrides(self, suite):
+ new = False
+
+ binaries = self.changes.binaries
+ source = self.changes.source
+ if source is not None and not source.package_list.fallback:
+ packages = source.package_list.packages_for_suite(suite)
+ binaries = [ entry for entry in packages ]
+
+ for b in binaries:
+ override = self._binary_override(suite, b)
+ if override is None:
+ self.warnings.append('binary:{0} is NEW.'.format(b.name))
+ new = True
+
+ return new
+
def _check_new(self, suite):
"""Check if upload is NEW
@return: C{True} if the upload is NEW, C{False} otherwise
"""
session = self.session
+ new = False
# Check for missing overrides
- for b in self.changes.binaries:
- override = self._binary_override(suite, b)
- if override is None:
- return True
-
+ if self._check_new_binary_overrides(suite):
+ new = True
if self.changes.source is not None:
override = self._source_override(suite, self.changes.source)
if override is None:
- return True
+ self.warnings.append('source:{0} is NEW.'.format(self.changes.source.dsc['Source']))
+ new = True
# Check if we reference a file only in a tainted archive
files = self.changes.files.values()
in_untainted_archive = (query_untainted.first() is not None)
if in_archive and not in_untainted_archive:
- return True
+ self.warnings.append('{0} is only available in NEW.'.format(f.filename))
+ new = True
+
+ return new
def _final_suites(self):
session = self.session
@type suite: L{daklib.dbconn.Suite}
@param suite: suite to get override for
- @type binary: L{daklib.upload.Binary}
+ @type binary: L{daklib.upload.Binary} or L{daklib.packagelist.PackageListEntry}
@param binary: binary to get override for
@rtype: L{daklib.dbconn.Override} or C{None}
if suite.overridesuite is not None:
suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
- query = self.session.query(Override).filter_by(suite=suite, package=binary.control['Package']) \
- .join(Component).filter(Component.component_name == binary.component) \
+ mapped_component = get_mapped_component(binary.component)
+ if mapped_component is None:
+ return None
+
+ query = self.session.query(Override).filter_by(suite=suite, package=binary.name) \
+ .join(Component).filter(Component.component_name == mapped_component.component_name) \
.join(OverrideType).filter(OverrideType.overridetype == binary.type)
try:
assert self.changes.valid_signature
try:
+ # Validate signatures and hashes before we do any real work:
for chk in (
- checks.SignatureCheck,
+ checks.SignatureAndHashesCheck,
checks.ChangesCheck,
- checks.TransitionCheck,
- checks.UploadBlockCheck,
- checks.HashesCheck,
+ checks.ExternalHashesCheck,
checks.SourceCheck,
checks.BinaryCheck,
checks.BinaryTimestampCheck,
- checks.ACLCheck,
checks.SingleDistributionCheck,
- checks.NoSourceOnlyCheck,
- checks.LintianCheck,
):
chk().check(self)
final_suites = self._final_suites()
if len(final_suites) == 0:
- self.reject_reasons.append('Ended with no suite to install to.')
+ self.reject_reasons.append('No target suite found. Please check your target distribution and that you uploaded to the right archive.')
return False
+ self.final_suites = final_suites
+
for chk in (
+ checks.TransitionCheck,
+ checks.ACLCheck,
+ checks.NoSourceOnlyCheck,
+ checks.LintianCheck,
+ ):
+ chk().check(self)
+
+ for chk in (
+ checks.ACLCheck,
checks.SourceFormatCheck,
checks.SuiteArchitectureCheck,
checks.VersionCheck,
if len(self.reject_reasons) != 0:
return False
- self.final_suites = final_suites
+ self._checked = True
return True
except checks.Reject as e:
self.reject_reasons.append(unicode(e))
if suite.copychanges:
src = os.path.join(self.directory, self.changes.filename)
dst = os.path.join(suite.archive.path, 'dists', suite.suite_name, self.changes.filename)
- self.transaction.fs.copy(src, dst)
+ self.transaction.fs.copy(src, dst, mode=suite.archive.mode)
return (db_source, db_binaries)
db_changes.changelog_id = changelog_id
db_changes.closes = self.changes.closed_bugs
- self.transaction.session.add(db_changes)
- self.transaction.session.flush()
+ try:
+ self.transaction.session.add(db_changes)
+ self.transaction.session.flush()
+ except sqlalchemy.exc.IntegrityError:
+ raise ArchiveException('{0} is already known.'.format(self.changes.filename))
return db_changes
self.transaction.session.flush()
dst = os.path.join(policy_queue.path, self.changes.filename)
- self.transaction.fs.copy(self.changes.path, dst)
+ self.transaction.fs.copy(self.changes.path, dst, mode=policy_queue.change_perms)
return u
assert len(self.reject_reasons) == 0
assert self.changes.valid_signature
assert self.final_suites is not None
+ assert self._checked
byhand = self.changes.byhand_files
if len(byhand) == 0:
remaining = []
for f in byhand:
- parts = f.filename.split('_', 2)
- if len(parts) != 3:
- print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
- remaining.append(f)
- continue
+ if '_' in f.filename:
+ parts = f.filename.split('_', 2)
+ if len(parts) != 3:
+ print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
+ remaining.append(f)
+ continue
- package, version, archext = parts
- arch, ext = archext.split('.', 1)
+ package, version, archext = parts
+ arch, ext = archext.split('.', 1)
+ else:
+ parts = f.filename.split('.')
+ if len(parts) < 2:
+ print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
+ remaining.append(f)
+ continue
+
+ package = parts[0]
+ version = '0'
+ arch = 'all'
+ ext = parts[-1]
- rule = automatic_byhand_packages.get(package)
- if rule is None:
+ try:
+ rule = automatic_byhand_packages.subtree(package)
+ except KeyError:
remaining.append(f)
continue
- if rule['Source'] != control['Source'] or rule['Section'] != f.section or rule['Extension'] != ext:
+ if rule['Source'] != self.changes.source_name \
+ or rule['Section'] != f.section \
+ or ('Extension' in rule and rule['Extension'] != ext):
remaining.append(f)
continue
script = rule['Script']
- retcode = subprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False)
+ retcode = daklib.daksubprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename)], shell=False)
if retcode != 0:
print "W: error processing {0}.".format(f.filename)
remaining.append(f)
src = os.path.join(self.directory, hashed_file.filename)
dst = os.path.join(policy_queue.path, hashed_file.filename)
- fs.copy(src, dst)
+ fs.copy(src, dst, mode=policy_queue.change_perms)
return byhand_file
for binary in self.changes.binaries:
control = binary.control
source_package, source_version = binary.source
- line = " ".join([control['Package'], control['Version'], source_package, source_version])
+ line = " ".join([control['Package'], control['Version'], control['Architecture'], source_package, source_version])
print >>debinfo, line
debinfo.close()
assert len(self.reject_reasons) == 0
assert self.changes.valid_signature
assert self.final_suites is not None
+ assert self._checked
assert not self.new
db_changes = self._install_changes()
if policy_queue is not None:
redirected_suite = policy_queue.suite
+ # source can be in the suite we install to or any suite we enhance
+ source_suite_ids = set([suite.suite_id, redirected_suite.suite_id])
+ for enhanced_suite_id, in self.session.query(VersionCheck.reference_id) \
+ .filter(VersionCheck.suite_id.in_(source_suite_ids)) \
+ .filter(VersionCheck.check == 'Enhances'):
+ source_suite_ids.add(enhanced_suite_id)
+
+ source_suites = self.session.query(Suite).filter(Suite.suite_id.in_(source_suite_ids)).subquery()
+
source_component_func = lambda source: self._source_override(overridesuite, source).component
binary_component_func = lambda binary: self._binary_component(overridesuite, binary)
- (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, extra_source_archives=[suite.archive])
+ (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])
if policy_queue is not None:
self._install_policy(policy_queue, suite, db_changes, db_source, db_binaries)
# copy to build queues
if policy_queue is None or policy_queue.send_to_build_queues:
- source_suites = self.session.query(Suite).filter_by(suite_id=suite.suite_id).subquery()
for build_queue in suite.copy_queues:
self._install_to_suite(build_queue.suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])
binaries = self.changes.binaries
byhand = self.changes.byhand_files
- new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one()
- if len(byhand) > 0:
- new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one()
- new_suite = new_queue.suite
-
# we need a suite to guess components
suites = list(self.final_suites)
assert len(suites) == 1, "NEW uploads must be to a single suite"
suite = suites[0]
+ # decide which NEW queue to use
+ if suite.new_queue is None:
+ new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one()
+ else:
+ new_queue = suite.new_queue
+ if len(byhand) > 0:
+ # There is only one global BYHAND queue
+ new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one()
+ new_suite = new_queue.suite
+
+
def binary_component_func(binary):
return self._binary_component(suite, binary, only_overrides=False)
source_component_name = guess
break
if source_component_name is None:
- raise Exception('Could not guess source component.')
- source_component = self.session.query(Component).filter_by(component_name=source_component_name).one()
+ source_component = self.session.query(Component).order_by(Component.component_id).first()
+ else:
+ source_component = self.session.query(Component).filter_by(component_name=source_component_name).one()
source_component_func = lambda source: source_component
db_changes = self._install_changes()
projects / dak.git / blobdiff