daklib/policy.py

   1 # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
   2 #
   3 # This program is free software; you can redistribute it and/or modify
   4 # it under the terms of the GNU General Public License as published by
   5 # the Free Software Foundation; either version 2 of the License, or
   6 # (at your option) any later version.
   7 #
   8 # This program is distributed in the hope that it will be useful,
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 # GNU General Public License for more details.
  12 #
  13 # You should have received a copy of the GNU General Public License along
  14 # with this program; if not, write to the Free Software Foundation, Inc.,
  15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16
  17 """module to process policy queue uploads"""
  18
  19 from .config import Config
  20 from .dbconn import BinaryMetadata, Component, MetadataKey, Override, OverrideType, Suite, get_mapped_component
  21 from .fstransactions import FilesystemTransaction
  22 from .regexes import re_file_changes, re_file_safe
  23 import daklib.utils as utils
  24
  25 import errno
  26 import os
  27 import shutil
  28 import tempfile
  29
  30 class UploadCopy(object):
  31     """export a policy queue upload
  32
  33     This class can be used in a with-statement::
  34
  35        with UploadCopy(...) as copy:
  36           ...
  37
  38     Doing so will provide a temporary copy of the upload in the directory
  39     given by the C{directory} attribute.  The copy will be removed on leaving
  40     the with-block.
  41     """
  42     def __init__(self, upload, group=None):
  43         """initializer
  44
  45         @type  upload: L{daklib.dbconn.PolicyQueueUpload}
  46         @param upload: upload to handle
  47         """
  48
  49         self.directory = None
  50         self.upload = upload
  51         self.group = group
  52
  53     def export(self, directory, mode=None, symlink=True, ignore_existing=False):
  54         """export a copy of the upload
  55
  56         @type  directory: str
  57         @param directory: directory to export to
  58
  59         @type  mode: int
  60         @param mode: permissions to use for the copied files
  61
  62         @type  symlink: bool
  63         @param symlink: use symlinks instead of copying the files
  64
  65         @type  ignore_existing: bool
  66         @param ignore_existing: ignore already existing files
  67         """
  68         with FilesystemTransaction() as fs:
  69             source = self.upload.source
  70             queue = self.upload.policy_queue
  71
  72             if source is not None:
  73                 for dsc_file in source.srcfiles:
  74                     f = dsc_file.poolfile
  75                     dst = os.path.join(directory, os.path.basename(f.filename))
  76                     if not os.path.exists(dst) or not ignore_existing:
  77                         fs.copy(f.fullpath, dst, mode=mode, symlink=symlink)
  78
  79             for binary in self.upload.binaries:
  80                 f = binary.poolfile
  81                 dst = os.path.join(directory, os.path.basename(f.filename))
  82                 if not os.path.exists(dst) or not ignore_existing:
  83                     fs.copy(f.fullpath, dst, mode=mode, symlink=symlink)
  84
  85             # copy byhand files
  86             for byhand in self.upload.byhand:
  87                 src = os.path.join(queue.path, byhand.filename)
  88                 dst = os.path.join(directory, byhand.filename)
  89                 if not os.path.exists(dst) or not ignore_existing:
  90                     fs.copy(src, dst, mode=mode, symlink=symlink)
  91
  92             # copy .changes
  93             src = os.path.join(queue.path, self.upload.changes.changesname)
  94             dst = os.path.join(directory, self.upload.changes.changesname)
  95             if not os.path.exists(dst) or not ignore_existing:
  96                 fs.copy(src, dst, mode=mode, symlink=symlink)
  97
  98     def __enter__(self):
  99         assert self.directory is None
 100
 101         mode = 0o0700
 102         symlink = True
 103         if self.group is not None:
 104             mode = 0o2750
 105             symlink = False
 106
 107         cnf = Config()
 108         self.directory = utils.temp_dirname(parent=cnf.get('Dir::TempPath'),
 109                                             mode=mode,
 110                                             group=self.group)
 111         self.export(self.directory, symlink=symlink)
 112         return self
 113
 114     def __exit__(self, *args):
 115         if self.directory is not None:
 116             shutil.rmtree(self.directory)
 117             self.directory = None
 118         return None
 119
 120 class PolicyQueueUploadHandler(object):
 121     """process uploads to policy queues
 122
 123     This class allows to accept or reject uploads and to get a list of missing
 124     overrides (for NEW processing).
 125     """
 126     def __init__(self, upload, session):
 127         """initializer
 128
 129         @type  upload: L{daklib.dbconn.PolicyQueueUpload}
 130         @param upload: upload to process
 131
 132         @param session: database session
 133         """
 134         self.upload = upload
 135         self.session = session
 136
 137     @property
 138     def _overridesuite(self):
 139         overridesuite = self.upload.target_suite
 140         if overridesuite.overridesuite is not None:
 141             overridesuite = self.session.query(Suite).filter_by(suite_name=overridesuite.overridesuite).one()
 142         return overridesuite
 143
 144     def _source_override(self, component_name):
 145         package = self.upload.source.source
 146         suite = self._overridesuite
 147         component = get_mapped_component(component_name, self.session)
 148         query = self.session.query(Override).filter_by(package=package, suite=suite) \
 149             .join(OverrideType).filter(OverrideType.overridetype == 'dsc') \
 150             .filter(Override.component == component)
 151         return query.first()
 152
 153     def _binary_override(self, binary, component_name):
 154         package = binary.package
 155         suite = self._overridesuite
 156         overridetype = binary.binarytype
 157         component = get_mapped_component(component_name, self.session)
 158         query = self.session.query(Override).filter_by(package=package, suite=suite) \
 159             .join(OverrideType).filter(OverrideType.overridetype == overridetype) \
 160             .filter(Override.component == component)
 161         return query.first()
 162
 163     def _binary_metadata(self, binary, key):
 164         metadata_key = self.session.query(MetadataKey).filter_by(key=key).first()
 165         if metadata_key is None:
 166             return None
 167         metadata = self.session.query(BinaryMetadata).filter_by(binary=binary, key=metadata_key).first()
 168         if metadata is None:
 169             return None
 170         return metadata.value
 171
 172     @property
 173     def _changes_prefix(self):
 174         changesname = self.upload.changes.changesname
 175         assert changesname.endswith('.changes')
 176         assert re_file_changes.match(changesname)
 177         return changesname[0:-8]
 178
 179     def accept(self):
 180         """mark upload as accepted"""
 181         assert len(self.missing_overrides()) == 0
 182
 183         fn1 = 'ACCEPT.{0}'.format(self._changes_prefix)
 184         fn = os.path.join(self.upload.policy_queue.path, 'COMMENTS', fn1)
 185         try:
 186             fh = os.open(fn, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
 187             os.write(fh, 'OK\n')
 188             os.close(fh)
 189         except OSError as e:
 190             if e.errno == errno.EEXIST:
 191                 pass
 192             else:
 193                 raise
 194
 195     def reject(self, reason):
 196         """mark upload as rejected
 197
 198         @type  reason: str
 199         @param reason: reason for the rejection
 200         """
 201         cnf = Config()
 202
 203         fn1 = 'REJECT.{0}'.format(self._changes_prefix)
 204         assert re_file_safe.match(fn1)
 205
 206         fn = os.path.join(self.upload.policy_queue.path, 'COMMENTS', fn1)
 207         try:
 208             fh = os.open(fn, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
 209             os.write(fh, 'NOTOK\n')
 210             os.write(fh, 'From: {0} <{1}>\n\n'.format(utils.whoami(), cnf['Dinstall::MyAdminAddress']))
 211             os.write(fh, reason)
 212             os.close(fh)
 213         except OSError as e:
 214             if e.errno == errno.EEXIST:
 215                 pass
 216             else:
 217                 raise
 218
 219     def get_action(self):
 220         """get current action
 221
 222         @rtype:  str
 223         @return: string giving the current action, one of 'ACCEPT', 'ACCEPTED', 'REJECT'
 224         """
 225         changes_prefix = self._changes_prefix
 226
 227         for action in ('ACCEPT', 'ACCEPTED', 'REJECT'):
 228             fn1 = '{0}.{1}'.format(action, changes_prefix)
 229             fn = os.path.join(self.upload.policy_queue.path, 'COMMENTS', fn1)
 230             if os.path.exists(fn):
 231                 return action
 232
 233         return None
 234
 235     def missing_overrides(self, hints=None):
 236         """get missing override entries for the upload
 237
 238         @type  hints: list of dict
 239         @param hints: suggested hints for new overrides in the same format as
 240                       the return value
 241
 242         @return: list of dicts with the following keys:
 243
 244                  - package: package name
 245                  - priority: default priority (from upload)
 246                  - section: default section (from upload)
 247                  - component: default component (from upload)
 248                  - type: type of required override ('dsc', 'deb' or 'udeb')
 249
 250                  All values are strings.
 251         """
 252         # TODO: use Package-List field
 253         missing = []
 254         components = set()
 255
 256         if hints is None:
 257             hints = []
 258         hints_map = dict([ ((o['type'], o['package']), o) for o in hints ])
 259
 260         for binary in self.upload.binaries:
 261             priority = self._binary_metadata(binary, 'Priority')
 262             section = self._binary_metadata(binary, 'Section')
 263             component = 'main'
 264             if section.find('/') != -1:
 265                 component = section.split('/', 1)[0]
 266             override = self._binary_override(binary, component)
 267             if override is None and not any(o['package'] == binary.package and o['type'] == binary.binarytype for o in missing):
 268                 hint = hints_map.get((binary.binarytype, binary.package))
 269                 if hint is not None:
 270                     missing.append(hint)
 271                     component = hint['component']
 272                 else:
 273                     missing.append(dict(
 274                             package = binary.package,
 275                             priority = priority,
 276                             section = section,
 277                             component = component,
 278                             type = binary.binarytype,
 279                             ))
 280             components.add(component)
 281
 282         source = self.upload.source
 283         source_component = '(unknown)'
 284         for component, in self.session.query(Component.component_name).order_by(Component.ordering):
 285             if component in components:
 286                 source_component = component
 287                 break
 288             else:
 289                 if source is not None:
 290                     if self._source_override(component) is not None:
 291                         source_component = component
 292                         break
 293
 294         if source is not None:
 295             override = self._source_override(source_component)
 296             if override is None:
 297                 hint = hints_map.get(('dsc', source.source))
 298                 if hint is not None:
 299                     missing.append(hint)
 300                 else:
 301                     section = 'misc'
 302                     if component != 'main':
 303                         section = "{0}/{1}".format(component, section)
 304                     missing.append(dict(
 305                             package = source.source,
 306                             priority = 'extra',
 307                             section = section,
 308                             component = source_component,
 309                             type = 'dsc',
 310                             ))
 311
 312         return missing