daklib/binary.py

   1 #!/usr/bin/python
   2
   3 """
   4 Functions related debian binary packages
   5
   6 @contact: Debian FTPMaster <ftpmaster@debian.org>
   7 @copyright: 2009  Mike O'Connor <stew@debian.org>
   8 @license: GNU General Public License version 2 or later
   9 """
  10
  11 # This program is free software; you can redistribute it and/or modify
  12 # it under the terms of the GNU General Public License as published by
  13 # the Free Software Foundation; either version 2 of the License, or
  14 # (at your option) any later version.
  15
  16 # This program is distributed in the hope that it will be useful,
  17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 # GNU General Public License for more details.
  20
  21 # You should have received a copy of the GNU General Public License
  22 # along with this program; if not, write to the Free Software
  23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  24
  25 ################################################################################
  26
  27 # <Ganneff> are we going the xorg way?
  28 # <Ganneff> a dak without a dak.conf?
  29 # <stew> automatically detect the wrong settings at runtime?
  30 # <Ganneff> yes!
  31 # <mhy> well, we'll probably always need dak.conf (how do you get the database setting
  32 # <mhy> but removing most of the config into the database seems sane
  33 # <Ganneff> mhy: dont spoil the fun
  34 # <Ganneff> mhy: and i know how. we nmap localhost and check all open ports
  35 # <Ganneff> maybe one answers to sql
  36 # <stew> we will discover projectb via avahi
  37 # <mhy> you're both sick
  38 # <mhy> really fucking sick
  39
  40 ################################################################################
  41
  42 import os
  43 import sys
  44 import shutil
  45 import tarfile
  46 import commands
  47 import traceback
  48 import atexit
  49
  50 try:
  51     # starting with squeeze
  52     from debian import deb822
  53 except:
  54     # up to lenny
  55     from debian_bundle import deb822
  56
  57 from dbconn import *
  58 from config import Config
  59 import utils
  60
  61 ################################################################################
  62
  63 __all__ = []
  64
  65 ################################################################################
  66
  67 class Binary(object):
  68     def __init__(self, filename, reject=None):
  69         """
  70         @type filename: string
  71         @param filename: path of a .deb
  72
  73         @type reject: function
  74         @param reject: a function to log reject messages to
  75         """
  76         self.filename = filename
  77         self.tmpdir = None
  78         self.chunks = None
  79         self.wrapped_reject = reject
  80         # Store rejects for later use
  81         self.rejects = []
  82
  83     def reject(self, message):
  84         """
  85         if we were given a reject function, send the reject message,
  86         otherwise send it to stderr.
  87         """
  88         print >> sys.stderr, message
  89         self.rejects.append(message)
  90         if self.wrapped_reject:
  91             self.wrapped_reject(message)
  92
  93     def __del__(self):
  94         """
  95         make sure we cleanup when we are garbage collected.
  96         """
  97         self._cleanup()
  98
  99     def _cleanup(self):
 100         """
 101         we need to remove the temporary directory, if we created one
 102         """
 103         if self.tmpdir and os.path.exists(self.tmpdir):
 104             shutil.rmtree(self.tmpdir)
 105             self.tmpdir = None
 106
 107     def __scan_ar(self):
 108         # get a list of the ar contents
 109         if not self.chunks:
 110
 111             cmd = "ar t %s" % (self.filename)
 112             (result, output) = commands.getstatusoutput(cmd)
 113             if result != 0:
 114                 rejected = True
 115                 print("%s: 'ar t' invocation failed." % (self.filename))
 116                 self.reject("%s: 'ar t' invocation failed." % (self.filename))
 117                 self.reject(utils.prefix_multi_line_string(output, " [ar output:] "))
 118             self.chunks = output.split('\n')
 119
 120
 121
 122     def __unpack(self):
 123         # Internal function which extracts the contents of the .ar to
 124         # a temporary directory
 125
 126         if not self.tmpdir:
 127             tmpdir = utils.temp_dirname()
 128             cwd = os.getcwd()
 129             try:
 130                 os.chdir( tmpdir )
 131                 cmd = "ar x %s %s %s" % (os.path.join(cwd,self.filename), self.chunks[1], self.chunks[2])
 132                 (result, output) = commands.getstatusoutput(cmd)
 133                 if result != 0:
 134                     print("%s: '%s' invocation failed." % (self.filename, cmd))
 135                     self.reject("%s: '%s' invocation failed." % (self.filename, cmd))
 136                     self.reject(utils.prefix_multi_line_string(output, " [ar output:] "))
 137                 else:
 138                     self.tmpdir = tmpdir
 139                     atexit.register( self._cleanup )
 140
 141             finally:
 142                 os.chdir( cwd )
 143
 144     def valid_deb(self, relaxed=False):
 145         """
 146         Check deb contents making sure the .deb contains:
 147           1. debian-binary
 148           2. control.tar.gz
 149           3. data.tar.gz or data.tar.bz2
 150         in that order, and nothing else.
 151         """
 152         self.__scan_ar()
 153         rejected = not self.chunks
 154         if relaxed:
 155             if len(self.chunks) < 3:
 156                 rejected = True
 157                 self.reject("%s: found %d chunks, expected at least 3." % (self.filename, len(self.chunks)))
 158         else:
 159             if len(self.chunks) != 3:
 160                 rejected = True
 161                 self.reject("%s: found %d chunks, expected 3." % (self.filename, len(self.chunks)))
 162         if self.chunks[0] != "debian-binary":
 163             rejected = True
 164             self.reject("%s: first chunk is '%s', expected 'debian-binary'." % (self.filename, self.chunks[0]))
 165         if not rejected and self.chunks[1] != "control.tar.gz":
 166             rejected = True
 167             self.reject("%s: second chunk is '%s', expected 'control.tar.gz'." % (self.filename, self.chunks[1]))
 168         if not rejected and self.chunks[2] not in [ "data.tar.bz2", "data.tar.gz" ]:
 169             rejected = True
 170             self.reject("%s: third chunk is '%s', expected 'data.tar.gz' or 'data.tar.bz2'." % (self.filename, self.chunks[2]))
 171
 172         return not rejected
 173
 174     def scan_package(self, bootstrap_id=0, relaxed=False, session=None):
 175         """
 176         Unpack the .deb, do sanity checking, and gather info from it.
 177
 178         Currently information gathering consists of getting the contents list. In
 179         the hopefully near future, it should also include gathering info from the
 180         control file.
 181
 182         @type bootstrap_id: int
 183         @param bootstrap_id: the id of the binary these packages
 184           should be associated or zero meaning we are not bootstrapping
 185           so insert into a temporary table
 186
 187         @return: True if the deb is valid and contents were imported
 188         """
 189         result = False
 190         rejected = not self.valid_deb(relaxed)
 191         if not rejected:
 192             self.__unpack()
 193
 194
 195             cwd = os.getcwd()
 196             if not rejected and self.tmpdir:
 197                 try:
 198                     os.chdir(self.tmpdir)
 199                     if self.chunks[1] == "control.tar.gz":
 200                         control = tarfile.open(os.path.join(self.tmpdir, "control.tar.gz" ), "r:gz")
 201                         control.extract('./control', self.tmpdir )
 202                     if self.chunks[2] == "data.tar.gz":
 203                         data = tarfile.open(os.path.join(self.tmpdir, "data.tar.gz"), "r:gz")
 204                     elif self.chunks[2] == "data.tar.bz2":
 205                         data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2")
 206
 207                     if bootstrap_id:
 208                         result = insert_content_paths(bootstrap_id, [tarinfo.name for tarinfo in data if not tarinfo.isdir()], session)
 209                     else:
 210                         pkgs = deb822.Packages.iter_paragraphs(file(os.path.join(self.tmpdir,'control')))
 211                         pkg = pkgs.next()
 212                         result = insert_pending_content_paths(pkg,
 213                                                               self.filename.endswith('.udeb'),
 214                                                               [tarinfo.name for tarinfo in data if not tarinfo.isdir()],
 215                                                               session)
 216
 217                 except:
 218                     traceback.print_exc()
 219
 220             os.chdir(cwd)
 221         self._cleanup()
 222         return result
 223
 224     def check_utf8_package(self, package):
 225         """
 226         Unpack the .deb, do sanity checking, and gather info from it.
 227
 228         Currently information gathering consists of getting the contents list. In
 229         the hopefully near future, it should also include gathering info from the
 230         control file.
 231
 232         @type package: string
 233         @param package: the name of the package to be checked
 234
 235         @rtype: boolean
 236         @return: True if the deb is valid and contents were imported
 237         """
 238         rejected = not self.valid_deb(True)
 239         self.__unpack()
 240
 241         if not rejected and self.tmpdir:
 242             cwd = os.getcwd()
 243             try:
 244                 os.chdir(self.tmpdir)
 245                 if self.chunks[1] == "control.tar.gz":
 246                     control = tarfile.open(os.path.join(self.tmpdir, "control.tar.gz" ), "r:gz")
 247                     control.extract('control', self.tmpdir )
 248                 if self.chunks[2] == "data.tar.gz":
 249                     data = tarfile.open(os.path.join(self.tmpdir, "data.tar.gz"), "r:gz")
 250                 elif self.chunks[2] == "data.tar.bz2":
 251                     data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2")
 252
 253                 for tarinfo in data:
 254                     try:
 255                         unicode( tarinfo.name )
 256                     except:
 257                         print >> sys.stderr, "E: %s has non-unicode filename: %s" % (package,tarinfo.name)
 258
 259                 result = True
 260
 261             except:
 262                 traceback.print_exc()
 263                 result = False
 264
 265             os.chdir(cwd)
 266
 267         return result
 268
 269 __all__.append('Binary')
 270
 271
 272 def copy_temporary_contents(binary, bin_association, reject, session=None):
 273     """
 274     copy the previously stored contents from the temp table to the permanant one
 275
 276     during process-unchecked, the deb should have been scanned and the
 277     contents stored in pending_content_associations
 278     """
 279
 280     cnf = Config()
 281
 282     privatetrans = False
 283     if session is None:
 284         session = DBConn().session()
 285         privatetrans = True
 286
 287     arch = get_architecture(archname, session=session)
 288
 289     pending = session.query(PendingBinContents).filter_by(package=binary.package,
 290                                                           version=binary.version,
 291                                                           arch=binary.arch).first()
 292
 293     if pending:
 294         # This should NOT happen.  We should have added contents
 295         # during process-unchecked.  if it did, log an error, and send
 296         # an email.
 297         subst = {
 298             "__PACKAGE__": package,
 299             "__VERSION__": version,
 300             "__ARCH__": arch,
 301             "__TO_ADDRESS__": cnf["Dinstall::MyAdminAddress"],
 302             "__DAK_ADDRESS__": cnf["Dinstall::MyEmailAddress"] }
 303
 304         message = utils.TemplateSubst(subst, cnf["Dir::Templates"]+"/missing-contents")
 305         utils.send_mail(message)
 306
 307         # rescan it now
 308         exists = Binary(deb, reject).scan_package()
 309
 310         if not exists:
 311             # LOG?
 312             return False
 313
 314     component = binary.poolfile.location.component
 315     override = session.query(Override).filter_by(package=binary.package,
 316                                                  suite=bin_association.suite,
 317                                                  component=component.id).first()
 318     if not override:
 319         # LOG?
 320         return False
 321
 322
 323     if not override.overridetype.type.endswith('deb'):
 324         return True
 325
 326     if override.overridetype.type == "udeb":
 327         table = "udeb_contents"
 328     elif override.overridetype.type == "deb":
 329         table = "deb_contents"
 330     else:
 331         return False
 332
 333
 334     if component.name == "main":
 335         component_str = ""
 336     else:
 337         component_str = component.name + "/"
 338
 339     vals = { 'package':binary.package,
 340              'version':binary.version,
 341              'arch':binary.architecture,
 342              'binary_id': binary.id,
 343              'component':component_str,
 344              'section':override.section.section
 345              }
 346
 347     session.execute( """INSERT INTO %s
 348     (binary_id,package,version.component,arch,section,filename)
 349     SELECT :binary_id, :package, :version, :component, :arch, :section
 350     FROM pending_bin_contents pbc
 351     WHERE pbc.package=:package
 352     AND pbc.version=:version
 353     AND pbc.arch=:arch""" % table, vals )
 354
 355     session.execute( """DELETE from pending_bin_contents package=:package
 356     AND version=:version
 357     AND arch=:arch""", vals )
 358
 359     if privatetrans:
 360         session.commit()
 361         session.close()
 362
 363     return exists
 364
 365 __all__.append('copy_temporary_contents')
 366
 367