]> git.donarmstrong.com Git - dak.git/blob - daklib/binary.py
And how about we make this mhy fault?
[dak.git] / daklib / binary.py
1 #!/usr/bin/python
2
3 """
4 Functions related debian binary packages
5
6 @contact: Debian FTPMaster <ftpmaster@debian.org>
7 @copyright: 2009  Mike O'Connor <stew@debian.org>
8 @license: GNU General Public License version 2 or later
9 """
10
11 # This program is free software; you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation; either version 2 of the License, or
14 # (at your option) any later version.
15
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19 # GNU General Public License for more details.
20
21 # You should have received a copy of the GNU General Public License
22 # along with this program; if not, write to the Free Software
23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
24
25 ################################################################################
26
27 # <Ganneff> are we going the xorg way?
28 # <Ganneff> a dak without a dak.conf?
29 # <stew> automatically detect the wrong settings at runtime?
30 # <Ganneff> yes!
31 # <mhy> well, we'll probably always need dak.conf (how do you get the database setting
32 # <mhy> but removing most of the config into the database seems sane
33 # <Ganneff> mhy: dont spoil the fun
34 # <Ganneff> mhy: and i know how. we nmap localhost and check all open ports
35 # <Ganneff> maybe one answers to sql
36 # <stew> we will discover projectb via avahi
37 # <mhy> you're both sick
38 # <mhy> really fucking sick
39
40 ################################################################################
41
42 import os
43 import sys
44 import shutil
45 import tarfile
46 import commands
47 import traceback
48 import atexit
49
50 from debian_bundle import deb822
51
52 from dbconn import *
53 from config import Config
54 import utils
55
56 ################################################################################
57
58 __all__ = []
59
60 ################################################################################
61
62 class Binary(object):
63     def __init__(self, filename, reject=None):
64         """
65         @type filename: string
66         @param filename: path of a .deb
67
68         @type reject: function
69         @param reject: a function to log reject messages to
70         """
71         self.filename = filename
72         self.tmpdir = None
73         self.chunks = None
74         self.wrapped_reject = reject
75         # Store rejects for later use
76         self.rejects = []
77
78     def reject(self, message):
79         """
80         if we were given a reject function, send the reject message,
81         otherwise send it to stderr.
82         """
83         print >> sys.stderr, message
84         self.rejects.append(message)
85         if self.wrapped_reject:
86             self.wrapped_reject(message)
87
88     def __del__(self):
89         """
90         make sure we cleanup when we are garbage collected.
91         """
92         self._cleanup()
93
94     def _cleanup(self):
95         """
96         we need to remove the temporary directory, if we created one
97         """
98         if self.tmpdir and os.path.exists(self.tmpdir):
99             shutil.rmtree(self.tmpdir)
100             self.tmpdir = None
101
102     def __scan_ar(self):
103         # get a list of the ar contents
104         if not self.chunks:
105
106             cmd = "ar t %s" % (self.filename)
107             (result, output) = commands.getstatusoutput(cmd)
108             if result != 0:
109                 rejected = True
110                 print("%s: 'ar t' invocation failed." % (self.filename))
111                 self.reject("%s: 'ar t' invocation failed." % (self.filename))
112                 self.reject(utils.prefix_multi_line_string(output, " [ar output:] "))
113             self.chunks = output.split('\n')
114
115
116
117     def __unpack(self):
118         # Internal function which extracts the contents of the .ar to
119         # a temporary directory
120
121         if not self.tmpdir:
122             tmpdir = utils.temp_dirname()
123             cwd = os.getcwd()
124             try:
125                 os.chdir( tmpdir )
126                 cmd = "ar x %s %s %s" % (os.path.join(cwd,self.filename), self.chunks[1], self.chunks[2])
127                 (result, output) = commands.getstatusoutput(cmd)
128                 if result != 0:
129                     print("%s: '%s' invocation failed." % (self.filename, cmd))
130                     self.reject("%s: '%s' invocation failed." % (self.filename, cmd))
131                     self.reject(utils.prefix_multi_line_string(output, " [ar output:] "))
132                 else:
133                     self.tmpdir = tmpdir
134                     atexit.register( self._cleanup )
135
136             finally:
137                 os.chdir( cwd )
138
139     def valid_deb(self, relaxed=False):
140         """
141         Check deb contents making sure the .deb contains:
142           1. debian-binary
143           2. control.tar.gz
144           3. data.tar.gz or data.tar.bz2
145         in that order, and nothing else.
146         """
147         self.__scan_ar()
148         rejected = not self.chunks
149         if relaxed:
150             if len(self.chunks) < 3:
151                 rejected = True
152                 self.reject("%s: found %d chunks, expected at least 3." % (self.filename, len(self.chunks)))
153         else:
154             if len(self.chunks) != 3:
155                 rejected = True
156                 self.reject("%s: found %d chunks, expected 3." % (self.filename, len(self.chunks)))
157         if self.chunks[0] != "debian-binary":
158             rejected = True
159             self.reject("%s: first chunk is '%s', expected 'debian-binary'." % (self.filename, self.chunks[0]))
160         if not rejected and self.chunks[1] != "control.tar.gz":
161             rejected = True
162             self.reject("%s: second chunk is '%s', expected 'control.tar.gz'." % (self.filename, self.chunks[1]))
163         if not rejected and self.chunks[2] not in [ "data.tar.bz2", "data.tar.gz" ]:
164             rejected = True
165             self.reject("%s: third chunk is '%s', expected 'data.tar.gz' or 'data.tar.bz2'." % (self.filename, self.chunks[2]))
166
167         return not rejected
168
169     def scan_package(self, bootstrap_id=0, relaxed=False, session=None):
170         """
171         Unpack the .deb, do sanity checking, and gather info from it.
172
173         Currently information gathering consists of getting the contents list. In
174         the hopefully near future, it should also include gathering info from the
175         control file.
176
177         @type bootstrap_id: int
178         @param bootstrap_id: the id of the binary these packages
179           should be associated or zero meaning we are not bootstrapping
180           so insert into a temporary table
181
182         @return: True if the deb is valid and contents were imported
183         """
184         result = False
185         rejected = not self.valid_deb(relaxed)
186         if not rejected:
187             self.__unpack()
188
189
190             cwd = os.getcwd()
191             if not rejected and self.tmpdir:
192                 try:
193                     os.chdir(self.tmpdir)
194                     if self.chunks[1] == "control.tar.gz":
195                         control = tarfile.open(os.path.join(self.tmpdir, "control.tar.gz" ), "r:gz")
196                         control.extract('./control', self.tmpdir )
197                     if self.chunks[2] == "data.tar.gz":
198                         data = tarfile.open(os.path.join(self.tmpdir, "data.tar.gz"), "r:gz")
199                     elif self.chunks[2] == "data.tar.bz2":
200                         data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2")
201
202                     if bootstrap_id:
203                         result = insert_content_paths(bootstrap_id, [tarinfo.name for tarinfo in data if not tarinfo.isdir()], session)
204                     else:
205                         pkgs = deb822.Packages.iter_paragraphs(file(os.path.join(self.tmpdir,'control')))
206                         pkg = pkgs.next()
207                         result = insert_pending_content_paths(pkg, [tarinfo.name for tarinfo in data if not tarinfo.isdir()], session)
208
209                 except:
210                     traceback.print_exc()
211
212             os.chdir(cwd)
213         self._cleanup()
214         return result
215
216     def check_utf8_package(self, package):
217         """
218         Unpack the .deb, do sanity checking, and gather info from it.
219
220         Currently information gathering consists of getting the contents list. In
221         the hopefully near future, it should also include gathering info from the
222         control file.
223
224         @type package: string
225         @param package: the name of the package to be checked
226
227         @rtype: boolean
228         @return: True if the deb is valid and contents were imported
229         """
230         rejected = not self.valid_deb(True)
231         self.__unpack()
232
233         if not rejected and self.tmpdir:
234             cwd = os.getcwd()
235             try:
236                 os.chdir(self.tmpdir)
237                 if self.chunks[1] == "control.tar.gz":
238                     control = tarfile.open(os.path.join(self.tmpdir, "control.tar.gz" ), "r:gz")
239                     control.extract('control', self.tmpdir )
240                 if self.chunks[2] == "data.tar.gz":
241                     data = tarfile.open(os.path.join(self.tmpdir, "data.tar.gz"), "r:gz")
242                 elif self.chunks[2] == "data.tar.bz2":
243                     data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2")
244
245                 for tarinfo in data:
246                     try:
247                         unicode( tarinfo.name )
248                     except:
249                         print >> sys.stderr, "E: %s has non-unicode filename: %s" % (package,tarinfo.name)
250
251             except:
252                 traceback.print_exc()
253                 result = False
254
255             os.chdir(cwd)
256
257 __all__.append('Binary')
258
259 def copy_temporary_contents(package, version, archname, deb, reject, session=None):
260     """
261     copy the previously stored contents from the temp table to the permanant one
262
263     during process-unchecked, the deb should have been scanned and the
264     contents stored in pending_content_associations
265     """
266
267     cnf = Config()
268
269     privatetrans = False
270     if session is None:
271         session = DBConn().session()
272         privatetrans = True
273
274     arch = get_architecture(archname, session=session)
275
276     # first see if contents exist:
277     in_pcaq = """SELECT 1 FROM pending_content_associations
278                                WHERE package=:package
279                                AND version=:version
280                                AND architecture=:archid LIMIT 1"""
281
282     vals = {'package': package,
283             'version': version,
284             'archid': arch.arch_id}
285
286     exists = None
287     check = session.execute(in_pcaq, vals)
288
289     if check.rowcount > 0:
290         # This should NOT happen.  We should have added contents
291         # during process-unchecked.  if it did, log an error, and send
292         # an email.
293         subst = {
294             "__PACKAGE__": package,
295             "__VERSION__": version,
296             "__ARCH__": arch,
297             "__TO_ADDRESS__": cnf["Dinstall::MyAdminAddress"],
298             "__DAK_ADDRESS__": cnf["Dinstall::MyEmailAddress"] }
299
300         message = utils.TemplateSubst(subst, cnf["Dir::Templates"]+"/missing-contents")
301         utils.send_mail(message)
302
303         # Temporarily disable contents storage until we re-do the table layout
304         #exists = Binary(deb, reject).scan_package()
305
306     if exists:
307         sql = """INSERT INTO content_associations(binary_pkg,filepath,filename)
308                  SELECT currval('binaries_id_seq'), filepath, filename FROM pending_content_associations
309                  WHERE package=:package AND version=:version AND architecture=:archid"""
310         session.execute(sql, vals)
311
312         sql = """DELETE from pending_content_associations
313                  WHERE package=:package AND version=:version AND architecture=:archid"""
314         session.execute(sql, vals)
315         session.commit()
316
317     if privatetrans:
318         session.close()
319
320     return exists
321
322 __all__.append('copy_temporary_contents')