3 # Check for obsolete binary packages
4 # Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 ################################################################################
22 # ``If you're claiming that's a "problem" that needs to be "fixed",
23 # you might as well write some letters to God about how unfair entropy
24 # is while you're at it.'' -- 20020802143104.GA5628@azure.humbug.org.au
26 ## TODO: fix NBS looping for version, implement Dubious NBS, fix up output of duplicate source package stuff, improve experimental ?, add support for non-US ?, add overrides, avoid ANAIS for duplicated packages
28 ################################################################################
30 import commands, pg, os, string, sys, time
31 import dak.lib.utils, dak.lib.database
34 ################################################################################
39 no_longer_in_suite = {}; # Really should be static to add_nbs, but I'm lazy
44 ################################################################################
46 def usage(exit_code=0):
47 print """Usage: dak cruft-report
48 Check for obsolete or duplicated packages.
50 -h, --help show this help and exit.
51 -m, --mode=MODE chose the MODE to run in (full or daily).
52 -s, --suite=SUITE check suite SUITE."""
55 ################################################################################
57 def add_nbs(nbs_d, source, version, package):
58 # Ensure the package is still in the suite (someone may have already removed it)
59 if no_longer_in_suite.has_key(package):
62 q = projectB.query("SELECT b.id FROM binaries b, bin_associations ba WHERE ba.bin = b.id AND ba.suite = %s AND b.package = '%s' LIMIT 1" % (suite_id, package))
64 no_longer_in_suite[package] = ""
67 nbs_d.setdefault(source, {})
68 nbs_d[source].setdefault(version, {})
69 nbs_d[source][version][package] = ""
71 ################################################################################
73 # Check for packages built on architectures they shouldn't be.
74 def do_anais(architecture, binaries_list, source):
75 if architecture == "any" or architecture == "all":
80 for arch in architecture.split():
81 architectures[arch.strip()] = ""
82 for binary in binaries_list:
83 q = projectB.query("SELECT a.arch_string, b.version FROM binaries b, bin_associations ba, architecture a WHERE ba.suite = %s AND ba.bin = b.id AND b.architecture = a.id AND b.package = '%s'" % (suite_id, binary))
89 if architectures.has_key(arch):
90 versions.append(version)
91 versions.sort(apt_pkg.VersionCompare)
93 latest_version = versions.pop()
96 # Check for 'invalid' architectures
101 if not architectures.has_key(arch):
102 versions_d.setdefault(version, [])
103 versions_d[version].append(arch)
106 anais_output += "\n (*) %s_%s [%s]: %s\n" % (binary, latest_version, source, architecture)
107 versions = versions_d.keys()
108 versions.sort(apt_pkg.VersionCompare)
109 for version in versions:
110 arches = versions_d[version]
112 anais_output += " o %s: %s\n" % (version, ", ".join(arches))
115 ################################################################################
118 experimental_id = dak.lib.database.get_suite_id("experimental")
119 if experimental_id == -1:
121 # Check for packages in experimental obsoleted by versions in unstable
122 q = projectB.query("""
123 SELECT s.source, s.version AS experimental, s2.version AS unstable
124 FROM src_associations sa, source s, source s2, src_associations sa2
125 WHERE sa.suite = %s AND sa2.suite = %d AND sa.source = s.id
126 AND sa2.source = s2.id AND s.source = s2.source
127 AND versioncmp(s.version, s2.version) < 0""" % (experimental_id,
128 dak.lib.database.get_suite_id("unstable")))
132 print "Newer version in unstable"
133 print "-------------------------"
136 (source, experimental_version, unstable_version) = i
137 print " o %s (%s, %s)" % (source, experimental_version, unstable_version)
138 nviu_to_remove.append(source)
140 print "Suggested command:"
141 print " dak rm -m \"[auto-cruft] NVIU\" -s experimental %s" % (" ".join(nviu_to_remove))
144 ################################################################################
146 def do_nbs(real_nbs):
147 output = "Not Built from Source\n"
148 output += "---------------------\n\n"
151 nbs_keys = real_nbs.keys()
153 for source in nbs_keys:
154 output += " * %s_%s builds: %s\n" % (source,
155 source_versions.get(source, "??"),
156 source_binaries.get(source, "(source does not exist)"))
157 output += " but no longer builds:\n"
158 versions = real_nbs[source].keys()
159 versions.sort(apt_pkg.VersionCompare)
160 for version in versions:
161 packages = real_nbs[source][version].keys()
164 nbs_to_remove.append(pkg)
165 output += " o %s: %s\n" % (version, ", ".join(packages))
172 print "Suggested command:"
173 print " dak rm -m \"[auto-cruft] NBS\" -b %s" % (" ".join(nbs_to_remove))
176 ################################################################################
178 def do_dubious_nbs(dubious_nbs):
183 dubious_nbs_keys = dubious_nbs.keys()
184 dubious_nbs_keys.sort()
185 for source in dubious_nbs_keys:
186 print " * %s_%s builds: %s" % (source,
187 source_versions.get(source, "??"),
188 source_binaries.get(source, "(source does not exist)"))
189 print " won't admit to building:"
190 versions = dubious_nbs[source].keys()
191 versions.sort(apt_pkg.VersionCompare)
192 for version in versions:
193 packages = dubious_nbs[source][version].keys()
195 print " o %s: %s" % (version, ", ".join(packages))
199 ################################################################################
201 def do_obsolete_source(duplicate_bins, bin2source):
203 for key in duplicate_bins.keys():
204 (source_a, source_b) = key.split('~')
205 for source in [ source_a, source_b ]:
206 if not obsolete.has_key(source):
207 if not source_binaries.has_key(source):
208 # Source has already been removed
211 obsolete[source] = map(string.strip,
212 source_binaries[source].split(','))
213 for binary in duplicate_bins[key]:
214 if bin2source.has_key(binary) and bin2source[binary]["source"] == source:
216 if binary in obsolete[source]:
217 obsolete[source].remove(binary)
220 output = "Obsolete source package\n"
221 output += "-----------------------\n\n"
222 obsolete_keys = obsolete.keys()
224 for source in obsolete_keys:
225 if not obsolete[source]:
226 to_remove.append(source)
227 output += " * %s (%s)\n" % (source, source_versions[source])
228 for binary in map(string.strip, source_binaries[source].split(',')):
229 if bin2source.has_key(binary):
230 output += " o %s (%s) is built by %s.\n" \
231 % (binary, bin2source[binary]["version"],
232 bin2source[binary]["source"])
234 output += " o %s is not built.\n" % binary
240 print "Suggested command:"
241 print " dak rm -S -p -m \"[auto-cruft] obsolete source package\" %s" % (" ".join(to_remove))
244 ################################################################################
247 global Cnf, projectB, suite_id, source_binaries, source_versions
249 Cnf = dak.lib.utils.get_conf()
251 Arguments = [('h',"help","Cruft-Report::Options::Help"),
252 ('m',"mode","Cruft-Report::Options::Mode", "HasArg"),
253 ('s',"suite","Cruft-Report::Options::Suite","HasArg")]
255 if not Cnf.has_key("Cruft-Report::Options::%s" % (i)):
256 Cnf["Cruft-Report::Options::%s" % (i)] = ""
257 Cnf["Cruft-Report::Options::Suite"] = Cnf["Dinstall::DefaultSuite"]
259 if not Cnf.has_key("Cruft-Report::Options::Mode"):
260 Cnf["Cruft-Report::Options::Mode"] = "daily"
262 apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
264 Options = Cnf.SubTree("Cruft-Report::Options")
268 # Set up checks based on mode
269 if Options["Mode"] == "daily":
270 checks = [ "nbs", "nviu", "obsolete source" ]
271 elif Options["Mode"] == "full":
272 checks = [ "nbs", "nviu", "obsolete source", "dubious nbs", "bnb", "bms", "anais" ]
274 dak.lib.utils.warn("%s is not a recognised mode - only 'full' or 'daily' are understood." % (Options["Mode"]))
277 projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
278 dak.lib.database.init(Cnf, projectB)
290 suite = Options["Suite"]
291 suite_id = dak.lib.database.get_suite_id(suite)
296 # Initalize a large hash table of all binary packages
298 sys.stderr.write("[Getting a list of binary packages in %s..." % (suite))
299 q = projectB.query("SELECT distinct b.package FROM binaries b, bin_associations ba WHERE ba.suite = %s AND ba.bin = b.id" % (suite_id))
301 sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before)))
303 bins_in_suite[i[0]] = ""
305 # Checks based on the Sources files
306 components = Cnf.ValueList("Suite::%s::Components" % (suite))
307 for component in components:
308 filename = "%s/dists/%s/%s/source/Sources.gz" % (Cnf["Dir::Root"], suite, component)
309 # apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance...
310 temp_filename = dak.lib.utils.temp_filename()
311 (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
313 sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
315 sources = dak.lib.utils.open_file(temp_filename)
316 Sources = apt_pkg.ParseTagFile(sources)
317 while Sources.Step():
318 source = Sources.Section.Find('Package')
319 source_version = Sources.Section.Find('Version')
320 architecture = Sources.Section.Find('Architecture')
321 binaries = Sources.Section.Find('Binary')
322 binaries_list = map(string.strip, binaries.split(','))
325 # Check for binaries not built on any architecture.
326 for binary in binaries_list:
327 if not bins_in_suite.has_key(binary):
328 bin_not_built.setdefault(source, {})
329 bin_not_built[source][binary] = ""
331 if "anais" in checks:
332 anais_output += do_anais(architecture, binaries_list, source)
334 # Check for duplicated packages and build indices for checking "no source" later
335 source_index = component + '/' + source
336 if src_pkgs.has_key(source):
337 print " %s is a duplicated source package (%s and %s)" % (source, source_index, src_pkgs[source])
338 src_pkgs[source] = source_index
339 for binary in binaries_list:
340 if bin_pkgs.has_key(binary):
341 key_list = [ source, bin_pkgs[binary] ]
343 key = '~'.join(key_list)
344 duplicate_bins.setdefault(key, [])
345 duplicate_bins[key].append(binary)
346 bin_pkgs[binary] = source
347 source_binaries[source] = binaries
348 source_versions[source] = source_version
351 os.unlink(temp_filename)
353 # Checks based on the Packages files
354 for component in components + ['main/debian-installer']:
355 architectures = filter(dak.lib.utils.real_arch, Cnf.ValueList("Suite::%s::Architectures" % (suite)))
356 for architecture in architectures:
357 filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (Cnf["Dir::Root"], suite, component, architecture)
358 # apt_pkg.ParseTagFile needs a real file handle
359 temp_filename = dak.lib.utils.temp_filename()
360 (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
362 sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
364 packages = dak.lib.utils.open_file(temp_filename)
365 Packages = apt_pkg.ParseTagFile(packages)
366 while Packages.Step():
367 package = Packages.Section.Find('Package')
368 source = Packages.Section.Find('Source', "")
369 version = Packages.Section.Find('Version')
372 if bin2source.has_key(package) and \
373 apt_pkg.VersionCompare(version, bin2source[package]["version"]) > 0:
374 bin2source[package]["version"] = version
375 bin2source[package]["source"] = source
377 bin2source[package] = {}
378 bin2source[package]["version"] = version
379 bin2source[package]["source"] = source
380 if source.find("(") != -1:
381 m = dak.lib.utils.re_extract_src_version.match(source)
384 if not bin_pkgs.has_key(package):
385 nbs.setdefault(source,{})
386 nbs[source].setdefault(package, {})
387 nbs[source][package][version] = ""
389 previous_source = bin_pkgs[package]
390 if previous_source != source:
391 key_list = [ source, previous_source ]
393 key = '~'.join(key_list)
394 duplicate_bins.setdefault(key, [])
395 if package not in duplicate_bins[key]:
396 duplicate_bins[key].append(package)
398 os.unlink(temp_filename)
400 if "obsolete source" in checks:
401 do_obsolete_source(duplicate_bins, bin2source)
403 # Distinguish dubious (version numbers match) and 'real' NBS (they don't)
406 for source in nbs.keys():
407 for package in nbs[source].keys():
408 versions = nbs[source][package].keys()
409 versions.sort(apt_pkg.VersionCompare)
410 latest_version = versions.pop()
411 source_version = source_versions.get(source,"0")
412 if apt_pkg.VersionCompare(latest_version, source_version) == 0:
413 add_nbs(dubious_nbs, source, latest_version, package)
415 add_nbs(real_nbs, source, latest_version, package)
425 if Options["Mode"] == "full":
430 print "Unbuilt binary packages"
431 print "-----------------------"
433 keys = bin_not_built.keys()
436 binaries = bin_not_built[source].keys()
438 print " o %s: %s" % (source, ", ".join(binaries))
442 print "Built from multiple source packages"
443 print "-----------------------------------"
445 keys = duplicate_bins.keys()
448 (source_a, source_b) = key.split("~")
449 print " o %s & %s => %s" % (source_a, source_b, ", ".join(duplicate_bins[key]))
452 if "anais" in checks:
453 print "Architecture Not Allowed In Source"
454 print "----------------------------------"
458 if "dubious nbs" in checks:
459 do_dubious_nbs(dubious_nbs)
462 ################################################################################
464 if __name__ == '__main__':