Merge from Joachim

diff --git a/ChangeLog b/ChangeLog
index b6b1ac90a7d9a5290869ae26b2bd6e401c435016..4bb52a3dd581c1973329c8e582838b0d5aff7479 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -15,6 +15,16 @@
        one-line rsync command, using a feature rsync only understands
        since sarge - damn new-fangled technology.
 
+2008-08-05  Joachim Breitner <nomeata@debian.org>
+
+       * dak/cruft_report.py: In full mode, report out-of-date binaries on
+       architectures that have set Not-For-Us for that package.
+
+       * scripts/nfu/get-w-b-db: Script to fetch the wanna-build database
+       dump from http://buildd.debian.org/
+
+       * config/debian/cron.weekly: Run the above script
+
 2008-08-03  Mark Hymers <mhy@debian.org>
 
        * dak/process_new.py: Apply jvw's patch so that process_new shows

diff --git a/config/debian/cron.weekly b/config/debian/cron.weekly
index 39224ce8865db2668222a90e4a7176433f62de8b..4baaf46c3a7c9e482ecfce074b1d1298111784fd 100755 (executable)
--- a/config/debian/cron.weekly
+++ b/config/debian/cron.weekly
@@ -49,6 +49,10 @@ echo "Cleanup apt-ftparchive's database"
 apt-ftparchive -q clean apt.conf
 apt-ftparchive -q clean apt.conf.buildd
 
+# Update wanna-build dump
+echo "Update wanna-build database dump"
+/org/ftp.debian.org/scripts/nfu/get-w-b-db
+
 echo "Finally, all is done, compressing logfile"
 exec > /dev/null 2>&1
 

diff --git a/dak/cruft_report.py b/dak/cruft_report.py
index 6837ebf7cd2c8e22f82c32a0cd2b7c6408a7a5b7..add44489ebb84e922b8553635d93515f39f7a921 100755 (executable)
--- a/dak/cruft_report.py
+++ b/dak/cruft_report.py
@@ -27,7 +27,7 @@
 
 ################################################################################
 
-import commands, pg, os, sys, time
+import commands, pg, os, sys, time, re
 import apt_pkg
 from daklib import database
 from daklib import utils
@@ -51,7 +51,8 @@ Check for obsolete or duplicated packages.
 
   -h, --help                show this help and exit.
   -m, --mode=MODE           chose the MODE to run in (full or daily).
-  -s, --suite=SUITE         check suite SUITE."""
+  -s, --suite=SUITE         check suite SUITE.
+  -w, --wanna-build-dump    where to find the copies of http://buildd.debian.org/stats/*.txt"""
     sys.exit(exit_code)
 
 ################################################################################
@@ -114,6 +115,61 @@ def do_anais(architecture, binaries_list, source):
                 anais_output += "    o %s: %s\n" % (version, ", ".join(arches))
     return anais_output
 
+
+################################################################################
+
+# Check for out-of-date binaries on architectures that do not want to build that
+# package any more, and have them listed as Not-For-Us
+def do_nfu(nfu_packages):
+    output = ""
+    
+    a2p = {}
+
+    for architecture in nfu_packages:
+        a2p[architecture] = []
+        for (package,bver,sver) in nfu_packages[architecture]:
+            output += "  * [%s] does not want %s (binary %s, source %s)\n" % (architecture, package, bver, sver)
+            a2p[architecture].append(package)
+
+
+    if output:
+        print "Obsolete by Not-For-Us"
+        print "----------------------"
+        print
+        print output
+
+        print "Suggested commands:"
+        for architecture in a2p:
+            if a2p[architecture]:
+                print (" dak rm -m \"[auto-cruft] NFU\" -s %s -a %s -b %s" % 
+                    (suite, architecture, " ".join(a2p[architecture])))
+        print
+
+def parse_nfu(architecture):
+    # utils/hpodder_1.1.5.0: Not-For-Us [optional:out-of-date]
+    r = re.compile("^\w+/([^_]+)_.*: Not-For-Us")
+
+    ret = set()
+    
+    filename = "%s/%s-all.txt" % (Cnf["Cruft-Report::Options::Wanna-Build-Dump"], architecture)
+
+    # Not all architectures may have a wanna-build dump, so we want to ignore missin
+    # files
+    if os.path.exists(filename):
+        f = utils.open_file(filename)
+        for line in f:
+            if line[0] == ' ':
+                continue
+
+            m = r.match(line)
+            if m:
+                ret.add(m.group(1))
+
+        f.close()
+    else:
+        warn("No wanna-build dump file for architecture %s", architecture)
+    return ret
+
 ################################################################################
 
 def do_nviu():
@@ -265,7 +321,8 @@ def main ():
 
     Arguments = [('h',"help","Cruft-Report::Options::Help"),
                  ('m',"mode","Cruft-Report::Options::Mode", "HasArg"),
-                 ('s',"suite","Cruft-Report::Options::Suite","HasArg")]
+                 ('s',"suite","Cruft-Report::Options::Suite","HasArg"),
+                 ('w',"wanna-build-dump","Cruft-Report::Options::Wanna-Build-Dump","HasArg")]
     for i in [ "help" ]:
         if not Cnf.has_key("Cruft-Report::Options::%s" % (i)):
             Cnf["Cruft-Report::Options::%s" % (i)] = ""
@@ -274,6 +331,9 @@ def main ():
     if not Cnf.has_key("Cruft-Report::Options::Mode"):
         Cnf["Cruft-Report::Options::Mode"] = "daily"
 
+    if not Cnf.has_key("Cruft-Report::Options::Wanna-Build-Dump"):
+        Cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp.debian.org/scripts/nfu"
+
     apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
 
     Options = Cnf.SubTree("Cruft-Report::Options")
@@ -284,7 +344,7 @@ def main ():
     if Options["Mode"] == "daily":
         checks = [ "nbs", "nviu", "obsolete source" ]
     elif Options["Mode"] == "full":
-        checks = [ "nbs", "nviu", "obsolete source", "dubious nbs", "bnb", "bms", "anais" ]
+        checks = [ "nbs", "nviu", "obsolete source", "nfu", "dubious nbs", "bnb", "bms", "anais" ]
     else:
         utils.warn("%s is not a recognised mode - only 'full' or 'daily' are understood." % (Options["Mode"]))
         usage(1)
@@ -302,6 +362,8 @@ def main ():
     anais_output = ""
     duplicate_bins = {}
 
+    nfu_packages = {}
+
     suite = Options["Suite"]
     suite_id = database.get_suite_id(suite)
 
@@ -372,6 +434,10 @@ def main ():
             if (result != 0):
                 sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
                 sys.exit(result)
+
+            nfu_packages.setdefault(architecture,[])
+            nfu_entries = parse_nfu(architecture)
+
             packages = utils.open_file(temp_filename)
             Packages = apt_pkg.ParseTagFile(packages)
             while Packages.Step():
@@ -405,6 +471,10 @@ def main ():
                         duplicate_bins.setdefault(key, [])
                         if package not in duplicate_bins[key]:
                             duplicate_bins[key].append(package)
+                    if package in nfu_entries and \
+                        version != source_versions[source]: # only suggest to remove out-of-date packages
+                        nfu_packages[architecture].append((package,version,source_versions[source]))
+                    
             packages.close()
             os.unlink(temp_filename)
 
@@ -437,6 +507,9 @@ def main ():
         print "="*75
         print
 
+    if "nfu" in checks:
+        do_nfu(nfu_packages)
+
     if "bnb" in checks:
         print "Unbuilt binary packages"
         print "-----------------------"

diff --git a/scripts/nfu/get-w-b-db b/scripts/nfu/get-w-b-db
new file mode 100755 (executable)
index 0000000..c34e5e3
--- /dev/null
+++ b/scripts/nfu/get-w-b-db
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e
+
+# list of architectures taken from
+# http://buildd.debian.org/stats/
+
+# For debugging, you can override the path using
+# the WB_DB_DIR enviroment variable
+if [ -z "$WB_DB_DIR" ]
+then
+       WB_DB_DIR=/srv/ftp.debian.org/scripts/nfu
+fi
+
+cd $WB_DB_DIR || { echo "Failed to cd to $WB_DB_DIR" ; exit 1 ;}
+
+for arch in alpha amd64 arm armel hppa i386 ia64 m68k mips mipsel powerpc s390 sparc
+do
+       rm -f $arch-all.txt
+       echo "Getting $arch-all.txt"
+       wget -q http://buildd.debian.org/stats/$arch-all.txt
+done