From: Don Armstrong <don@donarmstrong.com>
Date: Sun, 20 May 2007 08:36:06 +0000 (-0700)
Subject:  * Add mldbm stuff
X-Git-Tag: release/2.6.0~565^2~6
X-Git-Url: https://git.donarmstrong.com/?a=commitdiff_plain;h=0fb0243d81f8b626988c73c6a9e54a865eeeb50c;hp=1639566d7762d46bb4e13b26db7bd76f671667df;p=debbugs.git

 * Add mldbm stuff
 * Make versions_time so we can archive intelligently
 * Put all scripts in the bin directory
---

diff --git a/examples/debian/versions/build-mldbm.pl b/examples/debian/versions/build-mldbm.pl
new file mode 100755
index 0000000..f91d3d5
--- /dev/null
+++ b/examples/debian/versions/build-mldbm.pl
@@ -0,0 +1,34 @@
+#! /usr/bin/perl -w
+use strict;
+use MLDBM qw(DB_File Storable);
+use Fcntl;
+
+$MLDBM::DumpMeth=q(portable);
+
+my %db;
+tie %db, "MLDBM", "versions.idx.new", O_CREAT|O_RDWR, 0664
+    or die "tie versions.idx.new: $!";
+tie %db2, "MLDBM", "versions_time.idx.new",O_CREAT|O_RDWR, 0664
+     or die "tie versions_time.idx.new failed: $!";
+
+my $archive = shift;
+my $dist = shift;
+my $arch = shift;
+print "$archive/$dist/$arch\n";
+
+my $time = time;
+my ($p, $v);
+while (<>) {
+    if (/^Package: (.*)/)    { $p = $1; }
+    elsif (/^Version: (.*)/) { $v = $1; }
+    elsif (/^$/) {
+	# see MLDBM(3pm)/BUGS
+	my $tmp = $db{$p};
+	$tmp->{$dist}{$arch} = $v;
+	$db{$p} = $tmp;
+	$tmp = $db2{$p};
+	$tmp->{$dist}{$arch}{$v} = $time if not exists
+	     $tmp->{$dist}{$arch}{$v};
+	$db2{$p} = $tmp;
+    }
+}
diff --git a/examples/debian/versions/update-mldbm b/examples/debian/versions/update-mldbm
new file mode 100755
index 0000000..53bc971
--- /dev/null
+++ b/examples/debian/versions/update-mldbm
@@ -0,0 +1,72 @@
+#! /bin/sh -e
+
+cd /org/bugs.debian.org/versions/indices
+
+ARCHIVES='ftp' # security -- should be included too, but too difficult to deal with
+
+# Nuke old versions of versions.idx.new in case there's one hanging about
+rm -f versions.idx.new
+# This index is much larger and keeps track of historic versions of
+# packages, and is used for expiring bugs
+rm -f versions_time.idx.new
+if [ -e versions_time.idx ]; then
+    cp versions_time.idx versions_time.idx.new;
+fi;
+
+set -e
+for archive in $ARCHIVES; do
+    case $archive in
+	ftp)
+	    SUITES='oldstable stable proposed-updates testing testing-proposed-updates unstable experimental'
+	    di_main='main main/debian-installer'
+	    ;;
+	nonus)
+	    SUITES='oldstable'
+	    di_main='main'
+	    ;;
+	security)
+	    SUITES='oldstable stable testing'
+	    di_main='main'
+	    ;;
+    esac
+    for suite in $SUITES; do
+        if [ "$suite" != "oldstable" ] || [ -d /org/bugs.debian.org/etc/indices/$archive/$suite ]; then
+	case $suite in
+	    oldstable|stable|proposed-updates)
+		ARCHES='alpha arm hppa i386 ia64 m68k mips mipsel powerpc s390 sparc'
+		;;
+	    testing|testing-proposed-updates)
+		ARCHES='alpha amd64 arm hppa i386 ia64 mips mipsel powerpc s390 sparc'
+		;;
+	    unstable|experimental)
+		ARCHES='alpha amd64 arm hppa hurd-i386 i386 ia64 m68k mips mipsel powerpc s390 sparc'
+		;;
+	esac
+	case $suite in
+	    oldstable|experimental)
+		COMPONENTS='main contrib non-free'
+		;;
+	    stable|proposed-updates|testing|testing-proposed-updates|unstable)
+		COMPONENTS="$di_main contrib non-free"
+		;;
+	esac
+	for component in $COMPONENTS; do
+	    for arch in $ARCHES; do
+		zcat "/org/bugs.debian.org/etc/indices/$archive/$suite/$component/binary-$arch/Packages.gz" | ../bin/build-mldbm.pl "$archive" "$suite" "$arch"
+	    done
+	    if [ "$component" != main/debian-installer ]; then
+		zcat "/org/bugs.debian.org/etc/indices/$archive/$suite/$component/source/Sources.gz" | ../bin/build-mldbm.pl "$archive" "$suite" source
+	    fi
+	done
+	fi
+    done
+done
+
+# This removes old versions
+../bin/versions_time_cleanup
+
+chmod 664 versions.idx.new
+mv versions.idx.new versions.idx
+
+chmod 664 versions_time.idx.new
+mv versions_time.idx versions_time.idx
\ No newline at end of file
diff --git a/examples/debian/versions/update-packages b/examples/debian/versions/update-packages
index 502a167..7a50240 100755
--- a/examples/debian/versions/update-packages
+++ b/examples/debian/versions/update-packages
@@ -3,4 +3,4 @@
 /home/debbugs/get-packages ftp; 
 /home/debbugs/get-packages nonus; 
 /home/debbugs/get-packages security; 
-/org/bugs.debian.org/versions/indices/update-mldbm >/dev/null
+/org/bugs.debian.org/versions/bin/update-mldbm >/dev/null
diff --git a/examples/debian/versions/versions_time_cleanup b/examples/debian/versions/versions_time_cleanup
new file mode 100755
index 0000000..5bd4516
--- /dev/null
+++ b/examples/debian/versions/versions_time_cleanup
@@ -0,0 +1,38 @@
+#! /usr/bin/perl
+
+# This script's job is to clean up the version file and remove old
+# versions when there are multiple versions older than the archive
+# date.
+
+use warnings;
+use strict;
+
+use Debbugs::Config qw(:config);
+use MLDBM qw(DB_File Storable);
+use Fcntl;
+
+$MLDBM::DumpMeth=q(portable);
+
+my %db;
+tie %db, "MLDBM", "versions_time.idx.new",O_CREAT|O_RDWR, 0664
+     or die "tie versions_time.idx.new failed: $!";
+
+my $time = time;
+for my $package (keys %db) {
+     my $temp = $db{$package};
+     for my $dist (keys %{$temp}) {
+	  for my $arch (keys %{$temp->{$dist}}) {
+	       my @versions =  (sort {$temp->{$dist}{$arch}{$a} <=>
+					   $temp->{$dist}{$arch}{$a}
+				      }
+				keys %{$temp->{$dist}{$arch}});
+	       next unless @versions > 1;
+	       for my $i (0 .. ($#versions-1)) {
+		    last if $temp->{$dist}{$arch}{$versions[$i+1]} > ($time - $config{remove_age}*60*60*24);
+		    last if keys %{$temp->{$dist}{$arch}} <= 1;
+		    delete $temp->{$dist}{$arch}{$versions[$i+1]};
+	       }
+	  }
+     }
+     $db{$package} = $temp;
+}