X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=trigger.daily;h=2d4380621918f65aed3f80e4aca71c870354af6f;hb=01e44aadfe82d4150966442c74c75aeb03d5f6cc;hp=898b09478e74a15437e6cdef4ba1eec0316b4a1e;hpb=6913572ba2f55d35ed61102bf945e730d645b597;p=wannabuild.git

diff --git a/trigger.daily b/trigger.daily
index 898b094..2d43806 100755
--- a/trigger.daily
+++ b/trigger.daily
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -x
 #
 # Updates wanna-build databases after the archive maintenance
 # finishes
@@ -8,51 +8,23 @@
 #     Sources.* == uncompressed, concat'd version
 
 LANG=C
-PATH="/bin:/usr/bin"
+PATH="/bin:/usr/bin:/org/wanna-build/bin"
 #testing must be before unstable so late upld don't build for testing needlessly
-#DISTS="oldstable-security stable-security testing-security oldstable stable testing unstable"
-DISTS="stable-security testing-security stable testing unstable"
+DISTS="oldstable-security stable-security testing-security oldstable stable testing unstable"
+# While oldstable is purged
+#DISTS="stable-security testing-security stable testing unstable"
 STATS_DISTS="unstable testing stable"
 SECTIONS="main contrib non-free"
-ARCHS_oldstable="m68k arm sparc alpha powerpc i386 mips mipsel ia64 hppa s390"
-ARCHS_stable="arm sparc alpha powerpc i386 amd64 mips mipsel ia64 hppa s390"
-ARCHS_testing="$ARCHS_stable armel"
-ARCHS_unstable="$ARCHS_testing"
 TMPDIR="/org/wanna-build/tmp"
+LIBTRIGGER="/org/wanna-build/libtrigger.sh"
 LOCKFILE="/org/wanna-build/tmp/DB_Maintenance_In_Progress"
 MASTER="http://incoming.debian.org/debian/dists"
 MASTERBUILDD="http://incoming.debian.org/buildd"
 SECMASTER="http://security-master.debian.org/debian-security/dists"
 SECMASTERBUILDD=http://security-master.debian.org/buildd
-CURLOPT="-q -s -S -R -f -Y 10 -y 120 -K /srv/wanna-build/trigger.curlrc"
 NEWARCH=""
 
-curl_index () {
-    local url tmpname destname appendname curlopt
-    url="$1"
-    destname="$2"
-    appendname="$3"
-    tmpname=".$destname"
-    rc=0
-    rm -f "$tmpname"
-    if [ -e "$destname" ]; then
-	refdate=`perl -e "print scalar gmtime(((stat ('$destname'))[9]))"`
-        curl $CURLOPT -z "$refdate" "$url" -o "$tmpname"
-        rc=$?
-    else
-	curl $CURLOPT "$url" -o "$tmpname"
-	rc=$?
-    fi
-    if [ $rc -eq 0 -a -e "$tmpname" ]; then
-	if gzip -t "$tmpname"; then
-	    mv "$tmpname" "$destname"
-	else
-	    rc=$?
-	fi
-    fi
-    gzip -dc "$destname" >> "$appendname"
-    return $rc
-}
+. "$LIBTRIGGER"
 
 DAY=`date +%w`
 
@@ -70,7 +42,7 @@ echo Updating wanna-build databases...
 umask 027
 
 if [ "$DAY" = "0" ]; then
-	savelog -c 26 -p /srv/wanna-build/db/merge.log
+	savelog -c 64 -p /srv/wanna-build/db/merge.log
 fi
 
 exec 3<&1 >> /srv/wanna-build/db/merge.log 2>&1
@@ -146,15 +118,20 @@ for d in $DISTS; do
 # double builds.
 #
     if [ "$d" = "unstable" ]; then
-	rm -f "Sources.$d.base"
-	cp "Sources.$d" "Sources.$d.base"
-	curl_index "$MASTERBUILDD/Sources.gz" "Sources-$d.accepted.gz" "Sources.$d"
+	mv "Sources.$d" "Sources.$d.base"
+	curl_index "$MASTERBUILDD/Sources.gz" "Sources-$d.accepted.gz" "/dev/null"
+	rm -f "Sources-$d.accepted"
+	gunzip "Sources-$d.accepted.gz"
+	keep-latest source "Sources.$d.base" "Sources-$d.accepted" > "Sources.$d"
+
 	curl_index "$MASTERBUILDD/Packages.gz" "Packages-$d.accepted.gz" "/dev/null"
+	rm -f "Packages-$d.accepted"
+	gunzip "Packages-$d.accepted.gz"
 
 	for a in $ARCHS; do
-	    rm -f "Packages.$d.$a.base"
-	    cp "Packages.$d.$a" "Packages.$d.$a.base"
+	    mv "Packages.$d.$a" "Packages.$d.$a.base"
 	    gzip -dc "Packages-$d.accepted.gz" >> "Packages.$d.$a"
+	    keep-latest $a "Packages.$d.$a.base" "Packages-$d.accepted" > "Packages.$d.$a"
 	done
     else
 	if [ -e "Sources.$d-p-u" ]; then
@@ -163,11 +140,11 @@ for d in $DISTS; do
     fi
 
     for a in $ARCHS; do
-	quinn-diff -A $a -a /srv/buildd.debian.org/web/quinn-diff/Packages-arch-specific -s Sources.$d -p Packages.$d.$a >> quinn-$d.$a 2>/dev/null
+	quinn-diff -A $a -a /srv/buildd.debian.org/web/quinn-diff/$dist/Packages-arch-specific -s Sources.$d -p Packages.$d.$a >> quinn-$d.$a 2>/dev/null
 	if echo $d | grep -qv -- -security; then
 	    if [ "$d" != "unstable" ]; then
 		cat "Packages.$d.$a" >> "Packages.$d-p-u.$a"
-		quinn-diff -A $a -a /srv/buildd.debian.org/web/quinn-diff/Packages-arch-specific -s Sources.$d-p-u -p Packages.$d-p-u.$a >> quinn-$d-p-u.$a 2>/dev/null
+		quinn-diff -A $a -a /srv/buildd.debian.org/web/quinn-diff/$dist/Packages-arch-specific -s Sources.$d-p-u -p Packages.$d-p-u.$a >> quinn-$d-p-u.$a 2>/dev/null
 		sed -e 's/\[[-a-z]*:[-a-z]*\]$//' quinn-$d.$a > quinn-$d.$a.grep
 		if [ ! -z quinn-$d.$a.grep ]; then
 		    fgrep -vf quinn-$d.$a.grep quinn-$d-p-u.$a > quinn-$d.$a
@@ -187,7 +164,12 @@ echo "fetch and quinn completed: `date -u`"
 echo "fetch and quinn completed: `date -u`" >&3
 
 umask 002
-for a in $ARCHS_unstable ; do
+# Gross hack: we need a list of all architectures, because this loop
+# insists on iterating on arches and then suites, instead of the
+# opposite.
+ALL_ARCHES=$(echo $ARCHS_oldstable $ARCHS_stable $ARCHS_testing $ARCHS_unstable \
+	     | tr ' ' '\n' | sort -u)
+for a in $ALL_ARCHES ; do
 	wanna-build --create-maintenance-lock --database=$a/build-db
 
 	for d in $DISTS ; do
@@ -227,7 +209,7 @@ for a in $ARCHS_unstable ; do
 		mv quinn-$d.$a quinn-$d.$a-old
 	done
 	if [ "$DAY" = "0" ]; then
-		savelog -p -c 26 /srv/wanna-build/db/$a/transactions.log
+		savelog -p -c 128 /srv/wanna-build/db/$a/transactions.log
 	fi
 	wanna-build --remove-maintenance-lock --database=$a/build-db
 done
@@ -250,7 +232,7 @@ if (( $last + $interval < $now )); then
 	rm -f "$LOCKFILE"
 	trap - 0
 	/srv/buildd.debian.org/bin/makegraph
-	for a in $ARCHS_stable; do
+	for a in $ALL_ARCHES; do
 	    echo Last Updated: `date -u` > /srv/buildd.debian.org/web/stats/$a.txt
 	    for d in $STATS_DISTS; do
 		/srv/wanna-build/bin/wanna-build-statistics --database=$a/build-db --dist=$d >> /srv/buildd.debian.org/web/stats/$a.txt