sync with reality (all changes from /org/wanna-build and currently live)

[wannabuild.git] / trigger.daily

diff --git a/trigger.daily b/trigger.daily
index 73b17768113ac50dadb6c39620e6791de38bcc73..fa345b3de94f053a329795294b1ad28b016aa17b 100755 (executable)
--- a/trigger.daily
+++ b/trigger.daily
@@ -8,10 +8,11 @@
 #     Sources.* == uncompressed, concat'd version
 
 LANG=C
-PATH="/bin:/usr/bin"
+PATH="/bin:/usr/bin:/org/wanna-build/bin"
 #testing must be before unstable so late upld don't build for testing needlessly
-#DISTS="oldstable-security stable-security testing-security oldstable stable testing unstable"
-DISTS="stable-security testing-security stable testing unstable"
+DISTS="oldstable-security stable-security testing-security oldstable stable testing unstable experimental"
+# While oldstable is purged
+#DISTS="stable-security testing-security stable testing unstable"
 STATS_DISTS="unstable testing stable"
 SECTIONS="main contrib non-free"
 TMPDIR="/org/wanna-build/tmp"
@@ -37,11 +38,14 @@ cleanup() {
 }
 trap cleanup 0
 
+echo Dumping wanna-build databases...
+save-database pre
+
 echo Updating wanna-build databases...
 umask 027
 
 if [ "$DAY" = "0" ]; then
-       savelog -c 26 -p /srv/wanna-build/db/merge.log
+       savelog -c 64 -p /srv/wanna-build/db/merge.log
 fi
 
 exec 3<&1 >> /srv/wanna-build/db/merge.log 2>&1
@@ -80,13 +84,13 @@ for d in $DISTS; do
     for s in $SECTIONS; do
        if echo $d | grep -qv -- -security; then
            curl_index "$MASTER/$dist/$s/source/Sources.gz" "Sources-$d.$s.gz" "Sources.$d"
-           if [ "$d" != "unstable" ]; then
+           if [ "$d" != "unstable" -a "$d" != "experimental" ]; then
                curl_index "$MASTER/$dist-proposed-updates/$s/source/Sources.gz" "Sources-$d-proposed-updates.$s.gz" "Sources.$d-p-u"
            fi
 
            for a in $ARCHS; do
                curl_index "$MASTER/$dist/$s/binary-$a/Packages.gz" "Packages-$d.$s.$a.gz" "Packages.$d.$a"
-               if [ "$d" != "unstable" ]; then
+               if [ "$d" != "unstable" -a "$d" != "experimental" ]; then
                    curl_index "$MASTER/$dist-proposed-updates/$s/binary-$a/Packages.gz" "Packages-$d-proposed-updates.$s.$a.gz" "Packages.$d-p-u.$a"
                    if [ "$d" != "oldstable" -o "$s" = "main" ]; then
                        curl_index "$MASTER/$dist-proposed-updates/$s/debian-installer/binary-$a/Packages.gz" "Packages-$d-proposed-updates-debian-installer.$s.$a.gz" "Packages.$d-p-u.$a"
@@ -117,15 +121,20 @@ for d in $DISTS; do
 # double builds.
 #
     if [ "$d" = "unstable" ]; then
-       rm -f "Sources.$d.base"
-       cp "Sources.$d" "Sources.$d.base"
-       curl_index "$MASTERBUILDD/Sources.gz" "Sources-$d.accepted.gz" "Sources.$d"
+       mv "Sources.$d" "Sources.$d.base"
+       curl_index "$MASTERBUILDD/Sources.gz" "Sources-$d.accepted.gz" "/dev/null"
+       rm -f "Sources-$d.accepted"
+       gunzip "Sources-$d.accepted.gz"
+       keep-latest source "Sources.$d.base" "Sources-$d.accepted" > "Sources.$d"
+
        curl_index "$MASTERBUILDD/Packages.gz" "Packages-$d.accepted.gz" "/dev/null"
+       rm -f "Packages-$d.accepted"
+       gunzip "Packages-$d.accepted.gz"
 
        for a in $ARCHS; do
-           rm -f "Packages.$d.$a.base"
-           cp "Packages.$d.$a" "Packages.$d.$a.base"
+           mv "Packages.$d.$a" "Packages.$d.$a.base"
            gzip -dc "Packages-$d.accepted.gz" >> "Packages.$d.$a"
+           keep-latest $a "Packages.$d.$a.base" "Packages-$d.accepted" > "Packages.$d.$a"
        done
     else
        if [ -e "Sources.$d-p-u" ]; then
@@ -136,7 +145,7 @@ for d in $DISTS; do
     for a in $ARCHS; do
        quinn-diff -A $a -a /srv/buildd.debian.org/web/quinn-diff/$dist/Packages-arch-specific -s Sources.$d -p Packages.$d.$a >> quinn-$d.$a 2>/dev/null
        if echo $d | grep -qv -- -security; then
-           if [ "$d" != "unstable" ]; then
+           if [ "$d" != "unstable" -a "$d" != "experimental" ]; then
                cat "Packages.$d.$a" >> "Packages.$d-p-u.$a"
                quinn-diff -A $a -a /srv/buildd.debian.org/web/quinn-diff/$dist/Packages-arch-specific -s Sources.$d-p-u -p Packages.$d-p-u.$a >> quinn-$d-p-u.$a 2>/dev/null
                sed -e 's/\[[-a-z]*:[-a-z]*\]$//' quinn-$d.$a > quinn-$d.$a.grep
@@ -158,9 +167,12 @@ echo "fetch and quinn completed: `date -u`"
 echo "fetch and quinn completed: `date -u`" >&3
 
 umask 002
-for a in $ARCHS_unstable ; do
-       wanna-build --create-maintenance-lock --database=$a/build-db
-
+# Gross hack: we need a list of all architectures, because this loop
+# insists on iterating on arches and then suites, instead of the
+# opposite.
+ALL_ARCHES=$(echo $ARCHS_oldstable $ARCHS_stable $ARCHS_testing $ARCHS_unstable \
+            | tr ' ' '\n' | sort -u)
+for a in $ALL_ARCHES ; do
        for d in $DISTS ; do
                dist=`echo $d | sed s/-.*$//`
                case "$dist" in
@@ -197,10 +209,6 @@ for a in $ARCHS_unstable ; do
                mv Packages.$d.$a Packages.$d.$a-old
                mv quinn-$d.$a quinn-$d.$a-old
        done
-       if [ "$DAY" = "0" ]; then
-               savelog -p -c 26 /srv/wanna-build/db/$a/transactions.log
-       fi
-       wanna-build --remove-maintenance-lock --database=$a/build-db
 done
 umask 022
 for d in $DISTS; do
@@ -208,24 +216,7 @@ for d in $DISTS; do
 done
 
 echo "merge ended: `date`"
-#
-# Only update stats if it's been at least 20h since the last time.
-#
-interval=72000
-last=`stat --format="%Y" /srv/wanna-build/etc/graph-data`
-now=`date +%s`
-if (( $last + $interval < $now )); then
-        echo "stats start: `date`"
-       /srv/wanna-build/bin/wb-graph >> /srv/wanna-build/etc/graph-data
-       /srv/wanna-build/bin/wb-graph -p >> /srv/wanna-build/etc/graph2-data
-       rm -f "$LOCKFILE"
-       trap - 0
-       /srv/buildd.debian.org/bin/makegraph
-       for a in $ARCHS_stable; do
-           echo Last Updated: `date -u` > /srv/buildd.debian.org/web/stats/$a.txt
-           for d in $STATS_DISTS; do
-               /srv/wanna-build/bin/wanna-build-statistics --database=$a/build-db --dist=$d >> /srv/buildd.debian.org/web/stats/$a.txt
-           done
-       done
-        echo "stats ended: `date`"
-fi
+
+save-database post
+do_stats
+