-#!/bin/bash
+#!/bin/bash -x
#
# Updates wanna-build databases after the archive maintenance
# finishes
# Sources.* == uncompressed, concat'd version
LANG=C
-PATH="/bin:/usr/bin"
+PATH="/bin:/usr/bin:/org/wanna-build/bin"
#testing must be before unstable so late upld don't build for testing needlessly
-#DISTS="oldstable-security stable-security testing-security oldstable stable testing unstable"
-DISTS="stable-security testing-security stable testing unstable"
+DISTS="oldstable-security stable-security testing-security oldstable stable testing unstable"
+# While oldstable is purged
+#DISTS="stable-security testing-security stable testing unstable"
STATS_DISTS="unstable testing stable"
SECTIONS="main contrib non-free"
-ARCHS_oldstable="m68k arm sparc alpha powerpc i386 mips mipsel ia64 hppa s390"
-ARCHS_stable="arm sparc alpha powerpc i386 amd64 mips mipsel ia64 hppa s390"
-ARCHS_testing="$ARCHS_stable armel"
-ARCHS_unstable="$ARCHS_testing"
TMPDIR="/org/wanna-build/tmp"
+LIBTRIGGER="/org/wanna-build/libtrigger.sh"
LOCKFILE="/org/wanna-build/tmp/DB_Maintenance_In_Progress"
MASTER="http://incoming.debian.org/debian/dists"
MASTERBUILDD="http://incoming.debian.org/buildd"
SECMASTER="http://security-master.debian.org/debian-security/dists"
SECMASTERBUILDD=http://security-master.debian.org/buildd
-CURLOPT="-q -s -S -R -f -Y 10 -y 120 -K /srv/wanna-build/trigger.curlrc"
NEWARCH=""
-curl_index () {
- local url tmpname destname appendname curlopt
- url="$1"
- destname="$2"
- appendname="$3"
- tmpname=".$destname"
- rc=0
- rm -f "$tmpname"
- if [ -e "$destname" ]; then
- refdate=`perl -e "print scalar gmtime(((stat ('$destname'))[9]))"`
- curl $CURLOPT -z "$refdate" "$url" -o "$tmpname"
- rc=$?
- else
- curl $CURLOPT "$url" -o "$tmpname"
- rc=$?
- fi
- if [ $rc -eq 0 -a -e "$tmpname" ]; then
- if gzip -t "$tmpname"; then
- mv "$tmpname" "$destname"
- else
- rc=$?
- fi
- fi
- gzip -dc "$destname" >> "$appendname"
- return $rc
-}
+. "$LIBTRIGGER"
DAY=`date +%w`
}
trap cleanup 0
+echo Dumping wanna-build databases...
+sudo -u postgres /usr/bin/pg_dumpall --cluster 8.4/wanna-build | bzip2 > /org/wanna-build/dumps/dump_pre_$(date +%Y.%m.%d-%H:%M:%S).bz2
+
echo Updating wanna-build databases...
umask 027
if [ "$DAY" = "0" ]; then
- savelog -c 26 -p /srv/wanna-build/db/merge.log
+ savelog -c 64 -p /srv/wanna-build/db/merge.log
fi
exec 3<&1 >> /srv/wanna-build/db/merge.log 2>&1
# double builds.
#
if [ "$d" = "unstable" ]; then
- rm -f "Sources.$d.base"
- cp "Sources.$d" "Sources.$d.base"
- curl_index "$MASTERBUILDD/Sources.gz" "Sources-$d.accepted.gz" "Sources.$d"
+ mv "Sources.$d" "Sources.$d.base"
+ curl_index "$MASTERBUILDD/Sources.gz" "Sources-$d.accepted.gz" "/dev/null"
+ rm -f "Sources-$d.accepted"
+ gunzip "Sources-$d.accepted.gz"
+ keep-latest source "Sources.$d.base" "Sources-$d.accepted" > "Sources.$d"
+
curl_index "$MASTERBUILDD/Packages.gz" "Packages-$d.accepted.gz" "/dev/null"
+ rm -f "Packages-$d.accepted"
+ gunzip "Packages-$d.accepted.gz"
for a in $ARCHS; do
- rm -f "Packages.$d.$a.base"
- cp "Packages.$d.$a" "Packages.$d.$a.base"
+ mv "Packages.$d.$a" "Packages.$d.$a.base"
gzip -dc "Packages-$d.accepted.gz" >> "Packages.$d.$a"
+ keep-latest $a "Packages.$d.$a.base" "Packages-$d.accepted" > "Packages.$d.$a"
done
else
if [ -e "Sources.$d-p-u" ]; then
fi
for a in $ARCHS; do
- quinn-diff -A $a -a /srv/buildd.debian.org/web/quinn-diff/Packages-arch-specific -s Sources.$d -p Packages.$d.$a >> quinn-$d.$a 2>/dev/null
+ quinn-diff -A $a -a /srv/buildd.debian.org/web/quinn-diff/$dist/Packages-arch-specific -s Sources.$d -p Packages.$d.$a >> quinn-$d.$a 2>/dev/null
if echo $d | grep -qv -- -security; then
if [ "$d" != "unstable" ]; then
cat "Packages.$d.$a" >> "Packages.$d-p-u.$a"
- quinn-diff -A $a -a /srv/buildd.debian.org/web/quinn-diff/Packages-arch-specific -s Sources.$d-p-u -p Packages.$d-p-u.$a >> quinn-$d-p-u.$a 2>/dev/null
+ quinn-diff -A $a -a /srv/buildd.debian.org/web/quinn-diff/$dist/Packages-arch-specific -s Sources.$d-p-u -p Packages.$d-p-u.$a >> quinn-$d-p-u.$a 2>/dev/null
sed -e 's/\[[-a-z]*:[-a-z]*\]$//' quinn-$d.$a > quinn-$d.$a.grep
if [ ! -z quinn-$d.$a.grep ]; then
fgrep -vf quinn-$d.$a.grep quinn-$d-p-u.$a > quinn-$d.$a
echo "fetch and quinn completed: `date -u`" >&3
umask 002
-for a in $ARCHS_unstable ; do
- wanna-build --create-maintenance-lock --database=$a/build-db
-
+# Gross hack: we need a list of all architectures, because this loop
+# insists on iterating on arches and then suites, instead of the
+# opposite.
+ALL_ARCHES=$(echo $ARCHS_oldstable $ARCHS_stable $ARCHS_testing $ARCHS_unstable \
+ | tr ' ' '\n' | sort -u)
+for a in $ALL_ARCHES ; do
for d in $DISTS ; do
dist=`echo $d | sed s/-.*$//`
case "$dist" in
mv Packages.$d.$a Packages.$d.$a-old
mv quinn-$d.$a quinn-$d.$a-old
done
- if [ "$DAY" = "0" ]; then
- savelog -p -c 26 /srv/wanna-build/db/$a/transactions.log
- fi
- wanna-build --remove-maintenance-lock --database=$a/build-db
done
umask 022
for d in $DISTS; do
done
echo "merge ended: `date`"
+
+sudo -u postgres /usr/bin/pg_dumpall --cluster 8.4/wanna-build | bzip2 > /org/wanna-build/dumps/dump_post_$(date +%Y.%m.%d-%H:%M:%S).bz2
+
+(cd /org/wanna-build/dumps && /org/wanna-build/expire_dumps -d . -f "dump_*")
+
#
# Only update stats if it's been at least 20h since the last time.
#
rm -f "$LOCKFILE"
trap - 0
/srv/buildd.debian.org/bin/makegraph
- for a in $ARCHS_stable; do
+ for a in $ALL_ARCHES; do
echo Last Updated: `date -u` > /srv/buildd.debian.org/web/stats/$a.txt
for d in $STATS_DISTS; do
/srv/wanna-build/bin/wanna-build-statistics --database=$a/build-db --dist=$d >> /srv/buildd.debian.org/web/stats/$a.txt
projects / wannabuild.git / blobdiff