#!/bin/bash
#
# Updates wanna-build databases after the archive maintenance
# finishes
#
# Files:
#     Sources-* == upstream fetched file
#     Sources.* == uncompressed, concat'd version

LANG=C
PATH="/bin:/usr/bin"
#testing must be before unstable so late upld don't build for testing needlessly
#DISTS="oldstable-security stable-security testing-security oldstable stable testing unstable"
DISTS="stable-security testing-security stable testing unstable"
STATS_DISTS="unstable testing stable"
SECTIONS="main contrib non-free"
TMPDIR="/org/wanna-build/tmp"
LIBTRIGGER="/org/wanna-build/libtrigger.sh"
LOCKFILE="/org/wanna-build/tmp/DB_Maintenance_In_Progress"
MASTER="http://incoming.debian.org/debian/dists"
MASTERBUILDD="http://incoming.debian.org/buildd"
SECMASTER="http://security-master.debian.org/debian-security/dists"
SECMASTERBUILDD=http://security-master.debian.org/buildd
NEWARCH=""

. "$LIBTRIGGER"

DAY=`date +%w`

if lockfile -! -l 3600 $LOCKFILE; then
    echo "Cannot lock $LOCKFILE"
    exit 1
fi

cleanup() {
    rm -f "$LOCKFILE"
}
trap cleanup 0

echo Updating wanna-build databases...
umask 027

if [ "$DAY" = "0" ]; then
	savelog -c 26 -p /srv/wanna-build/db/merge.log
fi

exec 3<&1 >> /srv/wanna-build/db/merge.log 2>&1

echo -------------------------------------------------------------------------
echo "merge triggered: `date`"
echo "merge triggered          : `date -u`" >&3

cd $TMPDIR

#
# Make one big Packages and Sources file.
#
for d in $DISTS; do
    dist=`echo $d | sed s/-.*$//`
    case "$dist" in
	oldstable)
	    ARCHS="$ARCHS_oldstable"
	    ;;
	stable)
	    ARCHS="$ARCHS_stable"
	    ;;
	testing)
	    ARCHS="$ARCHS_testing"
	    ;;
	*)
	    ARCHS="$ARCHS_unstable"
	    ;;
    esac
    rm -f Sources.$d Sources.$d-p-u

    for a in $ARCHS; do
	rm -f Packages.$d.$a Packages.$d-p-u.$a quinn-$d-p-u.$a
    done

    for s in $SECTIONS; do
	if echo $d | grep -qv -- -security; then
	    curl_index "$MASTER/$dist/$s/source/Sources.gz" "Sources-$d.$s.gz" "Sources.$d"
	    if [ "$d" != "unstable" ]; then
		curl_index "$MASTER/$dist-proposed-updates/$s/source/Sources.gz" "Sources-$d-proposed-updates.$s.gz" "Sources.$d-p-u"
	    fi

	    for a in $ARCHS; do
		curl_index "$MASTER/$dist/$s/binary-$a/Packages.gz" "Packages-$d.$s.$a.gz" "Packages.$d.$a"
		if [ "$d" != "unstable" ]; then
		    curl_index "$MASTER/$dist-proposed-updates/$s/binary-$a/Packages.gz" "Packages-$d-proposed-updates.$s.$a.gz" "Packages.$d-p-u.$a"
		    if [ "$d" != "oldstable" -o "$s" = "main" ]; then
			curl_index "$MASTER/$dist-proposed-updates/$s/debian-installer/binary-$a/Packages.gz" "Packages-$d-proposed-updates-debian-installer.$s.$a.gz" "Packages.$d-p-u.$a"
		    fi
		fi
		if [ "$d" != "oldstable" -o "$s" = "main" ]; then
		    curl_index "$MASTER/$d/$s/debian-installer/binary-$a/Packages.gz" "Packages-$d-debian-installer.$s.$a.gz" "Packages.$d.$a"
		fi
	    done
	else
	    curl_index "$SECMASTER/$dist/updates/$s/source/Sources.gz" "Sources-$d.$s.gz" "Sources.$d"
	    if [ "$s" = "main" ]; then
		curl_index "$SECMASTERBUILDD/$dist/Sources.gz" "Sources-$d.accepted.gz" "Sources.$d"
		curl_index "$SECMASTERBUILDD/$dist/Packages.gz" "Packages-$d.accepted.gz" "/dev/null"
	    fi
	    for a in $ARCHS ; do
		curl_index "$SECMASTER/$dist/updates/$s/binary-$a/Packages.gz" "Packages-$d.$s.$a.gz" "Packages.$d.$a"
		if [ "$s" = "main" ]; then
		    gzip -dc "Packages-$d.accepted.gz" >> "Packages.$d.$a"
		fi
	    done
	fi
    done

#
# It's important that the accepted file is rebuilt before the daily run,
# and that you have that should-be-empty version, to avoid unaccepts and
# double builds.
#
    if [ "$d" = "unstable" ]; then
	rm -f "Sources.$d.base"
	cp "Sources.$d" "Sources.$d.base"
	curl_index "$MASTERBUILDD/Sources.gz" "Sources-$d.accepted.gz" "Sources.$d"
	curl_index "$MASTERBUILDD/Packages.gz" "Packages-$d.accepted.gz" "/dev/null"

	for a in $ARCHS; do
	    rm -f "Packages.$d.$a.base"
	    cp "Packages.$d.$a" "Packages.$d.$a.base"
	    gzip -dc "Packages-$d.accepted.gz" >> "Packages.$d.$a"
	done
    else
	if [ -e "Sources.$d-p-u" ]; then
	    cat "Sources.$d" >> "Sources.$d-p-u"
	fi
    fi

    for a in $ARCHS; do
	quinn-diff -A $a -a /srv/buildd.debian.org/web/quinn-diff/$dist/Packages-arch-specific -s Sources.$d -p Packages.$d.$a >> quinn-$d.$a 2>/dev/null
	if echo $d | grep -qv -- -security; then
	    if [ "$d" != "unstable" ]; then
		cat "Packages.$d.$a" >> "Packages.$d-p-u.$a"
		quinn-diff -A $a -a /srv/buildd.debian.org/web/quinn-diff/$dist/Packages-arch-specific -s Sources.$d-p-u -p Packages.$d-p-u.$a >> quinn-$d-p-u.$a 2>/dev/null
		sed -e 's/\[[-a-z]*:[-a-z]*\]$//' quinn-$d.$a > quinn-$d.$a.grep
		if [ ! -z quinn-$d.$a.grep ]; then
		    fgrep -vf quinn-$d.$a.grep quinn-$d-p-u.$a > quinn-$d.$a
		else
		    cp quinn-$d-p-u.$a quinn-$d.$a
		fi
		rm quinn-$d.$a.grep
		mv "Packages.$d-p-u.$a" "Packages.$d.$a"
	    fi
	fi
    done
    if [ -e "Sources.$d-p-u" ]; then
	mv "Sources.$d-p-u" "Sources.$d"
    fi
done
echo "fetch and quinn completed: `date -u`"
echo "fetch and quinn completed: `date -u`" >&3

umask 002
# Gross hack: we need a list of all architectures, because this loop
# insists on iterating on arches and then suites, instead of the
# opposite.
ALL_ARCHES=$(echo $ARCHS_oldstable $ARCHS_stable $ARCHS_testing $ARCHS_unstable \
	     | tr ' ' '\n' | sort -u)
for a in $ALL_ARCHES ; do
	wanna-build --create-maintenance-lock --database=$a/build-db

	for d in $DISTS ; do
		dist=`echo $d | sed s/-.*$//`
		case "$dist" in
			oldstable)
				if echo $ARCHS_oldstable | grep -q -v "\b$a\b"; then
					continue
				fi
				;;
			stable)
				if echo $ARCHS_stable | grep -q -v "\b$a\b"; then
					continue
				fi
				;;
			testing)
				if echo $ARCHS_testing | grep -q -v "\b$a\b"; then
					continue
				fi
				;;
			*)
				if echo $ARCHS_unstable | grep -q -v "\b$a\b"; then
					continue
				fi
				;;
		esac
		# non-free is excluded for legal reasons.  buildd needs to
		# confirm on a per-package and per-buildd basis whether or not
		# it is buildable
		perl -pi -e 's#^(non-free)/.*$##msg' quinn-$d.$a
		if [ "$a" = "$NEWARCH" ]; then
		wanna-build -v --merge-all --arch=$a --dist=$d --database=$a/build-db Packages.$d.$a quinn-$d.$a Sources.unstable.$NEWARCH
		else
		wanna-build -v --merge-all --arch=$a --dist=$d --database=$a/build-db Packages.$d.$a quinn-$d.$a Sources.$d
		fi
		mv Packages.$d.$a Packages.$d.$a-old
		mv quinn-$d.$a quinn-$d.$a-old
	done
	if [ "$DAY" = "0" ]; then
		savelog -p -c 26 /srv/wanna-build/db/$a/transactions.log
	fi
	wanna-build --remove-maintenance-lock --database=$a/build-db
done
umask 022
for d in $DISTS; do
    mv Sources.$d Sources.$d-old
done

echo "merge ended: `date`"
#
# Only update stats if it's been at least 20h since the last time.
#
interval=72000
last=`stat --format="%Y" /srv/wanna-build/etc/graph-data`
now=`date +%s`
if (( $last + $interval < $now )); then
        echo "stats start: `date`"
	/srv/wanna-build/bin/wb-graph >> /srv/wanna-build/etc/graph-data
	/srv/wanna-build/bin/wb-graph -p >> /srv/wanna-build/etc/graph2-data
	rm -f "$LOCKFILE"
	trap - 0
	/srv/buildd.debian.org/bin/makegraph
	for a in $ARCHS_stable; do
	    echo Last Updated: `date -u` > /srv/buildd.debian.org/web/stats/$a.txt
	    for d in $STATS_DISTS; do
		/srv/wanna-build/bin/wanna-build-statistics --database=$a/build-db --dist=$d >> /srv/buildd.debian.org/web/stats/$a.txt
	    done
	done
        echo "stats ended: `date`"
fi