From: Philipp Kern Date: Thu, 11 Feb 2010 19:58:05 +0000 (+0000) Subject: Rewrite triggers X-Git-Url: https://git.donarmstrong.com/?p=wannabuild.git;a=commitdiff_plain;h=e810b1094fa8c0f9fc4846d4121d079099b851a7 Rewrite triggers --- diff --git a/triggers/common b/triggers/common new file mode 100644 index 0000000..d48cbfc --- /dev/null +++ b/triggers/common @@ -0,0 +1,136 @@ +# vim:set et ts=4 sw=4 ft=bash ai: + +set -eE + +LANG=C +PATH="/bin:/usr/bin:/org/wanna-build/bin:/org/buildd.debian.org/bin:/org/wanna-build/triggers" +ARCHIVE_MAIN="/org/wanna-build/tmp/archive/debian" +ARCHIVE_BASE="/org/wanna-build/tmp/archive/${ARCHIVE}" +PAS_BASE="/org/buildd.debian.org/web/quinn-diff" +LOCKFILE="${ARCHIVE_BASE}/lock" +ARCHS_etch="alpha amd64 arm hppa i386 ia64 mips mipsel powerpc s390 sparc" +ARCHS_lenny="alpha amd64 arm armel hppa i386 ia64 mips mipsel powerpc s390 sparc" +ARCHS_squeeze="amd64 armel hppa i386 ia64 mips mipsel powerpc s390 sparc kfreebsd-amd64 kfreebsd-i386" +ARCHS_unstable="$ARCHS_squeeze alpha hurd-i386" + +# Creates a working environment within the corresponding archive +# directory and jumps there. +ensure_workdir() { + mkdir -p "${ARCHIVE_BASE}/workdir" + cd "${ARCHIVE_BASE}/workdir" +} + +# Puts the list of architectures for a given suite onto stdout. +get_architectures() { + SUITE="$1" + + if [ "$SUITE" = "etch" -o "$SUITE" = "oldstable" ]; then echo $ARCHS_etch + elif [ "$SUITE" = "lenny" -o "$SUITE" = "stable" ]; then echo $ARCHS_lenny + elif [ "$SUITE" = "squeeze" -o "$SUITE" = "testing" ]; then echo $ARCHS_squeeze + elif [ "$SUITE" = "unstable" -o "$SUITE" = "sid" ]; then echo $ARCHS_unstable + elif [ "$SUITE" = "experimental" ]; then echo $ARCHS_unstable + else + echo "Unknown suite encountered, aborting." >2 + exit 1 + fi +} + +# Runs quinn-diff with the right arguments. +do_quinn() { + SUITE="$1" + ARCH="$2" + WHITELIST_FILE="$3" + quinn-diff \ + -A $ARCH \ + -a "$PAS_BASE/$SUITE/Packages-arch-specific" \ + -s Sources.$SUITE \ + -p Packages.$SUITE.$ARCH \ + | filter-q-d "${WHITELIST_FILE}" > quinn-$SUITE.$ARCH 2>/dev/null +} + +# Call wanna-build's importing function. WB_SUITE might differ from SUITE +# for other archives than the main one. +do_wb_merge_all() { + SUITE="$1" + ARCH="$2" + WB_SUITE="$3" + OPTIONS="$4" + + if [ -z "$WB_SUITE" ] + then + WB_SUITE="$SUITE" + fi + + if [ "$4" != "overlay" ] + then + wanna-build -v --merge-all --arch=$ARCH --dist=$WB_SUITE \ + Packages.$SUITE.$ARCH quinn-$SUITE.$ARCH Sources.$SUITE + else + wanna-build -v --merge-all --arch=$ARCH --dist=$WB_SUITE \ + Packages.$SUITE.$ARCH-all quinn-$SUITE.$ARCH Sources.$SUITE-all + fi +} + +cleanup() { + rm -rf "$LOCKFILE" +} + +ensure_lock() { + if lockfile -! -r 10 -l 6000 "$LOCKFILE" + then + echo "Cannot lock $LOCKFILE, aborting." + exit 1 + fi +} + +trigger_wb_update() { + SUITE="$1" + ARCHS="$2" + SOURCES="$3" + PACKAGES="$4" + WB_SUITE="$5" + + if [ -z "$WB_SUITE" ] + then + WB_SUITE="${SUITE}" + fi + + echo "`date`: Processing ${ARCHIVE}/${SUITE} ..." + + new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE} + parallel -l 5 -i trigger_wb_update_arch_loop "${SUITE}" "{}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" -- ${ARCHS} + #for ARCH in ${ARCHS}; do + #trigger_wb_update_arch_loop "${SUITE}" "${ARCH}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" + #done + + echo "`date`: ${ARCHIVE}/${SUITE} done" +} + +trigger_wb_update_for_overlay() { + SUITE="$1" + ARCHS="$2" + SOURCES_BASE="$3" + PACKAGES_BASE="$4" + SOURCES="$5" + PACKAGES="$6" + WB_SUITE="$7" + WHITELIST_FILE="$8" + + if [ -z "$WB_SUITE" ] + then + WB_SUITE="${SUITE}" + fi + + echo "`date`: Processing ${ARCHIVE}/${SUITE} ..." + + new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE} + new-keep-latest source $(eval echo Sources.${SUITE} ${SOURCES_BASE}) > Sources.${SUITE}-all + + parallel -l 5 -i trigger_wb_update_for_overlay_arch_loop "${SUITE}" "{}" "${SOURCES_BASE}" "${PACKAGES_BASE}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" "${WHITELIST_FILE}" -- ${ARCHS} + #for ARCH in ${ARCHS}; do + #trigger_wb_update_for_overlay_arch_loop "${SUITE}" "${ARCH}" "${SOURCES_BASE}" "${PACKAGES_BASE}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" "${WHITELIST_FILE}" + #done + + echo "`date`: ${ARCHIVE}/${SUITE} done" +} + diff --git a/triggers/trigger.backports b/triggers/trigger.backports new file mode 100755 index 0000000..a9d35bb --- /dev/null +++ b/triggers/trigger.backports @@ -0,0 +1,49 @@ +#!/bin/bash +# vim:set et ts=4 sw=4 ft=bash ai: + +ARCHIVE="backports" + +. /org/wanna-build/triggers/common + +exec >> /org/wanna-build/db/merge.$ARCHIVE.log 2>&1 + +if [ -f /org/wanna-build/NO-TRIGGERS ] +then + echo Trigger for $ARCHIVE skipped due to NO-TRIGGERS, aborting. >&2 + exit 0 +fi + +echo "`date`: Running trigger for $ARCHIVE ..." + +main() { + set -eE + trap cleanup ERR TERM HUP INT QUIT + + ensure_lock + ensure_workdir + + # Fetch the most recent Packages and Sources files. + sync.sh debian + sync.sh $ARCHIVE nolock + + # Autodetect suites by inspecting the directories in dists/. + SUITES="$(find ${ARCHIVE_BASE}/archive/ -maxdepth 1 -mindepth 1 -type d | \ + xargs -n 1 basename | \ + sort | uniq)" + + for suite in $SUITES + do + base_suite=${suite%%-backports} + SOURCES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib}/source/Sources.gz" + PACKAGES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib}/binary-%ARCH%/Packages.gz" + SOURCES_BASE="${ARCHIVE_MAIN}/archive/${base_suite}/{main,contrib}/source/Sources.gz" + PACKAGES_BASE="${ARCHIVE_MAIN}/archive/${base_suite}/{main,contrib}/binary-%ARCH%/Packages.gz" + trigger_wb_update_for_overlay "$base_suite" "`get_architectures $base_suite`" "$SOURCES_BASE" "$PACKAGES_BASE" "$SOURCES" "$PACKAGES" "$suite" + done + + cleanup +} + +main & +exit 0 + diff --git a/triggers/trigger.debian b/triggers/trigger.debian new file mode 100755 index 0000000..c433006 --- /dev/null +++ b/triggers/trigger.debian @@ -0,0 +1,84 @@ +#!/bin/bash -x +# vim:set et ts=4 sw=4 ft=sh ai: + +ARCHIVE="debian" +SUITES="oldstable stable testing" +# Please note that SUITES_FAST_AUTOBUILD does not pull proposed-updates +# because there was no need for that yet. +SUITES_FAST_AUTOBUILD="unstable experimental" + +. /org/wanna-build/triggers/common + +exec >> /org/wanna-build/db/merge.$ARCHIVE.log 2>&1 + +if [ -f /org/wanna-build/NO-TRIGGERS ] +then + echo Trigger for $ARCHIVE skipped due to NO-TRIGGERS, aborting. >&2 + exit 0 +fi + +echo "`date`: Running trigger for $ARCHIVE ..." + +main_sync() { + ensure_lock + ensure_workdir + + # Fetch the most recent Packages and Sources files. + sync.sh $ARCHIVE nolock + + for SUITE in $SUITES + do + # There's one sources file for -proposed-updates. There are packages + # files for -proposed-updates itself and additionally for + # debian-installer. + SOURCES="${ARCHIVE_BASE}/archive/${SUITE}-proposed-updates/{main,contrib}/source/Sources.gz" + PACKAGES="${ARCHIVE_BASE}/archive/${SUITE}-proposed-updates/{main,contrib}/binary-%ARCH%/Packages.gz" + PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/archive/${SUITE}-proposed-updates/{main,contrib}/debian-installer/binary-%ARCH%/Packages.gz" + # Now add the overlay suite as above. + SOURCES_BASE="${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib}/source/Sources.gz" + PACKAGES_BASE="${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib}/binary-%ARCH%/Packages.gz" + PACKAGES_BASE="${PACKAGES_BASE} ${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib}/debian-installer/binary-%ARCH%/Packages.gz" + trigger_wb_update_for_overlay "$SUITE" "`get_architectures $SUITE`" "$SOURCES_BASE" "$PACKAGES_BASE" "$SOURCES" "$PACKAGES" + done + + for SUITE in $SUITES_FAST_AUTOBUILD + do + SOURCES="${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib}/source/Sources.gz" + SOURCES="${SOURCES} ${ARCHIVE_BASE}/buildd-${SUITE}/Sources.gz" + PACKAGES="${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib}/binary-%ARCH%/Packages.gz" + PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib}/debian-installer/binary-%ARCH%/Packages.gz" + PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/buildd-${SUITE}/Packages.gz" + + if [ "$SUITE" != "experimental" ] + then + # The "it's not an overlay" part. + trigger_wb_update "${SUITE}" "`get_architectures ${SUITE}`" "$SOURCES" "$PACKAGES" + else + BASE_SUITE=unstable + SOURCES_BASE="${ARCHIVE_BASE}/archive/${BASE_SUITE}/{main,contrib}/source/Sources.gz" + SOURCES_BASE="${SOURCES_BASE} ${ARCHIVE_BASE}/buildd-${BASE_SUITE}/Sources.gz" + PACKAGES_BASE="${ARCHIVE_MAIN}/archive/${BASE_SUITE}/{main,contrib}/binary-%ARCH%/Packages.gz" + PACKAGES_BASE="${PACKAGES_BASE} ${ARCHIVE_MAIN}/archive/${BASE_SUITE}/{main,contrib}/debian-installer/binary-%ARCH%/Packages.gz" + PACKAGES_BASE="${PACKAGES_BASE} ${ARCHIVE_MAIN}/buildd-${BASE_SUITE}/Packages.gz" + trigger_wb_update_for_overlay "$SUITE" "`get_architectures $SUITE`" "$SOURCES_BASE" "$PACKAGES_BASE" "$SOURCES" "$PACKAGES" + fi + done + + cleanup +} + +main() { + set -eE + trap cleanup ERR TERM HUP INT QUIT + + save-database pre + + main_sync + + save-database post + do_stats +} + +main & +exit 0 + diff --git a/triggers/trigger.debian-buildd b/triggers/trigger.debian-buildd new file mode 100755 index 0000000..ff398d6 --- /dev/null +++ b/triggers/trigger.debian-buildd @@ -0,0 +1,55 @@ +#!/bin/bash +# vim:set et ts=4 sw=4 ft=bash ai: + +ARCHIVE="debian" + +. /org/wanna-build/triggers/common + +exec >> /org/wanna-build/db/merge.$ARCHIVE.log 2>&1 + +if [ -f /org/wanna-build/NO-TRIGGERS ] +then + echo Trigger for $ARCHIVE skipped due to NO-TRIGGERS, aborting. >&2 + exit 0 +fi + +echo "`date`: Running trigger for $ARCHIVE-buildd ..." + +main() { + set -eE + trap cleanup ERR TERM HUP INT QUIT + + ensure_lock + ensure_workdir + + # Fetch the most recent Packages and Sources files. + sync.sh $ARCHIVE nolock + + # Handle unstable as a normal suite. + suite=unstable + SOURCES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib}/source/Sources.gz" + SOURCES="${SOURCES} ${ARCHIVE_BASE}/buildd-${suite}/Sources.gz" + PACKAGES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib}/binary-%ARCH%/Packages.gz" + PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/archive/${suite}/main/debian-installer/binary-%ARCH%/Packages.gz" + PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/buildd-${suite}/Packages.gz" + trigger_wb_update "$suite" "`get_architectures $suite`" "$SOURCES" "$PACKAGES" + + # Handle experimental as an overlay suite. + suite=experimental + base_suite=unstable + SOURCES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib}/source/Sources.gz" + SOURCES="${SOURCES} ${ARCHIVE_BASE}/buildd-${suite}/Sources.gz" + PACKAGES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib}/binary-%ARCH%/Packages.gz" + PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/buildd-${suite}/Packages.gz" + + SOURCES_BASE="${ARCHIVE_MAIN}/archive/${base_suite}/{main,contrib}/source/Sources.gz" + PACKAGES_BASE="${ARCHIVE_MAIN}/archive/${base_suite}/{main,contrib}/binary-%ARCH%/Packages.gz" + PACKAGES_BASE="${PACKAGES_BASE} ${ARCHIVE_MAIN}/archive/${base_suite}/main/debian-installer/binary-%ARCH%/Packages.gz" + trigger_wb_update_for_overlay "$suite" "`get_architectures $suite`" "$SOURCES_BASE" "$PACKAGES_BASE" "$SOURCES" "$PACKAGES" + + cleanup +} + +main & +exit 0 + diff --git a/triggers/trigger.edu b/triggers/trigger.edu new file mode 100755 index 0000000..a3f5bae --- /dev/null +++ b/triggers/trigger.edu @@ -0,0 +1,48 @@ +#!/bin/bash +# vim:set et ts=4 sw=4 ft=bash ai: + +ARCHIVE="debian-edu" +SUITES="lenny etch" +ARCHES="i386 amd64 powerpc" + +. /org/wanna-build/triggers/common + +exec >> /org/wanna-build/db/merge.$ARCHIVE.log 2>&1 + +if [ -f /org/wanna-build/NO-TRIGGERS ] +then + echo Trigger for $ARCHIVE skipped due to NO-TRIGGERS, aborting. >2 + exit 0 +fi + +echo "`date`: Running trigger for $ARCHIVE ..." + +main() { + set -eE + trap cleanup ERR TERM HUP INT QUIT + + ensure_lock + ensure_workdir + + # Fetch the most recent Packages and Sources files. + sync.sh debian + sync.sh $ARCHIVE nolock + + for suite in $SUITES + do + base_suite=$suite + SOURCES="${ARCHIVE_BASE}/archive/${suite}-test/local/source/Sources.gz" + PACKAGES="${ARCHIVE_BASE}/archive/${suite}-test/local/binary-%ARCH%/Packages.gz" + SOURCES_BASE="${ARCHIVE_MAIN}/archive/${base_suite}/{main,contrib}/source/Sources.gz" + SOURCES_BASE="${SOURCES_BASE} ${ARCHIVE_BASE}/archive/${suite}/local/source/Sources.gz" + PACKAGES_BASE="${ARCHIVE_MAIN}/archive/${base_suite}/{main,contrib}/binary-%ARCH%/Packages.gz" + PACKAGES_BASE="${PACKAGES_BASE} ${ARCHIVE_BASE}/archive/${suite}/local/binary-%ARCH%/Packages.gz" + trigger_wb_update_for_overlay "$base_suite" "$ARCHES" "$SOURCES_BASE" "$PACKAGES_BASE" "$SOURCES" "$PACKAGES" "$suite-edu" + done + + cleanup +} + +main & +exit 0 + diff --git a/triggers/trigger.security b/triggers/trigger.security new file mode 100755 index 0000000..012d94f --- /dev/null +++ b/triggers/trigger.security @@ -0,0 +1,58 @@ +#!/bin/bash +# vim:set et ts=4 sw=4 ft=sh ai: + +ARCHIVE="debian-security" + +set -e + +if [ -n "$SSH_ORIGINAL_COMMAND" ] +then + set -- $SSH_ORIGINAL_COMMAND + # The name of the script will be put into $1 instead of $0, + # so shift the arguments to discard our name. + shift +fi + +SUITE="$1" + +if [ -z "$SUITE" ] +then + echo "Missing suite to act upon, aborting." >&2 + exit 1 +fi + +. /org/wanna-build/triggers/common + +exec >> /org/wanna-build/db/merge.$ARCHIVE.log 2>&1 + +if [ -f /org/wanna-build/NO-TRIGGERS ] +then + echo Trigger for $ARCHIVE skipped due to NO-TRIGGERS, aborting. >&2 + exit 0 +fi + + +echo "`date`: Running trigger for $ARCHIVE/$SUITE ..." + +main() { + set -eE + trap cleanup ERR TERM HUP INT QUIT + + ensure_lock + ensure_workdir + + # Fetch the most recent Packages and Sources files. + sync.sh $ARCHIVE nolock + + SOURCES="${ARCHIVE_BASE}/archive/${SUITE}/updates/{main,contrib}/source/Sources.gz" + SOURCES="${SOURCES} ${ARCHIVE_BASE}/buildd-${SUITE}/Sources.gz" + PACKAGES="${ARCHIVE_BASE}/archive/${SUITE}/updates/{main,contrib}/binary-%ARCH%/Packages.gz" + PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/buildd-${SUITE}/Packages.gz" + trigger_wb_update "${SUITE}" "`get_architectures ${SUITE}`" "$SOURCES" "$PACKAGES" "${SUITE}-security" + + cleanup +} + +main +exit 0 + diff --git a/triggers/trigger.volatile b/triggers/trigger.volatile new file mode 100755 index 0000000..6aab693 --- /dev/null +++ b/triggers/trigger.volatile @@ -0,0 +1,52 @@ +#!/bin/bash +# vim:set et ts=4 sw=4 ft=bash ai: + +ARCHIVE="debian-volatile" + +. /org/wanna-build/triggers/common + +exec >> /org/wanna-build/db/merge.$ARCHIVE.log 2>&1 + +if [ -f /org/wanna-build/NO-TRIGGERS ] +then + echo Trigger for $ARCHIVE skipped due to NO-TRIGGERS, aborting. >2 + exit 0 +fi + +echo "`date`: Running trigger for $ARCHIVE ..." + +main() { + set -eE + trap cleanup ERR TERM HUP INT QUIT + + ensure_lock + ensure_workdir + + # Fetch the most recent Packages and Sources files. + sync.sh debian + sync.sh $ARCHIVE nolock + + # Autodetect suites by inspecting the directories in dists/. + SUITES="$(find ${ARCHIVE_BASE}/archive/ -maxdepth 1 -mindepth 1 -type d | \ + xargs -n 1 basename | \ + awk -F- '{print $1}'| \ + sort | uniq)" + + for suite in $SUITES + do + SOURCES="${ARCHIVE_BASE}/archive/${suite}/volatile/{main,contrib}/source/Sources.gz" + SOURCES="${SOURCES} ${ARCHIVE_BASE}/archive/${suite}-proposed-updates/volatile/{main,contrib}/source/Sources.gz" + PACKAGES="${ARCHIVE_BASE}/archive/${suite}/volatile/{main,contrib}/binary-%ARCH%/Packages.gz" + PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/archive/${suite}-proposed-updates/volatile/{main,contrib}/binary-%ARCH%/Packages.gz" + #As volatile is an overlay, also add Debian ${suite}: + SOURCES_BASE="${ARCHIVE_MAIN}/archive/${suite}/{main,contrib}/source/Sources.gz" + PACKAGES_BASE="${ARCHIVE_MAIN}/archive/${suite}/{main,contrib}/binary-%ARCH%/Packages.gz" + trigger_wb_update_for_overlay "$suite" "`get_architectures $suite`" "$SOURCES_BASE" "$PACKAGES_BASE" "$SOURCES" "$PACKAGES" "$suite-volatile" + done + + cleanup +} + +main & +exit 0 + diff --git a/triggers/trigger_wb_update_arch_loop b/triggers/trigger_wb_update_arch_loop new file mode 100755 index 0000000..7d4473e --- /dev/null +++ b/triggers/trigger_wb_update_arch_loop @@ -0,0 +1,16 @@ +#!/bin/bash -x + +set -eE + +. /org/wanna-build/triggers/common + +SUITE="$1" +ARCH="$2" +SOURCES="$3" +PACKAGES="$4" +WB_SUITE="$5" + +new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH} +do_quinn ${SUITE} ${ARCH} +do_wb_merge_all ${SUITE} ${ARCH} ${WB_SUITE} + diff --git a/triggers/trigger_wb_update_for_overlay_arch_loop b/triggers/trigger_wb_update_for_overlay_arch_loop new file mode 100755 index 0000000..8981db1 --- /dev/null +++ b/triggers/trigger_wb_update_for_overlay_arch_loop @@ -0,0 +1,20 @@ +#!/bin/bash -x + +set -eE + +. /org/wanna-build/triggers/common + +SUITE="$1" +ARCH="$2" +SOURCES_BASE="$3" +PACKAGES_BASE="$4" +SOURCES="$5" +PACKAGES="$6" +WB_SUITE="$7" +WHITELIST_FILE="$8" + +new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH} +new-keep-latest ${ARCH} Packages.${SUITE}.${ARCH} $(eval echo $(echo ${PACKAGES_BASE} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}-all +do_quinn ${SUITE} ${ARCH} "${WHITELIST_FILE}" +do_wb_merge_all ${SUITE} ${ARCH} ${WB_SUITE} overlay +