trigger.daily

   1 #!/bin/bash
   2 #
   3 # Updates wanna-build databases after the archive maintenance
   4 # finishes
   5 #
   6 # Files:
   7 #     Sources-* == upstream fetched file
   8 #     Sources.* == uncompressed, concat'd version
   9
  10 LANG=C
  11 PATH="/bin:/usr/bin"
  12 #testing must be before unstable so late upld don't build for testing needlessly
  13 #DISTS="oldstable-security stable-security testing-security oldstable stable testing unstable"
  14 DISTS="stable-security testing-security stable testing unstable"
  15 STATS_DISTS="unstable testing stable"
  16 SECTIONS="main contrib non-free"
  17 TMPDIR="/org/wanna-build/tmp"
  18 LIBTRIGGER="/org/wanna-build/libtrigger.sh"
  19 LOCKFILE="/org/wanna-build/tmp/DB_Maintenance_In_Progress"
  20 MASTER="http://incoming.debian.org/debian/dists"
  21 MASTERBUILDD="http://incoming.debian.org/buildd"
  22 SECMASTER="http://security-master.debian.org/debian-security/dists"
  23 SECMASTERBUILDD=http://security-master.debian.org/buildd
  24 NEWARCH=""
  25
  26 . "$LIBTRIGGER"
  27
  28 DAY=`date +%w`
  29
  30 if lockfile -! -l 3600 $LOCKFILE; then
  31     echo "Cannot lock $LOCKFILE"
  32     exit 1
  33 fi
  34
  35 cleanup() {
  36     rm -f "$LOCKFILE"
  37 }
  38 trap cleanup 0
  39
  40 echo Updating wanna-build databases...
  41 umask 027
  42
  43 if [ "$DAY" = "0" ]; then
  44         savelog -c 26 -p /srv/wanna-build/db/merge.log
  45 fi
  46
  47 exec 3<&1 >> /srv/wanna-build/db/merge.log 2>&1
  48
  49 echo -------------------------------------------------------------------------
  50 echo "merge triggered: `date`"
  51 echo "merge triggered          : `date -u`" >&3
  52
  53 cd $TMPDIR
  54
  55 #
  56 # Make one big Packages and Sources file.
  57 #
  58 for d in $DISTS; do
  59     dist=`echo $d | sed s/-.*$//`
  60     case "$dist" in
  61         oldstable)
  62             ARCHS="$ARCHS_oldstable"
  63             ;;
  64         stable)
  65             ARCHS="$ARCHS_stable"
  66             ;;
  67         testing)
  68             ARCHS="$ARCHS_testing"
  69             ;;
  70         *)
  71             ARCHS="$ARCHS_unstable"
  72             ;;
  73     esac
  74     rm -f Sources.$d Sources.$d-p-u
  75
  76     for a in $ARCHS; do
  77         rm -f Packages.$d.$a Packages.$d-p-u.$a quinn-$d-p-u.$a
  78     done
  79
  80     for s in $SECTIONS; do
  81         if echo $d | grep -qv -- -security; then
  82             curl_index "$MASTER/$dist/$s/source/Sources.gz" "Sources-$d.$s.gz" "Sources.$d"
  83             if [ "$d" != "unstable" ]; then
  84                 curl_index "$MASTER/$dist-proposed-updates/$s/source/Sources.gz" "Sources-$d-proposed-updates.$s.gz" "Sources.$d-p-u"
  85             fi
  86
  87             for a in $ARCHS; do
  88                 curl_index "$MASTER/$dist/$s/binary-$a/Packages.gz" "Packages-$d.$s.$a.gz" "Packages.$d.$a"
  89                 if [ "$d" != "unstable" ]; then
  90                     curl_index "$MASTER/$dist-proposed-updates/$s/binary-$a/Packages.gz" "Packages-$d-proposed-updates.$s.$a.gz" "Packages.$d-p-u.$a"
  91                     if [ "$d" != "oldstable" -o "$s" = "main" ]; then
  92                         curl_index "$MASTER/$dist-proposed-updates/$s/debian-installer/binary-$a/Packages.gz" "Packages-$d-proposed-updates-debian-installer.$s.$a.gz" "Packages.$d-p-u.$a"
  93                     fi
  94                 fi
  95                 if [ "$d" != "oldstable" -o "$s" = "main" ]; then
  96                     curl_index "$MASTER/$d/$s/debian-installer/binary-$a/Packages.gz" "Packages-$d-debian-installer.$s.$a.gz" "Packages.$d.$a"
  97                 fi
  98             done
  99         else
 100             curl_index "$SECMASTER/$dist/updates/$s/source/Sources.gz" "Sources-$d.$s.gz" "Sources.$d"
 101             if [ "$s" = "main" ]; then
 102                 curl_index "$SECMASTERBUILDD/$dist/Sources.gz" "Sources-$d.accepted.gz" "Sources.$d"
 103                 curl_index "$SECMASTERBUILDD/$dist/Packages.gz" "Packages-$d.accepted.gz" "/dev/null"
 104             fi
 105             for a in $ARCHS ; do
 106                 curl_index "$SECMASTER/$dist/updates/$s/binary-$a/Packages.gz" "Packages-$d.$s.$a.gz" "Packages.$d.$a"
 107                 if [ "$s" = "main" ]; then
 108                     gzip -dc "Packages-$d.accepted.gz" >> "Packages.$d.$a"
 109                 fi
 110             done
 111         fi
 112     done
 113
 114 #
 115 # It's important that the accepted file is rebuilt before the daily run,
 116 # and that you have that should-be-empty version, to avoid unaccepts and
 117 # double builds.
 118 #
 119     if [ "$d" = "unstable" ]; then
 120         rm -f "Sources.$d.base"
 121         cp "Sources.$d" "Sources.$d.base"
 122         curl_index "$MASTERBUILDD/Sources.gz" "Sources-$d.accepted.gz" "Sources.$d"
 123         curl_index "$MASTERBUILDD/Packages.gz" "Packages-$d.accepted.gz" "/dev/null"
 124
 125         for a in $ARCHS; do
 126             rm -f "Packages.$d.$a.base"
 127             cp "Packages.$d.$a" "Packages.$d.$a.base"
 128             gzip -dc "Packages-$d.accepted.gz" >> "Packages.$d.$a"
 129         done
 130     else
 131         if [ -e "Sources.$d-p-u" ]; then
 132             cat "Sources.$d" >> "Sources.$d-p-u"
 133         fi
 134     fi
 135
 136     for a in $ARCHS; do
 137         quinn-diff -A $a -a /srv/buildd.debian.org/web/quinn-diff/$dist/Packages-arch-specific -s Sources.$d -p Packages.$d.$a >> quinn-$d.$a 2>/dev/null
 138         if echo $d | grep -qv -- -security; then
 139             if [ "$d" != "unstable" ]; then
 140                 cat "Packages.$d.$a" >> "Packages.$d-p-u.$a"
 141                 quinn-diff -A $a -a /srv/buildd.debian.org/web/quinn-diff/$dist/Packages-arch-specific -s Sources.$d-p-u -p Packages.$d-p-u.$a >> quinn-$d-p-u.$a 2>/dev/null
 142                 sed -e 's/\[[-a-z]*:[-a-z]*\]$//' quinn-$d.$a > quinn-$d.$a.grep
 143                 if [ ! -z quinn-$d.$a.grep ]; then
 144                     fgrep -vf quinn-$d.$a.grep quinn-$d-p-u.$a > quinn-$d.$a
 145                 else
 146                     cp quinn-$d-p-u.$a quinn-$d.$a
 147                 fi
 148                 rm quinn-$d.$a.grep
 149                 mv "Packages.$d-p-u.$a" "Packages.$d.$a"
 150             fi
 151         fi
 152     done
 153     if [ -e "Sources.$d-p-u" ]; then
 154         mv "Sources.$d-p-u" "Sources.$d"
 155     fi
 156 done
 157 echo "fetch and quinn completed: `date -u`"
 158 echo "fetch and quinn completed: `date -u`" >&3
 159
 160 umask 002
 161 # Gross hack: we need a list of all architectures, because this loop
 162 # insists on iterating on arches and then suites, instead of the
 163 # opposite.
 164 ALL_ARCHES=$(echo $ARCHS_oldstable $ARCHS_stable $ARCHS_testing $ARCHS_unstable \
 165              | tr ' ' '\n' | sort -u)
 166 for a in $ALL_ARCHES ; do
 167         wanna-build --create-maintenance-lock --database=$a/build-db
 168
 169         for d in $DISTS ; do
 170                 dist=`echo $d | sed s/-.*$//`
 171                 case "$dist" in
 172                         oldstable)
 173                                 if echo $ARCHS_oldstable | grep -q -v "\b$a\b"; then
 174                                         continue
 175                                 fi
 176                                 ;;
 177                         stable)
 178                                 if echo $ARCHS_stable | grep -q -v "\b$a\b"; then
 179                                         continue
 180                                 fi
 181                                 ;;
 182                         testing)
 183                                 if echo $ARCHS_testing | grep -q -v "\b$a\b"; then
 184                                         continue
 185                                 fi
 186                                 ;;
 187                         *)
 188                                 if echo $ARCHS_unstable | grep -q -v "\b$a\b"; then
 189                                         continue
 190                                 fi
 191                                 ;;
 192                 esac
 193                 # non-free is excluded for legal reasons.  buildd needs to
 194                 # confirm on a per-package and per-buildd basis whether or not
 195                 # it is buildable
 196                 perl -pi -e 's#^(non-free)/.*$##msg' quinn-$d.$a
 197                 if [ "$a" = "$NEWARCH" ]; then
 198                 wanna-build -v --merge-all --arch=$a --dist=$d --database=$a/build-db Packages.$d.$a quinn-$d.$a Sources.unstable.$NEWARCH
 199                 else
 200                 wanna-build -v --merge-all --arch=$a --dist=$d --database=$a/build-db Packages.$d.$a quinn-$d.$a Sources.$d
 201                 fi
 202                 mv Packages.$d.$a Packages.$d.$a-old
 203                 mv quinn-$d.$a quinn-$d.$a-old
 204         done
 205         if [ "$DAY" = "0" ]; then
 206                 savelog -p -c 26 /srv/wanna-build/db/$a/transactions.log
 207         fi
 208         wanna-build --remove-maintenance-lock --database=$a/build-db
 209 done
 210 umask 022
 211 for d in $DISTS; do
 212     mv Sources.$d Sources.$d-old
 213 done
 214
 215 echo "merge ended: `date`"
 216 #
 217 # Only update stats if it's been at least 20h since the last time.
 218 #
 219 interval=72000
 220 last=`stat --format="%Y" /srv/wanna-build/etc/graph-data`
 221 now=`date +%s`
 222 if (( $last + $interval < $now )); then
 223         echo "stats start: `date`"
 224         /srv/wanna-build/bin/wb-graph >> /srv/wanna-build/etc/graph-data
 225         /srv/wanna-build/bin/wb-graph -p >> /srv/wanna-build/etc/graph2-data
 226         rm -f "$LOCKFILE"
 227         trap - 0
 228         /srv/buildd.debian.org/bin/makegraph
 229         for a in $ARCHS_stable; do
 230             echo Last Updated: `date -u` > /srv/buildd.debian.org/web/stats/$a.txt
 231             for d in $STATS_DISTS; do
 232                 /srv/wanna-build/bin/wanna-build-statistics --database=$a/build-db --dist=$d >> /srv/buildd.debian.org/web/stats/$a.txt
 233             done
 234         done
 235         echo "stats ended: `date`"
 236 fi