trigger.daily

   1 #!/bin/bash -x
   2 #
   3 # Updates wanna-build databases after the archive maintenance
   4 # finishes
   5 #
   6 # Files:
   7 #     Sources-* == upstream fetched file
   8 #     Sources.* == uncompressed, concat'd version
   9
  10 LANG=C
  11 PATH="/bin:/usr/bin:/org/wanna-build/bin"
  12 #testing must be before unstable so late upld don't build for testing needlessly
  13 DISTS="oldstable-security stable-security testing-security oldstable stable testing unstable"
  14 # While oldstable is purged
  15 #DISTS="stable-security testing-security stable testing unstable"
  16 STATS_DISTS="unstable testing stable"
  17 SECTIONS="main contrib non-free"
  18 TMPDIR="/org/wanna-build/tmp"
  19 LIBTRIGGER="/org/wanna-build/libtrigger.sh"
  20 LOCKFILE="/org/wanna-build/tmp/DB_Maintenance_In_Progress"
  21 MASTER="http://incoming.debian.org/debian/dists"
  22 MASTERBUILDD="http://incoming.debian.org/buildd"
  23 SECMASTER="http://security-master.debian.org/debian-security/dists"
  24 SECMASTERBUILDD=http://security-master.debian.org/buildd
  25 NEWARCH=""
  26
  27 . "$LIBTRIGGER"
  28
  29 DAY=`date +%w`
  30
  31 if lockfile -! -l 3600 $LOCKFILE; then
  32     echo "Cannot lock $LOCKFILE"
  33     exit 1
  34 fi
  35
  36 cleanup() {
  37     rm -f "$LOCKFILE"
  38 }
  39 trap cleanup 0
  40
  41 echo Dumping wanna-build databases...
  42 pg_dump service=wanna-build-privileged | bzip2 > /org/wanna-build/dumps/dump_pre_$(date +%Y.%m.%d-%H:%M:%S).bz2
  43
  44 echo Updating wanna-build databases...
  45 umask 027
  46
  47 if [ "$DAY" = "0" ]; then
  48         savelog -c 64 -p /srv/wanna-build/db/merge.log
  49 fi
  50
  51 exec 3<&1 >> /srv/wanna-build/db/merge.log 2>&1
  52
  53 echo -------------------------------------------------------------------------
  54 echo "merge triggered: `date`"
  55 echo "merge triggered          : `date -u`" >&3
  56
  57 cd $TMPDIR
  58
  59 #
  60 # Make one big Packages and Sources file.
  61 #
  62 for d in $DISTS; do
  63     dist=`echo $d | sed s/-.*$//`
  64     case "$dist" in
  65         oldstable)
  66             ARCHS="$ARCHS_oldstable"
  67             ;;
  68         stable)
  69             ARCHS="$ARCHS_stable"
  70             ;;
  71         testing)
  72             ARCHS="$ARCHS_testing"
  73             ;;
  74         *)
  75             ARCHS="$ARCHS_unstable"
  76             ;;
  77     esac
  78     rm -f Sources.$d Sources.$d-p-u
  79
  80     for a in $ARCHS; do
  81         rm -f Packages.$d.$a Packages.$d-p-u.$a quinn-$d-p-u.$a
  82     done
  83
  84     for s in $SECTIONS; do
  85         if echo $d | grep -qv -- -security; then
  86             curl_index "$MASTER/$dist/$s/source/Sources.gz" "Sources-$d.$s.gz" "Sources.$d"
  87             if [ "$d" != "unstable" ]; then
  88                 curl_index "$MASTER/$dist-proposed-updates/$s/source/Sources.gz" "Sources-$d-proposed-updates.$s.gz" "Sources.$d-p-u"
  89             fi
  90
  91             for a in $ARCHS; do
  92                 curl_index "$MASTER/$dist/$s/binary-$a/Packages.gz" "Packages-$d.$s.$a.gz" "Packages.$d.$a"
  93                 if [ "$d" != "unstable" ]; then
  94                     curl_index "$MASTER/$dist-proposed-updates/$s/binary-$a/Packages.gz" "Packages-$d-proposed-updates.$s.$a.gz" "Packages.$d-p-u.$a"
  95                     if [ "$d" != "oldstable" -o "$s" = "main" ]; then
  96                         curl_index "$MASTER/$dist-proposed-updates/$s/debian-installer/binary-$a/Packages.gz" "Packages-$d-proposed-updates-debian-installer.$s.$a.gz" "Packages.$d-p-u.$a"
  97                     fi
  98                 fi
  99                 if [ "$d" != "oldstable" -o "$s" = "main" ]; then
 100                     curl_index "$MASTER/$d/$s/debian-installer/binary-$a/Packages.gz" "Packages-$d-debian-installer.$s.$a.gz" "Packages.$d.$a"
 101                 fi
 102             done
 103         else
 104             curl_index "$SECMASTER/$dist/updates/$s/source/Sources.gz" "Sources-$d.$s.gz" "Sources.$d"
 105             if [ "$s" = "main" ]; then
 106                 curl_index "$SECMASTERBUILDD/$dist/Sources.gz" "Sources-$d.accepted.gz" "Sources.$d"
 107                 curl_index "$SECMASTERBUILDD/$dist/Packages.gz" "Packages-$d.accepted.gz" "/dev/null"
 108             fi
 109             for a in $ARCHS ; do
 110                 curl_index "$SECMASTER/$dist/updates/$s/binary-$a/Packages.gz" "Packages-$d.$s.$a.gz" "Packages.$d.$a"
 111                 if [ "$s" = "main" ]; then
 112                     gzip -dc "Packages-$d.accepted.gz" >> "Packages.$d.$a"
 113                 fi
 114             done
 115         fi
 116     done
 117
 118 #
 119 # It's important that the accepted file is rebuilt before the daily run,
 120 # and that you have that should-be-empty version, to avoid unaccepts and
 121 # double builds.
 122 #
 123     if [ "$d" = "unstable" ]; then
 124         mv "Sources.$d" "Sources.$d.base"
 125         curl_index "$MASTERBUILDD/Sources.gz" "Sources-$d.accepted.gz" "/dev/null"
 126         rm -f "Sources-$d.accepted"
 127         gunzip "Sources-$d.accepted.gz"
 128         keep-latest source "Sources.$d.base" "Sources-$d.accepted" > "Sources.$d"
 129
 130         curl_index "$MASTERBUILDD/Packages.gz" "Packages-$d.accepted.gz" "/dev/null"
 131         rm -f "Packages-$d.accepted"
 132         gunzip "Packages-$d.accepted.gz"
 133
 134         for a in $ARCHS; do
 135             mv "Packages.$d.$a" "Packages.$d.$a.base"
 136             gzip -dc "Packages-$d.accepted.gz" >> "Packages.$d.$a"
 137             keep-latest $a "Packages.$d.$a.base" "Packages-$d.accepted" > "Packages.$d.$a"
 138         done
 139     else
 140         if [ -e "Sources.$d-p-u" ]; then
 141             cat "Sources.$d" >> "Sources.$d-p-u"
 142         fi
 143     fi
 144
 145     for a in $ARCHS; do
 146         quinn-diff -A $a -a /srv/buildd.debian.org/web/quinn-diff/$dist/Packages-arch-specific -s Sources.$d -p Packages.$d.$a >> quinn-$d.$a 2>/dev/null
 147         if echo $d | grep -qv -- -security; then
 148             if [ "$d" != "unstable" ]; then
 149                 cat "Packages.$d.$a" >> "Packages.$d-p-u.$a"
 150                 quinn-diff -A $a -a /srv/buildd.debian.org/web/quinn-diff/$dist/Packages-arch-specific -s Sources.$d-p-u -p Packages.$d-p-u.$a >> quinn-$d-p-u.$a 2>/dev/null
 151                 sed -e 's/\[[-a-z]*:[-a-z]*\]$//' quinn-$d.$a > quinn-$d.$a.grep
 152                 if [ ! -z quinn-$d.$a.grep ]; then
 153                     fgrep -vf quinn-$d.$a.grep quinn-$d-p-u.$a > quinn-$d.$a
 154                 else
 155                     cp quinn-$d-p-u.$a quinn-$d.$a
 156                 fi
 157                 rm quinn-$d.$a.grep
 158                 mv "Packages.$d-p-u.$a" "Packages.$d.$a"
 159             fi
 160         fi
 161     done
 162     if [ -e "Sources.$d-p-u" ]; then
 163         mv "Sources.$d-p-u" "Sources.$d"
 164     fi
 165 done
 166 echo "fetch and quinn completed: `date -u`"
 167 echo "fetch and quinn completed: `date -u`" >&3
 168
 169 umask 002
 170 # Gross hack: we need a list of all architectures, because this loop
 171 # insists on iterating on arches and then suites, instead of the
 172 # opposite.
 173 ALL_ARCHES=$(echo $ARCHS_oldstable $ARCHS_stable $ARCHS_testing $ARCHS_unstable \
 174              | tr ' ' '\n' | sort -u)
 175 for a in $ALL_ARCHES ; do
 176         for d in $DISTS ; do
 177                 dist=`echo $d | sed s/-.*$//`
 178                 case "$dist" in
 179                         oldstable)
 180                                 if echo $ARCHS_oldstable | grep -q -v "\b$a\b"; then
 181                                         continue
 182                                 fi
 183                                 ;;
 184                         stable)
 185                                 if echo $ARCHS_stable | grep -q -v "\b$a\b"; then
 186                                         continue
 187                                 fi
 188                                 ;;
 189                         testing)
 190                                 if echo $ARCHS_testing | grep -q -v "\b$a\b"; then
 191                                         continue
 192                                 fi
 193                                 ;;
 194                         *)
 195                                 if echo $ARCHS_unstable | grep -q -v "\b$a\b"; then
 196                                         continue
 197                                 fi
 198                                 ;;
 199                 esac
 200                 # non-free is excluded for legal reasons.  buildd needs to
 201                 # confirm on a per-package and per-buildd basis whether or not
 202                 # it is buildable
 203                 perl -pi -e 's#^(non-free)/.*$##msg' quinn-$d.$a
 204                 if [ "$a" = "$NEWARCH" ]; then
 205                 wanna-build -v --merge-all --arch=$a --dist=$d --database=$a/build-db Packages.$d.$a quinn-$d.$a Sources.unstable.$NEWARCH
 206                 else
 207                 wanna-build -v --merge-all --arch=$a --dist=$d --database=$a/build-db Packages.$d.$a quinn-$d.$a Sources.$d
 208                 fi
 209                 mv Packages.$d.$a Packages.$d.$a-old
 210                 mv quinn-$d.$a quinn-$d.$a-old
 211         done
 212         if [ "$DAY" = "0" ]; then
 213                 savelog -p -c 128 /srv/wanna-build/db/$a/transactions.log
 214         fi
 215 done
 216 umask 022
 217 for d in $DISTS; do
 218     mv Sources.$d Sources.$d-old
 219 done
 220
 221 echo "merge ended: `date`"
 222
 223 pg_dump service=wanna-build-privileged | bzip2 > /org/wanna-build/dumps/dump_post_$(date +%Y.%m.%d-%H:%M:%S).bz2
 224
 225 #
 226 # Only update stats if it's been at least 20h since the last time.
 227 #
 228 interval=72000
 229 last=`stat --format="%Y" /srv/wanna-build/etc/graph-data`
 230 now=`date +%s`
 231 if (( $last + $interval < $now )); then
 232         echo "stats start: `date`"
 233         /srv/wanna-build/bin/wb-graph >> /srv/wanna-build/etc/graph-data
 234         /srv/wanna-build/bin/wb-graph -p >> /srv/wanna-build/etc/graph2-data
 235         rm -f "$LOCKFILE"
 236         trap - 0
 237         /srv/buildd.debian.org/bin/makegraph
 238         for a in $ALL_ARCHES; do
 239             echo Last Updated: `date -u` > /srv/buildd.debian.org/web/stats/$a.txt
 240             for d in $STATS_DISTS; do
 241                 /srv/wanna-build/bin/wanna-build-statistics --database=$a/build-db --dist=$d >> /srv/buildd.debian.org/web/stats/$a.txt
 242             done
 243         done
 244         echo "stats ended: `date`"
 245 fi