trigger.daily

   1 #!/bin/bash
   2 #
   3 # Updates wanna-build databases after the archive maintenance
   4 # finishes
   5 #
   6 # Files:
   7 #     Sources-* == upstream fetched file
   8 #     Sources.* == uncompressed, concat'd version
   9
  10 LANG=C
  11 PATH="/bin:/usr/bin"
  12 #testing must be before unstable so late upld don't build for testing needlessly
  13 #DISTS="oldstable-security stable-security testing-security oldstable stable testing unstable"
  14 DISTS="stable-security testing-security stable testing unstable"
  15 STATS_DISTS="unstable testing stable"
  16 SECTIONS="main contrib non-free"
  17 ARCHS_oldstable="m68k arm sparc alpha powerpc i386 mips mipsel ia64 hppa s390"
  18 ARCHS_stable="arm sparc alpha powerpc i386 amd64 mips mipsel ia64 hppa s390"
  19 ARCHS_testing="$ARCHS_stable armel"
  20 ARCHS_unstable="$ARCHS_testing"
  21 TMPDIR="/org/wanna-build/tmp"
  22 LOCKFILE="/org/wanna-build/tmp/DB_Maintenance_In_Progress"
  23 MASTER="http://incoming.debian.org/debian/dists"
  24 MASTERBUILDD="http://incoming.debian.org/buildd"
  25 SECMASTER="http://security-master.debian.org/debian-security/dists"
  26 SECMASTERBUILDD=http://security-master.debian.org/buildd
  27 CURLOPT="-q -s -S -R -f -Y 10 -y 120 -K /srv/wanna-build/trigger.curlrc"
  28 NEWARCH=""
  29
  30 curl_index () {
  31     local url tmpname destname appendname curlopt
  32     url="$1"
  33     destname="$2"
  34     appendname="$3"
  35     tmpname=".$destname"
  36     rc=0
  37     rm -f "$tmpname"
  38     if [ -e "$destname" ]; then
  39         refdate=`perl -e "print scalar gmtime(((stat ('$destname'))[9]))"`
  40         curl $CURLOPT -z "$refdate" "$url" -o "$tmpname"
  41         rc=$?
  42     else
  43         curl $CURLOPT "$url" -o "$tmpname"
  44         rc=$?
  45     fi
  46     if [ $rc -eq 0 -a -e "$tmpname" ]; then
  47         if gzip -t "$tmpname"; then
  48             mv "$tmpname" "$destname"
  49         else
  50             rc=$?
  51         fi
  52     fi
  53     gzip -dc "$destname" >> "$appendname"
  54     return $rc
  55 }
  56
  57 DAY=`date +%w`
  58
  59 if lockfile -! -l 3600 $LOCKFILE; then
  60     echo "Cannot lock $LOCKFILE"
  61     exit 1
  62 fi
  63
  64 cleanup() {
  65     rm -f "$LOCKFILE"
  66 }
  67 trap cleanup 0
  68
  69 echo Updating wanna-build databases...
  70 umask 027
  71
  72 if [ "$DAY" = "0" ]; then
  73         savelog -c 26 -p /srv/wanna-build/db/merge.log
  74 fi
  75
  76 exec 3<&1 >> /srv/wanna-build/db/merge.log 2>&1
  77
  78 echo -------------------------------------------------------------------------
  79 echo "merge triggered: `date`"
  80 echo "merge triggered          : `date -u`" >&3
  81
  82 cd $TMPDIR
  83
  84 #
  85 # Make one big Packages and Sources file.
  86 #
  87 for d in $DISTS; do
  88     dist=`echo $d | sed s/-.*$//`
  89     case "$dist" in
  90         oldstable)
  91             ARCHS="$ARCHS_oldstable"
  92             ;;
  93         stable)
  94             ARCHS="$ARCHS_stable"
  95             ;;
  96         testing)
  97             ARCHS="$ARCHS_testing"
  98             ;;
  99         *)
 100             ARCHS="$ARCHS_unstable"
 101             ;;
 102     esac
 103     rm -f Sources.$d Sources.$d-p-u
 104
 105     for a in $ARCHS; do
 106         rm -f Packages.$d.$a Packages.$d-p-u.$a quinn-$d-p-u.$a
 107     done
 108
 109     for s in $SECTIONS; do
 110         if echo $d | grep -qv -- -security; then
 111             curl_index "$MASTER/$dist/$s/source/Sources.gz" "Sources-$d.$s.gz" "Sources.$d"
 112             if [ "$d" != "unstable" ]; then
 113                 curl_index "$MASTER/$dist-proposed-updates/$s/source/Sources.gz" "Sources-$d-proposed-updates.$s.gz" "Sources.$d-p-u"
 114             fi
 115
 116             for a in $ARCHS; do
 117                 curl_index "$MASTER/$dist/$s/binary-$a/Packages.gz" "Packages-$d.$s.$a.gz" "Packages.$d.$a"
 118                 if [ "$d" != "unstable" ]; then
 119                     curl_index "$MASTER/$dist-proposed-updates/$s/binary-$a/Packages.gz" "Packages-$d-proposed-updates.$s.$a.gz" "Packages.$d-p-u.$a"
 120                     if [ "$d" != "oldstable" -o "$s" = "main" ]; then
 121                         curl_index "$MASTER/$dist-proposed-updates/$s/debian-installer/binary-$a/Packages.gz" "Packages-$d-proposed-updates-debian-installer.$s.$a.gz" "Packages.$d-p-u.$a"
 122                     fi
 123                 fi
 124                 if [ "$d" != "oldstable" -o "$s" = "main" ]; then
 125                     curl_index "$MASTER/$d/$s/debian-installer/binary-$a/Packages.gz" "Packages-$d-debian-installer.$s.$a.gz" "Packages.$d.$a"
 126                 fi
 127             done
 128         else
 129             curl_index "$SECMASTER/$dist/updates/$s/source/Sources.gz" "Sources-$d.$s.gz" "Sources.$d"
 130             if [ "$s" = "main" ]; then
 131                 curl_index "$SECMASTERBUILDD/$dist/Sources.gz" "Sources-$d.accepted.gz" "Sources.$d"
 132                 curl_index "$SECMASTERBUILDD/$dist/Packages.gz" "Packages-$d.accepted.gz" "/dev/null"
 133             fi
 134             for a in $ARCHS ; do
 135                 curl_index "$SECMASTER/$dist/updates/$s/binary-$a/Packages.gz" "Packages-$d.$s.$a.gz" "Packages.$d.$a"
 136                 if [ "$s" = "main" ]; then
 137                     gzip -dc "Packages-$d.accepted.gz" >> "Packages.$d.$a"
 138                 fi
 139             done
 140         fi
 141     done
 142
 143 #
 144 # It's important that the accepted file is rebuilt before the daily run,
 145 # and that you have that should-be-empty version, to avoid unaccepts and
 146 # double builds.
 147 #
 148     if [ "$d" = "unstable" ]; then
 149         rm -f "Sources.$d.base"
 150         cp "Sources.$d" "Sources.$d.base"
 151         curl_index "$MASTERBUILDD/Sources.gz" "Sources-$d.accepted.gz" "Sources.$d"
 152         curl_index "$MASTERBUILDD/Packages.gz" "Packages-$d.accepted.gz" "/dev/null"
 153
 154         for a in $ARCHS; do
 155             rm -f "Packages.$d.$a.base"
 156             cp "Packages.$d.$a" "Packages.$d.$a.base"
 157             gzip -dc "Packages-$d.accepted.gz" >> "Packages.$d.$a"
 158         done
 159     else
 160         if [ -e "Sources.$d-p-u" ]; then
 161             cat "Sources.$d" >> "Sources.$d-p-u"
 162         fi
 163     fi
 164
 165     for a in $ARCHS; do
 166         quinn-diff -A $a -a /srv/buildd.debian.org/web/quinn-diff/$dist/Packages-arch-specific -s Sources.$d -p Packages.$d.$a >> quinn-$d.$a 2>/dev/null
 167         if echo $d | grep -qv -- -security; then
 168             if [ "$d" != "unstable" ]; then
 169                 cat "Packages.$d.$a" >> "Packages.$d-p-u.$a"
 170                 quinn-diff -A $a -a /srv/buildd.debian.org/web/quinn-diff/$dist/Packages-arch-specific -s Sources.$d-p-u -p Packages.$d-p-u.$a >> quinn-$d-p-u.$a 2>/dev/null
 171                 sed -e 's/\[[-a-z]*:[-a-z]*\]$//' quinn-$d.$a > quinn-$d.$a.grep
 172                 if [ ! -z quinn-$d.$a.grep ]; then
 173                     fgrep -vf quinn-$d.$a.grep quinn-$d-p-u.$a > quinn-$d.$a
 174                 else
 175                     cp quinn-$d-p-u.$a quinn-$d.$a
 176                 fi
 177                 rm quinn-$d.$a.grep
 178                 mv "Packages.$d-p-u.$a" "Packages.$d.$a"
 179             fi
 180         fi
 181     done
 182     if [ -e "Sources.$d-p-u" ]; then
 183         mv "Sources.$d-p-u" "Sources.$d"
 184     fi
 185 done
 186 echo "fetch and quinn completed: `date -u`"
 187 echo "fetch and quinn completed: `date -u`" >&3
 188
 189 umask 002
 190 for a in $ARCHS_unstable ; do
 191         wanna-build --create-maintenance-lock --database=$a/build-db
 192
 193         for d in $DISTS ; do
 194                 dist=`echo $d | sed s/-.*$//`
 195                 case "$dist" in
 196                         oldstable)
 197                                 if echo $ARCHS_oldstable | grep -q -v "\b$a\b"; then
 198                                         continue
 199                                 fi
 200                                 ;;
 201                         stable)
 202                                 if echo $ARCHS_stable | grep -q -v "\b$a\b"; then
 203                                         continue
 204                                 fi
 205                                 ;;
 206                         testing)
 207                                 if echo $ARCHS_testing | grep -q -v "\b$a\b"; then
 208                                         continue
 209                                 fi
 210                                 ;;
 211                         *)
 212                                 if echo $ARCHS_unstable | grep -q -v "\b$a\b"; then
 213                                         continue
 214                                 fi
 215                                 ;;
 216                 esac
 217                 # non-free is excluded for legal reasons.  buildd needs to
 218                 # confirm on a per-package and per-buildd basis whether or not
 219                 # it is buildable
 220                 perl -pi -e 's#^(non-free)/.*$##msg' quinn-$d.$a
 221                 if [ "$a" = "$NEWARCH" ]; then
 222                 wanna-build -v --merge-all --arch=$a --dist=$d --database=$a/build-db Packages.$d.$a quinn-$d.$a Sources.unstable.$NEWARCH
 223                 else
 224                 wanna-build -v --merge-all --arch=$a --dist=$d --database=$a/build-db Packages.$d.$a quinn-$d.$a Sources.$d
 225                 fi
 226                 mv Packages.$d.$a Packages.$d.$a-old
 227                 mv quinn-$d.$a quinn-$d.$a-old
 228         done
 229         if [ "$DAY" = "0" ]; then
 230                 savelog -p -c 26 /srv/wanna-build/db/$a/transactions.log
 231         fi
 232         wanna-build --remove-maintenance-lock --database=$a/build-db
 233 done
 234 umask 022
 235 for d in $DISTS; do
 236     mv Sources.$d Sources.$d-old
 237 done
 238
 239 echo "merge ended: `date`"
 240 #
 241 # Only update stats if it's been at least 20h since the last time.
 242 #
 243 interval=72000
 244 last=`stat --format="%Y" /srv/wanna-build/etc/graph-data`
 245 now=`date +%s`
 246 if (( $last + $interval < $now )); then
 247         echo "stats start: `date`"
 248         /srv/wanna-build/bin/wb-graph >> /srv/wanna-build/etc/graph-data
 249         /srv/wanna-build/bin/wb-graph -p >> /srv/wanna-build/etc/graph2-data
 250         rm -f "$LOCKFILE"
 251         trap - 0
 252         /srv/buildd.debian.org/bin/makegraph
 253         for a in $ARCHS_stable; do
 254             echo Last Updated: `date -u` > /srv/buildd.debian.org/web/stats/$a.txt
 255             for d in $STATS_DISTS; do
 256                 /srv/wanna-build/bin/wanna-build-statistics --database=$a/build-db --dist=$d >> /srv/buildd.debian.org/web/stats/$a.txt
 257             done
 258         done
 259         echo "stats ended: `date`"
 260 fi