3 # Updates wanna-build databases after the archive maintenance
7 # Sources-* == upstream fetched file
8 # Sources.* == uncompressed, concat'd version
12 #testing must be before unstable so late upld don't build for testing needlessly
13 #DISTS="oldstable-security stable-security testing-security oldstable stable testing unstable"
14 DISTS="stable-security testing-security stable testing unstable"
15 STATS_DISTS="unstable testing stable"
16 SECTIONS="main contrib non-free"
17 ARCHS_oldstable="m68k arm sparc alpha powerpc i386 mips mipsel ia64 hppa s390"
18 ARCHS_stable="arm sparc alpha powerpc i386 amd64 mips mipsel ia64 hppa s390"
19 ARCHS_testing="$ARCHS_stable armel"
20 ARCHS_unstable="$ARCHS_testing"
21 TMPDIR="/org/wanna-build/tmp"
22 LOCKFILE="/org/wanna-build/tmp/DB_Maintenance_In_Progress"
23 MASTER="http://incoming.debian.org/debian/dists"
24 MASTERBUILDD="http://incoming.debian.org/buildd"
25 SECMASTER="http://security-master.debian.org/debian-security/dists"
26 SECMASTERBUILDD=http://security-master.debian.org/buildd
27 CURLOPT="-q -s -S -R -f -Y 10 -y 120 -K /srv/wanna-build/trigger.curlrc"
31 local url tmpname destname appendname curlopt
38 if [ -e "$destname" ]; then
39 refdate=`perl -e "print scalar gmtime(((stat ('$destname'))[9]))"`
40 curl $CURLOPT -z "$refdate" "$url" -o "$tmpname"
43 curl $CURLOPT "$url" -o "$tmpname"
46 if [ $rc -eq 0 -a -e "$tmpname" ]; then
47 if gzip -t "$tmpname"; then
48 mv "$tmpname" "$destname"
53 gzip -dc "$destname" >> "$appendname"
59 if lockfile -! -l 3600 $LOCKFILE; then
60 echo "Cannot lock $LOCKFILE"
69 echo Updating wanna-build databases...
72 if [ "$DAY" = "0" ]; then
73 savelog -c 26 -p /srv/wanna-build/db/merge.log
76 exec 3<&1 >> /srv/wanna-build/db/merge.log 2>&1
78 echo -------------------------------------------------------------------------
79 echo "merge triggered: `date`"
80 echo "merge triggered : `date -u`" >&3
85 # Make one big Packages and Sources file.
88 dist=`echo $d | sed s/-.*$//`
91 ARCHS="$ARCHS_oldstable"
97 ARCHS="$ARCHS_testing"
100 ARCHS="$ARCHS_unstable"
103 rm -f Sources.$d Sources.$d-p-u
106 rm -f Packages.$d.$a Packages.$d-p-u.$a quinn-$d-p-u.$a
109 for s in $SECTIONS; do
110 if echo $d | grep -qv -- -security; then
111 curl_index "$MASTER/$dist/$s/source/Sources.gz" "Sources-$d.$s.gz" "Sources.$d"
112 if [ "$d" != "unstable" ]; then
113 curl_index "$MASTER/$dist-proposed-updates/$s/source/Sources.gz" "Sources-$d-proposed-updates.$s.gz" "Sources.$d-p-u"
117 curl_index "$MASTER/$dist/$s/binary-$a/Packages.gz" "Packages-$d.$s.$a.gz" "Packages.$d.$a"
118 if [ "$d" != "unstable" ]; then
119 curl_index "$MASTER/$dist-proposed-updates/$s/binary-$a/Packages.gz" "Packages-$d-proposed-updates.$s.$a.gz" "Packages.$d-p-u.$a"
120 if [ "$d" != "oldstable" -o "$s" = "main" ]; then
121 curl_index "$MASTER/$dist-proposed-updates/$s/debian-installer/binary-$a/Packages.gz" "Packages-$d-proposed-updates-debian-installer.$s.$a.gz" "Packages.$d-p-u.$a"
124 if [ "$d" != "oldstable" -o "$s" = "main" ]; then
125 curl_index "$MASTER/$d/$s/debian-installer/binary-$a/Packages.gz" "Packages-$d-debian-installer.$s.$a.gz" "Packages.$d.$a"
129 curl_index "$SECMASTER/$dist/updates/$s/source/Sources.gz" "Sources-$d.$s.gz" "Sources.$d"
130 if [ "$s" = "main" ]; then
131 curl_index "$SECMASTERBUILDD/$dist/Sources.gz" "Sources-$d.accepted.gz" "Sources.$d"
132 curl_index "$SECMASTERBUILDD/$dist/Packages.gz" "Packages-$d.accepted.gz" "/dev/null"
135 curl_index "$SECMASTER/$dist/updates/$s/binary-$a/Packages.gz" "Packages-$d.$s.$a.gz" "Packages.$d.$a"
136 if [ "$s" = "main" ]; then
137 gzip -dc "Packages-$d.accepted.gz" >> "Packages.$d.$a"
144 # It's important that the accepted file is rebuilt before the daily run,
145 # and that you have that should-be-empty version, to avoid unaccepts and
148 if [ "$d" = "unstable" ]; then
149 rm -f "Sources.$d.base"
150 cp "Sources.$d" "Sources.$d.base"
151 curl_index "$MASTERBUILDD/Sources.gz" "Sources-$d.accepted.gz" "Sources.$d"
152 curl_index "$MASTERBUILDD/Packages.gz" "Packages-$d.accepted.gz" "/dev/null"
155 rm -f "Packages.$d.$a.base"
156 cp "Packages.$d.$a" "Packages.$d.$a.base"
157 gzip -dc "Packages-$d.accepted.gz" >> "Packages.$d.$a"
160 if [ -e "Sources.$d-p-u" ]; then
161 cat "Sources.$d" >> "Sources.$d-p-u"
166 quinn-diff -A $a -a /srv/buildd.debian.org/web/quinn-diff/$dist/Packages-arch-specific -s Sources.$d -p Packages.$d.$a >> quinn-$d.$a 2>/dev/null
167 if echo $d | grep -qv -- -security; then
168 if [ "$d" != "unstable" ]; then
169 cat "Packages.$d.$a" >> "Packages.$d-p-u.$a"
170 quinn-diff -A $a -a /srv/buildd.debian.org/web/quinn-diff/$dist/Packages-arch-specific -s Sources.$d-p-u -p Packages.$d-p-u.$a >> quinn-$d-p-u.$a 2>/dev/null
171 sed -e 's/\[[-a-z]*:[-a-z]*\]$//' quinn-$d.$a > quinn-$d.$a.grep
172 if [ ! -z quinn-$d.$a.grep ]; then
173 fgrep -vf quinn-$d.$a.grep quinn-$d-p-u.$a > quinn-$d.$a
175 cp quinn-$d-p-u.$a quinn-$d.$a
178 mv "Packages.$d-p-u.$a" "Packages.$d.$a"
182 if [ -e "Sources.$d-p-u" ]; then
183 mv "Sources.$d-p-u" "Sources.$d"
186 echo "fetch and quinn completed: `date -u`"
187 echo "fetch and quinn completed: `date -u`" >&3
190 for a in $ARCHS_unstable ; do
191 wanna-build --create-maintenance-lock --database=$a/build-db
194 dist=`echo $d | sed s/-.*$//`
197 if echo $ARCHS_oldstable | grep -q -v "\b$a\b"; then
202 if echo $ARCHS_stable | grep -q -v "\b$a\b"; then
207 if echo $ARCHS_testing | grep -q -v "\b$a\b"; then
212 if echo $ARCHS_unstable | grep -q -v "\b$a\b"; then
217 # non-free is excluded for legal reasons. buildd needs to
218 # confirm on a per-package and per-buildd basis whether or not
220 perl -pi -e 's#^(non-free)/.*$##msg' quinn-$d.$a
221 if [ "$a" = "$NEWARCH" ]; then
222 wanna-build -v --merge-all --arch=$a --dist=$d --database=$a/build-db Packages.$d.$a quinn-$d.$a Sources.unstable.$NEWARCH
224 wanna-build -v --merge-all --arch=$a --dist=$d --database=$a/build-db Packages.$d.$a quinn-$d.$a Sources.$d
226 mv Packages.$d.$a Packages.$d.$a-old
227 mv quinn-$d.$a quinn-$d.$a-old
229 if [ "$DAY" = "0" ]; then
230 savelog -p -c 26 /srv/wanna-build/db/$a/transactions.log
232 wanna-build --remove-maintenance-lock --database=$a/build-db
236 mv Sources.$d Sources.$d-old
239 echo "merge ended: `date`"
241 # Only update stats if it's been at least 20h since the last time.
244 last=`stat --format="%Y" /srv/wanna-build/etc/graph-data`
246 if (( $last + $interval < $now )); then
247 echo "stats start: `date`"
248 /srv/wanna-build/bin/wb-graph >> /srv/wanna-build/etc/graph-data
249 /srv/wanna-build/bin/wb-graph -p >> /srv/wanna-build/etc/graph2-data
252 /srv/buildd.debian.org/bin/makegraph
253 for a in $ARCHS_stable; do
254 echo Last Updated: `date -u` > /srv/buildd.debian.org/web/stats/$a.txt
255 for d in $STATS_DISTS; do
256 /srv/wanna-build/bin/wanna-build-statistics --database=$a/build-db --dist=$d >> /srv/buildd.debian.org/web/stats/$a.txt
259 echo "stats ended: `date`"