3 # Updates wanna-build databases after the archive maintenance
7 # Sources-* == upstream fetched file
8 # Sources.* == uncompressed, concat'd version
11 PATH="/bin:/usr/bin:/org/wanna-build/bin"
12 #testing must be before unstable so late upld don't build for testing needlessly
13 DISTS="oldstable-security stable-security testing-security oldstable stable testing unstable"
14 # While oldstable is purged
15 #DISTS="stable-security testing-security stable testing unstable"
16 STATS_DISTS="unstable testing stable"
17 SECTIONS="main contrib non-free"
18 TMPDIR="/org/wanna-build/tmp"
19 LIBTRIGGER="/org/wanna-build/libtrigger.sh"
20 LOCKFILE="/org/wanna-build/tmp/DB_Maintenance_In_Progress"
21 MASTER="http://incoming.debian.org/debian/dists"
22 MASTERBUILDD="http://incoming.debian.org/buildd"
23 SECMASTER="http://security-master.debian.org/debian-security/dists"
24 SECMASTERBUILDD=http://security-master.debian.org/buildd
31 if lockfile -! -l 3600 $LOCKFILE; then
32 echo "Cannot lock $LOCKFILE"
41 echo Updating wanna-build databases...
44 if [ "$DAY" = "0" ]; then
45 savelog -c 64 -p /srv/wanna-build/db/merge.log
48 exec 3<&1 >> /srv/wanna-build/db/merge.log 2>&1
50 echo -------------------------------------------------------------------------
51 echo "merge triggered: `date`"
52 echo "merge triggered : `date -u`" >&3
57 # Make one big Packages and Sources file.
60 dist=`echo $d | sed s/-.*$//`
63 ARCHS="$ARCHS_oldstable"
69 ARCHS="$ARCHS_testing"
72 ARCHS="$ARCHS_unstable"
75 rm -f Sources.$d Sources.$d-p-u
78 rm -f Packages.$d.$a Packages.$d-p-u.$a quinn-$d-p-u.$a
81 for s in $SECTIONS; do
82 if echo $d | grep -qv -- -security; then
83 curl_index "$MASTER/$dist/$s/source/Sources.gz" "Sources-$d.$s.gz" "Sources.$d"
84 if [ "$d" != "unstable" ]; then
85 curl_index "$MASTER/$dist-proposed-updates/$s/source/Sources.gz" "Sources-$d-proposed-updates.$s.gz" "Sources.$d-p-u"
89 curl_index "$MASTER/$dist/$s/binary-$a/Packages.gz" "Packages-$d.$s.$a.gz" "Packages.$d.$a"
90 if [ "$d" != "unstable" ]; then
91 curl_index "$MASTER/$dist-proposed-updates/$s/binary-$a/Packages.gz" "Packages-$d-proposed-updates.$s.$a.gz" "Packages.$d-p-u.$a"
92 if [ "$d" != "oldstable" -o "$s" = "main" ]; then
93 curl_index "$MASTER/$dist-proposed-updates/$s/debian-installer/binary-$a/Packages.gz" "Packages-$d-proposed-updates-debian-installer.$s.$a.gz" "Packages.$d-p-u.$a"
96 if [ "$d" != "oldstable" -o "$s" = "main" ]; then
97 curl_index "$MASTER/$d/$s/debian-installer/binary-$a/Packages.gz" "Packages-$d-debian-installer.$s.$a.gz" "Packages.$d.$a"
101 curl_index "$SECMASTER/$dist/updates/$s/source/Sources.gz" "Sources-$d.$s.gz" "Sources.$d"
102 if [ "$s" = "main" ]; then
103 curl_index "$SECMASTERBUILDD/$dist/Sources.gz" "Sources-$d.accepted.gz" "Sources.$d"
104 curl_index "$SECMASTERBUILDD/$dist/Packages.gz" "Packages-$d.accepted.gz" "/dev/null"
107 curl_index "$SECMASTER/$dist/updates/$s/binary-$a/Packages.gz" "Packages-$d.$s.$a.gz" "Packages.$d.$a"
108 if [ "$s" = "main" ]; then
109 gzip -dc "Packages-$d.accepted.gz" >> "Packages.$d.$a"
116 # It's important that the accepted file is rebuilt before the daily run,
117 # and that you have that should-be-empty version, to avoid unaccepts and
120 if [ "$d" = "unstable" ]; then
121 mv "Sources.$d" "Sources.$d.base"
122 curl_index "$MASTERBUILDD/Sources.gz" "Sources-$d.accepted.gz" "/dev/null"
123 rm -f "Sources-$d.accepted"
124 gunzip "Sources-$d.accepted.gz"
125 keep-latest source "Sources.$d.base" "Sources-$d.accepted" > "Sources.$d"
127 curl_index "$MASTERBUILDD/Packages.gz" "Packages-$d.accepted.gz" "/dev/null"
128 rm -f "Packages-$d.accepted"
129 gunzip "Packages-$d.accepted.gz"
132 mv "Packages.$d.$a" "Packages.$d.$a.base"
133 gzip -dc "Packages-$d.accepted.gz" >> "Packages.$d.$a"
134 keep-latest $a "Packages.$d.$a.base" "Packages-$d.accepted" > "Packages.$d.$a"
137 if [ -e "Sources.$d-p-u" ]; then
138 cat "Sources.$d" >> "Sources.$d-p-u"
143 quinn-diff -A $a -a /srv/buildd.debian.org/web/quinn-diff/$dist/Packages-arch-specific -s Sources.$d -p Packages.$d.$a >> quinn-$d.$a 2>/dev/null
144 if echo $d | grep -qv -- -security; then
145 if [ "$d" != "unstable" ]; then
146 cat "Packages.$d.$a" >> "Packages.$d-p-u.$a"
147 quinn-diff -A $a -a /srv/buildd.debian.org/web/quinn-diff/$dist/Packages-arch-specific -s Sources.$d-p-u -p Packages.$d-p-u.$a >> quinn-$d-p-u.$a 2>/dev/null
148 sed -e 's/\[[-a-z]*:[-a-z]*\]$//' quinn-$d.$a > quinn-$d.$a.grep
149 if [ ! -z quinn-$d.$a.grep ]; then
150 fgrep -vf quinn-$d.$a.grep quinn-$d-p-u.$a > quinn-$d.$a
152 cp quinn-$d-p-u.$a quinn-$d.$a
155 mv "Packages.$d-p-u.$a" "Packages.$d.$a"
159 if [ -e "Sources.$d-p-u" ]; then
160 mv "Sources.$d-p-u" "Sources.$d"
163 echo "fetch and quinn completed: `date -u`"
164 echo "fetch and quinn completed: `date -u`" >&3
167 # Gross hack: we need a list of all architectures, because this loop
168 # insists on iterating on arches and then suites, instead of the
170 ALL_ARCHES=$(echo $ARCHS_oldstable $ARCHS_stable $ARCHS_testing $ARCHS_unstable \
171 | tr ' ' '\n' | sort -u)
172 for a in $ALL_ARCHES ; do
173 wanna-build --create-maintenance-lock --database=$a/build-db
176 dist=`echo $d | sed s/-.*$//`
179 if echo $ARCHS_oldstable | grep -q -v "\b$a\b"; then
184 if echo $ARCHS_stable | grep -q -v "\b$a\b"; then
189 if echo $ARCHS_testing | grep -q -v "\b$a\b"; then
194 if echo $ARCHS_unstable | grep -q -v "\b$a\b"; then
199 # non-free is excluded for legal reasons. buildd needs to
200 # confirm on a per-package and per-buildd basis whether or not
202 perl -pi -e 's#^(non-free)/.*$##msg' quinn-$d.$a
203 if [ "$a" = "$NEWARCH" ]; then
204 wanna-build -v --merge-all --arch=$a --dist=$d --database=$a/build-db Packages.$d.$a quinn-$d.$a Sources.unstable.$NEWARCH
206 wanna-build -v --merge-all --arch=$a --dist=$d --database=$a/build-db Packages.$d.$a quinn-$d.$a Sources.$d
208 mv Packages.$d.$a Packages.$d.$a-old
209 mv quinn-$d.$a quinn-$d.$a-old
211 if [ "$DAY" = "0" ]; then
212 savelog -p -c 128 /srv/wanna-build/db/$a/transactions.log
214 wanna-build --remove-maintenance-lock --database=$a/build-db
218 mv Sources.$d Sources.$d-old
221 echo "merge ended: `date`"
223 # Only update stats if it's been at least 20h since the last time.
226 last=`stat --format="%Y" /srv/wanna-build/etc/graph-data`
228 if (( $last + $interval < $now )); then
229 echo "stats start: `date`"
230 /srv/wanna-build/bin/wb-graph >> /srv/wanna-build/etc/graph-data
231 /srv/wanna-build/bin/wb-graph -p >> /srv/wanna-build/etc/graph2-data
234 /srv/buildd.debian.org/bin/makegraph
235 for a in $ARCHS_stable; do
236 echo Last Updated: `date -u` > /srv/buildd.debian.org/web/stats/$a.txt
237 for d in $STATS_DISTS; do
238 /srv/wanna-build/bin/wanna-build-statistics --database=$a/build-db --dist=$d >> /srv/buildd.debian.org/web/stats/$a.txt
241 echo "stats ended: `date`"