2 # No way I try to deal with a crippled sh just for POSIX foo.
4 # Copyright (C) 2009 Joerg Jaspert <joerg@debian.org>
6 # This program is free software; you can redistribute it and/or
7 # modify it under the terms of the GNU General Public License as
8 # published by the Free Software Foundation; version 2.
10 # This program is distributed in the hope that it will be useful, but
11 # WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 # General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program; if not, write to the Free Software
17 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 # Homer: Are you saying you're never going to eat any animal again? What
25 # Lisa: Dad, those all come from the same animal.
26 # Homer: Heh heh heh. Ooh, yeah, right, Lisa. A wonderful, magical animal.
30 # make sure to only use defined variables
32 # ERR traps should be inherited from functions too. (And command
33 # substitutions and subshells and whatnot, but for us the functions is
34 # the important part here)
37 # import the general variable set.
38 export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
41 ########################################################################
43 ########################################################################
44 # common functions are "outsourced"
45 . "${configdir}/common"
47 # Timestamp. Used for dinstall stat graphs
50 echo "Archive maintenance timestamp $TS ($1): $(date +%H:%M:%S)"
56 rm -f ${LOCK_ACCEPTED}
59 # If we error out this one is called, *FOLLOWED* by cleanup above
61 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
62 cat "${STAGEFILE}.log" | mail -s "ATTENTION ATTENTION! dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)" cron@ftp-master.debian.org
65 ########################################################################
66 # the actual dinstall functions follow #
67 ########################################################################
69 # Setup the notice file to tell bad mirrors they used the wrong time
73 Packages are currently being installed and indices rebuilt.
74 Maintenance is automatic, starting at 01|07|13|19:52 UTC,
75 and ending about an hour later. This file is then removed.
77 You should not mirror the archive during this period.
81 # pushing merkels QA user, part one
83 log "Telling merkels QA user that we start dinstall"
84 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@merkel.debian.org sleep 1
87 # Create the postgres dump files
88 function pgdump_pre() {
89 log "Creating pre-daily-cron-job backup of projectb database..."
90 pg_dump projectb > $base/backup/dump_pre_$(date +%Y.%m.%d-%H:%M:%S)
93 function pgdump_post() {
94 log "Creating post-daily-cron-job backup of projectb database..."
96 POSTDUMP=$(date +%Y.%m.%d-%H:%M:%S)
97 pg_dump projectb > $base/backup/dump_$POSTDUMP
98 pg_dumpall --globals-only > $base/backup/dumpall_$POSTDUMP
99 ln -sf $base/backup/dump_$POSTDUMP current
100 ln -sf $base/backup/dumpall_$POSTDUMP currentall
103 # Load the dak-dev projectb
104 function pgdakdev() {
106 echo "drop database projectb" | psql -p 5433 template1
107 cat currentall | psql -p 5433 template1
108 createdb -p 5433 -T template0 projectb
109 fgrep -v '\connect' current | psql -p 5433 projectb
112 # Updating various files
114 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
116 $scriptsdir/update-bugdoctxt
117 $scriptsdir/update-mirrorlists
118 $scriptsdir/update-mailingliststxt
119 $scriptsdir/update-pseudopackages.sh
122 # Process (oldstable)-proposed-updates "NEW" queue
123 function punew_do() {
124 cd "${queuedir}/${1}"
126 dak process-new -a -C COMMENTS >> REPORT || true
130 log "Doing automated p-u-new processing"
134 log "Doing automated o-p-u-new processing"
138 # The first i18n one, syncing new descriptions
140 log "Synchronizing i18n package descriptions"
141 # First sync their newest data
142 cd ${scriptdir}/i18nsync
143 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
145 # Now check if we still know about the packages for which they created the files
146 # is the timestamp signed by us?
147 if $(gpgv --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
148 # now read it. As its signed by us we are sure the content is what we expect, no need
149 # to do more here. And we only test -d a directory on it anyway.
150 TSTAMP=$(cat timestamp)
151 # do we have the dir still?
152 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
154 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
155 # Yay, worked, lets copy around
156 for dir in squeeze sid; do
157 if [ -d dists/${dir}/ ]; then
158 cd dists/${dir}/main/i18n
159 rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
161 cd ${scriptdir}/i18nsync
164 echo "ARRRR, bad guys, wrong files, ARRR"
165 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org
168 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
169 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org
172 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
173 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org
177 # Process the accepted queue
178 function accepted() {
179 log "Processing queue/accepted"
180 rm -f "$accepted/REPORT"
181 dak process-accepted -pa -d "$accepted" > "$accepted/REPORT"
182 cat "$accepted/REPORT" | mail -s "Install for $(date +"%D - %R")" ftpmaster@ftp-master.debian.org
183 chgrp debadmin "$accepted/REPORT"
184 chmod 664 "$accepted/REPORT"
188 log "Checking for cruft in overrides"
191 log "Fixing symlinks in $ftpdir"
192 symlinks -d -r $ftpdir
196 log "Generating suite file lists for apt-ftparchive"
197 dak make-suite-file-list
200 function fingerprints() {
201 log "Updating fingerprints"
202 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
205 function overrides() {
206 log "Writing overrides into text files"
211 rm -f override.sid.all3
212 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
216 log "Generating package / file mapping"
217 dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
220 function packages() {
221 log "Generating Packages and Sources files"
223 apt-ftparchive generate apt.conf
227 log "Generating pdiff files"
228 dak generate-index-diffs
232 log "Generating Release files"
233 dak generate-releases
236 function dakcleanup() {
237 log "Cleanup old packages/files"
238 dak clean-suites -m 10000
243 # Needs to be rebuilt, as files have moved. Due to unaccepts, we need to
244 # update this before wanna-build is updated.
245 log "Regenerating wanna-build/buildd information"
246 psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
247 symlinks -d /srv/incoming.debian.org/buildd > /dev/null
248 apt-ftparchive generate apt.conf.buildd
251 function buildd_dir() {
252 # Rebuilt the buildd dir to avoid long times of 403
253 log "Regenerating the buildd incoming dir"
254 STAMP=$(date "+%Y%m%d%H%M")
259 log "Running various scripts from $scriptsdir"
269 echo "Regenerating \"public\" mirror/ hardlink fun"
271 rsync -aH --link-dest ${ftpdir} --exclude Archive_Maintenance_In_Progress --delete --delete-after --ignore-errors ${ftpdir}/. .
275 log "Trigger daily wanna-build run"
276 ssh -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 wbadm@buildd /org/wanna-build/trigger.daily || echo "W-B trigger.daily failed" | mail -s "W-B Daily trigger failed" ftpmaster@ftp-master.debian.org
280 log "Expiring old database dumps..."
282 $scriptsdir/expire_dumps -d . -p -f "dump_*"
285 function transitionsclean() {
286 log "Removing out of date transitions..."
288 dak transitions -c -a
292 # Send a report on NEW/BYHAND packages
293 log "Nagging ftpteam about NEW/BYHAND packages"
294 dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org
295 # and one on crufty packages
296 log "Sending information about crufty packages"
297 dak cruft-report > $webdir/cruft-report-daily.txt
298 dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
299 cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
303 log "Updating DM html page"
304 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
308 log "Categorizing uncategorized bugs filed against ftp.debian.org"
313 # Push dak@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
314 log "Trigger merkel/flotows projectb sync"
315 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb dak@merkel.debian.org sleep 1
316 # Also trigger flotow, the ftpmaster test box
317 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_flotow_projectb dak@flotow.debconf.org sleep 1
321 # Push dak@merkel to tell it to sync the dd accessible parts. Returns immediately, the sync runs detached
322 log "Trigger merkels dd accessible parts sync"
323 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_ddaccess dak@merkel.debian.org sleep 1
326 function runparts() {
327 log "Using run-parts to run scripts in $base/scripts/distmnt"
328 run-parts --report $base/scripts/distmnt
332 log "Exporting package data foo for i18n project"
333 STAMP=$(date "+%Y%m%d%H%M")
334 mkdir -p ${scriptdir}/i18n/${STAMP}
335 cd ${scriptdir}/i18n/${STAMP}
336 dak control-suite -l stable > lenny
337 dak control-suite -l testing > squeeze
338 dak control-suite -l unstable > sid
339 echo "${STAMP}" > timestamp
340 gpg --secret-keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 6070D3A1 --detach-sign -o timestamp.gpg timestamp
344 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
347 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
351 log "Updating stats data"
353 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
354 R --slave --vanilla < $base/misc/ftpstats.R
355 dak stats arch-space > $webdir/arch-space
356 dak stats pkg-nums > $webdir/pkg-nums
359 function aptftpcleanup() {
360 log "Clean up apt-ftparchive's databases"
362 apt-ftparchive -q clean apt.conf
365 function compress() {
366 log "Compress old psql backups"
368 find -maxdepth 1 -mindepth 1 -type f -name 'dump_pre_*' -mtime +2 -print0 | xargs -0 --no-run-if-empty rm
370 find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mmin +720 |
371 while read dumpname; do
372 echo "Compressing $dumpname"
373 bzip2 -9fv "$dumpname"
375 find -maxdepth 1 -mindepth 1 -type f -name "dumpall_*" \! -name '*.bz2' \! -name '*.gz' -mmin +720 |
376 while read dumpname; do
377 echo "Compressing $dumpname"
378 bzip2 -9fv "$dumpname"
380 finddup -l -d $base/backup
383 function logstats() {
384 $masterdir/tools/logs.py "$1"
387 # save timestamp when we start
388 function savetimestamp() {
389 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
390 echo ${NOW} > "${dbdir}/dinstallstart"
393 function maillogfile() {
394 cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
397 function renamelogfile() {
398 if [ -f "${dbdir}/dinstallstart" ]; then
399 NOW=$(cat "${dbdir}/dinstallstart")
401 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
402 logstats "$logdir/dinstall_${NOW}.log"
403 bzip2 -9 "$logdir/dinstall_${NOW}.log"
405 error "Problem, I don't know when dinstall started, unable to do log statistics."
406 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
408 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
409 bzip2 -9 "$logdir/dinstall_${NOW}.log"
413 function testingsourcelist() {
414 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
417 # do a last run of process-unchecked before dinstall is on.
418 function process_unchecked() {
419 log "Processing the unchecked queue"
421 UNCHECKED_WITHOUT_LOCK="-p"
426 ########################################################################
427 ########################################################################
429 # Function to save which stage we are in, so we can restart an interrupted
430 # dinstall. Or even run actions in parallel, if we dare to, by simply
431 # backgrounding the call to this function. But that should only really be
432 # done for things we dont care much about.
434 # This should be called with the first argument being an array, with the
436 # - FUNC - the function name to call
437 # - ARGS - Possible arguments to hand to the function. Can be the empty string
438 # - TS - The timestamp name. Can be the empty string
439 # - ERR - if this is the string false, then the call will be surrounded by
440 # set +e ... set -e calls, so errors in the function do not exit
441 # dinstall. Can be the empty string, meaning true.
443 # MAKE SURE TO KEEP THIS THE LAST FUNCTION, AFTER ALL THE VARIOUS ONES
444 # ADDED FOR DINSTALL FEATURES!
449 STAGEFILE="${stagedir}/${FUNC}"
450 if [ -f "${STAGEFILE}" ]; then
451 stamptime=$(/usr/bin/stat -c %Z "${STAGEFILE}")
453 difference=$(( $unixtime - $stamptime ))
454 if [ ${difference} -ge 14400 ]; then
455 log_error "Did already run ${FUNC}, stagefile exists, but that was ${difference} seconds ago. Please check."
457 log "Did already run ${FUNC}, not calling again..."
462 debug "Now calling function ${FUNC}. Arguments: ${ARGS}. Timestamp: ${TS}"
464 # Make sure we are always at the same place. If a function wants to be elsewhere,
465 # it has to cd first!
468 # Now redirect the output into $STAGEFILE.log. In case it errors out somewhere our
469 # errorhandler trap can then mail the contents of $STAGEFILE.log only, instead of a whole
470 # dinstall logfile. Short error mails ftw!
471 exec >> "${STAGEFILE}.log" 2>&1
473 if [ -f "${LOCK_STOP}" ]; then
474 log "${LOCK_STOP} exists, exiting immediately"
478 if [ "${ERR}" = "false" ]; then
483 # No matter what happened in the function, we make sure we have set -e default state back
486 # Make sure we are always at the same place.
491 if [ -n "${TIME}" ]; then
495 # And the output goes back to the normal logfile
496 exec >> "$LOGFILE" 2>&1
498 # Now we should make sure that we have a usable dinstall.log, so append the $STAGEFILE.log
500 cat "${STAGEFILE}.log" >> "${LOGFILE}"
501 rm -f "${STAGEFILE}.log"
503 if [ -f "${LOCK_STOP}" ]; then
504 log "${LOCK_STOP} exists, exiting immediately"
509 ########################################################################
512 LOGFILE="$logdir/dinstall.log"
514 exec >> "$LOGFILE" 2>&1
516 # usually we are not using debug logs. Set to 1 if you want them.
522 # where do we want mails to go? For example log entries made with error()
523 if [ "x$(hostname -s)x" != "xriesx" ]; then
524 # Not our ftpmaster host
525 MAILTO=${MAILTO:-"root"}
528 MAILTO=${MAILTO:-"ftpmaster@debian.org"}
531 # How many logfiles to keep
532 LOGROTATE=${LOGROTATE:-400}
534 # Marker for dinstall start
535 DINSTALLSTART="${lockdir}/dinstallstart"
536 # Marker for dinstall end
537 DINSTALLEND="${lockdir}/dinstallend"
539 # Timestamps start at -1. so first gets 0
541 touch "${DINSTALLSTART}"
544 # Tell everyone we are doing some work
545 NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
547 # lock cron.unchecked (it immediately exits when this exists)
548 LOCK_DAILY="$lockdir/daily.lock"
550 # Lock cron.unchecked from doing work
551 LOCK_ACCEPTED="$lockdir/unchecked.lock"
553 # Lock process-new from doing work
554 LOCK_NEW="$lockdir/processnew.lock"
556 # This file is simply used to indicate to britney whether or not
557 # the Packages file updates completed sucessfully. It's not a lock
558 # from our point of view
559 LOCK_BRITNEY="$lockdir/britney.lock"
561 # If this file exists we exit immediately after the currently running
563 LOCK_STOP="$lockdir/archive.stop"
565 lockfile -l 3600 "${LOCK_DAILY}"
567 trap cleanup EXIT TERM HUP INT QUIT
569 touch "${LOCK_BRITNEY}"
605 TIME="External Updates"
635 lockfile "$LOCK_ACCEPTED"
639 FUNC="process_unchecked"
671 rm -f "$LOCK_ACCEPTED"
676 TIME="make-suite-file-list"
684 TIME="import-keyring"
700 TIME="pkg-file-mapping"
708 TIME="apt-ftparchive"
756 TIME="mirror hardlinks"
771 rm -f "${LOCK_DAILY}"
773 ts "locked part finished"
792 FUNC="transitionsclean"
793 TIME="transitionsclean"
825 TIME="merkel projectb push"
856 FUNC="testingsourcelist"
863 rm -f ${LOCK_BRITNEY}
875 TIME="apt-ftparchive cleanup"
883 TIME="merkel ddaccessible sync"
897 log "Daily cron scripts successful, all done"
899 exec > "$logdir/afterdinstall.log" 2>&1
910 # Now, at the very (successful) end of dinstall, make sure we remove
911 # our stage files, so the next dinstall run will do it all again.
913 touch "${DINSTALLEND}"