2 # Timestamp. Used for dinstall stat graphs
4 echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
8 function remove_daily_lock() {
13 function remove_all_locks() {
14 rm -f $LOCK_DAILY $LOCK_ACCEPTED $LOCK_NEW
17 # If we error out this one is called, *FOLLOWED* by cleanup above
19 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
21 subject="ATTENTION ATTENTION!"
22 if [ "${error}" = "false" ]; then
23 subject="${subject} (continued)"
25 subject="${subject} (interrupted)"
27 subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)"
29 if [ -r "${STAGEFILE}.log" ]; then
30 cat "${STAGEFILE}.log"
32 echo "file ${STAGEFILE}.log does not exist, sorry"
33 fi | mail -s "${subject}" -a "X-Debian: DAK" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
36 ########################################################################
37 # the actual dinstall functions follow #
38 ########################################################################
40 # pushing merkels QA user, part one
42 log "Telling merkels QA user that we start dinstall"
43 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1
46 # Create the postgres dump files
47 function pgdump_pre() {
48 log "Creating pre-daily-cron-job backup of $PGDATABASE database..."
49 pg_dump > $base/backup/dump_pre_$(date +%Y.%m.%d-%H:%M:%S)
52 function pgdump_post() {
53 log "Creating post-daily-cron-job backup of $PGDATABASE database..."
55 POSTDUMP=$(date +%Y.%m.%d-%H:%M:%S)
56 pg_dump > $base/backup/dump_$POSTDUMP
57 #pg_dumpall --globals-only > $base/backup/dumpall_$POSTDUMP
58 ln -sf $base/backup/dump_$POSTDUMP current
59 #ln -sf $base/backup/dumpall_$POSTDUMP currentall
62 # Updating various files
64 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
66 $scriptsdir/update-bugdoctxt
67 $scriptsdir/update-mirrorlists
68 $scriptsdir/update-mailingliststxt
69 $scriptsdir/update-pseudopackages.sh
72 # Process (oldstable)-proposed-updates "NEW" queue
75 dak process-policy $1 | tee -a REPORT | mail -a "X-Debian: DAK" -e -s "NEW changes in $1" debian-release@lists.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
79 log "Doing automated p-u-new processing"
80 cd "${queuedir}/p-u-new"
84 log "Doing automated o-p-u-new processing"
85 cd "${queuedir}/o-p-u-new"
89 # The first i18n one, syncing new descriptions
91 log "Synchronizing i18n package descriptions"
92 # First sync their newest data
93 cd ${scriptdir}/i18nsync
94 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
96 # Now check if we still know about the packages for which they created the files
97 # is the timestamp signed by us?
98 if $(gpgv --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
99 # now read it. As its signed by us we are sure the content is what we expect, no need
100 # to do more here. And we only test -d a directory on it anyway.
101 TSTAMP=$(cat timestamp)
102 # do we have the dir still?
103 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
105 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
106 # Yay, worked, lets copy around
107 for dir in wheezy sid; do
108 if [ -d dists/${dir}/ ]; then
109 cd dists/${dir}/main/i18n
110 rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
112 cd ${scriptdir}/i18nsync
115 echo "ARRRR, bad guys, wrong files, ARRR"
116 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
119 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
120 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
123 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
124 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
129 log "Checking for cruft in overrides"
133 function dominate() {
134 log "Removing obsolete source and binary associations"
138 function filelist() {
139 log "Generating file lists for apt-ftparchive"
140 dak generate-filelist
143 function fingerprints() {
144 log "Updating fingerprints"
145 dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
148 dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}"
150 if [ -s "${OUTFILE}" ]; then
151 /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org <<EOF
152 From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>
153 To: <debian-project@lists.debian.org>
154 Subject: Debian Maintainers Keyring changes
155 Content-Type: text/plain; charset=utf-8
159 The following changes to the debian-maintainers keyring have just been activated:
163 Debian distribution maintenance software,
164 on behalf of the Keyring maintainers
171 function overrides() {
172 log "Writing overrides into text files"
177 rm -f override.sid.all3
178 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
182 log "Generating package / file mapping"
183 dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
186 function packages() {
187 log "Generating Packages and Sources files"
189 #apt-ftparchive generate apt.conf
190 dak generate-packages-sources
191 dak contents generate
195 log "Generating pdiff files"
196 dak generate-index-diffs
200 log "Generating Release files"
201 dak generate-releases
204 function dakcleanup() {
205 log "Cleanup old packages/files"
206 dak clean-suites -m 10000
210 function buildd_dir() {
211 # Rebuilt the buildd dir to avoid long times of 403
212 log "Regenerating the buildd incoming dir"
213 STAMP=$(date "+%Y%m%d%H%M")
222 log "Removing any core files ..."
223 find -type f -name core -print0 | xargs -0r rm -v
225 log "Checking permissions on files in the FTP tree ..."
226 find -type f \( \! -perm -444 -o -perm +002 \) -ls
227 find -type d \( \! -perm -555 -o -perm +002 \) -ls
229 log "Checking symlinks ..."
232 log "Creating recursive directory listing ... "
233 rm -f .${FILENAME}.new
234 TZ=UTC ls -lR > .${FILENAME}.new
236 if [ -r ${FILENAME}.gz ] ; then
237 mv -f ${FILENAME}.gz ${FILENAME}.old.gz
238 mv -f .${FILENAME}.new ${FILENAME}
239 rm -f ${FILENAME}.patch.gz
240 zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip -9cfn - >${FILENAME}.patch.gz
241 rm -f ${FILENAME}.old.gz
243 mv -f .${FILENAME}.new ${FILENAME}
246 gzip -9cfN ${FILENAME} >${FILENAME}.gz
250 function mkmaintainers() {
251 log 'Creating Maintainers index ... '
254 dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers | \
255 sed -e "s/~[^ ]*\([ ]\)/\1/" | \
256 awk '{printf "%-20s ", $1; for (i=2; i<=NF; i++) printf "%s ", $i; printf "\n";}' > .new-maintainers
258 if ! cmp -s .new-maintainers Maintainers || [ ! -f Maintainers ]; then
259 log "installing Maintainers ... "
260 mv -f .new-maintainers Maintainers
261 gzip -9v <Maintainers >.new-maintainers.gz
262 mv -f .new-maintainers.gz Maintainers.gz
264 rm -f .new-maintainers
268 function mkuploaders() {
269 log 'Creating Uploaders index ... '
272 dak make-maintainers -u ${scriptdir}/masterfiles/pseudo-packages.maintainers | \
273 sed -e "s/~[^ ]*\([ ]\)/\1/" | \
274 awk '{printf "%-20s ", $1; for (i=2; i<=NF; i++) printf "%s ", $i; printf "\n";}' > .new-uploaders
276 if ! cmp -s .new-uploaders Uploaders || [ ! -f Uploaders ]; then
277 log "installing Uploaders ... "
278 mv -f .new-uploaders Uploaders
279 gzip -9v <Uploaders >.new-uploaders.gz
280 mv -f .new-uploaders.gz Uploaders.gz
286 function copyoverrides() {
287 log 'Copying override files into public view ...'
289 for ofile in $copyoverrides ; do
291 chmod g+w override.$ofile
295 newofile=override.$ofile.gz
296 rm -f .newover-$ofile.gz
297 pc="`gzip 2>&1 -9nv <$overridedir/override.$ofile >.newover-$ofile.gz`"
298 if ! cmp -s .newover-$ofile.gz $newofile || [ ! -f $newofile ]; then
299 log " installing new $newofile $pc"
300 mv -f .newover-$ofile.gz $newofile
303 rm -f .newover-$ofile.gz
308 function mkfilesindices() {
310 cd $base/ftp/indices/files/components
314 log "Querying $PGDATABASE..."
315 echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql -At | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST
318 perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
321 perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);'
324 log "Generating sources list"
326 sed -n 's/|$//p' $ARCHLIST
328 find ./dists -maxdepth 1 \! -type d
329 find ./dists \! -type d | grep "/source/"
330 ) | sort -u | gzip -9 > source.list.gz
332 log "Generating arch lists"
334 ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u)
336 (sed -n "s/|$a$//p" $ARCHLIST
337 sed -n 's/|all$//p' $ARCHLIST
340 find ./dists -maxdepth 1 \! -type d
341 find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
342 ) | sort -u | gzip -9 > arch-$a.list.gz
345 log "Generating suite lists"
348 printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)\n' $1 $1 | psql -F' ' -A -t
350 printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d\n' $1 | psql -F' ' -A -t
353 printf 'SELECT id, suite_name FROM suite\n' | psql -F' ' -At |
354 while read id suite; do
355 [ -e $base/ftp/dists/$suite ] || continue
358 distname=$(cd dists; readlink $suite || echo $suite)
359 find ./dists/$distname \! -type d
360 for distdir in ./dists/*; do
361 [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
364 suite_list $id | tr -d ' ' | sed 's,^/srv/ftp-master.debian.org/ftp,.,'
365 ) | sort -u | gzip -9 > suite-${suite}.list.gz
368 log "Finding everything on the ftp site to generate sundries"
369 (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST
372 zcat *.list.gz | cat - *.list | sort -u |
373 diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list
375 log "Generating files list"
378 (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) |
379 cat - sundries.list dists.list project.list docs.list indices.list |
380 sort -u | poolfirst > ../arch-$a.files
384 for dist in sid wheezy; do
385 find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
389 (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-wheezy.list.gz) |
390 sort -u | poolfirst > ../typical.files
396 function mkchecksums() {
397 dsynclist=$dbdir/dsync.list
398 md5list=$indices/md5sums
400 log -n "Creating md5 / dsync index file ... "
403 ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
404 ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz
405 ${bindir}/dsync-flist -q link-dups $dsynclist || true
409 log "Regenerating \"public\" mirror/ hardlink fun"
410 DATE_SERIAL=$(date +"%Y%m%d01")
411 FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} )
412 if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then
413 SERIAL="$DATE_SERIAL"
415 SERIAL="$FILESOAPLUS1"
417 date -u > ${TRACEFILE}
418 echo "Using dak v1" >> ${TRACEFILE}
419 echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
420 echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
422 rsync -aH --link-dest ${ftpdir} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${ftpdir}/. .
426 log "Expiring old database dumps..."
428 $scriptsdir/expire_dumps -d . -p -f "dump_*"
431 function transitionsclean() {
432 log "Removing out of date transitions..."
434 dak transitions -c -a
438 log "Updating DM html page"
439 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
443 log "Categorizing uncategorized bugs filed against ftp.debian.org"
448 # Push dak@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
449 log "Trigger merkel/flotows $PGDATABASE sync"
450 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb dak@merkel.debian.org sleep 1
451 # Also trigger flotow, the ftpmaster test box
452 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_flotow_projectb dak@flotow.debconf.org sleep 1
456 # Push dak@merkel to tell it to sync the dd accessible parts. Returns immediately, the sync runs detached
457 log "Trigger merkels dd accessible parts sync"
458 ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_ddaccess dak@merkel.debian.org sleep 1
461 function mirrorpush() {
462 log "Starting the mirrorpush"
463 date -u > /srv/ftp.debian.org/web/mirrorstart
464 echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart
465 echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/mirrorstart
466 sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
470 log "Exporting package data foo for i18n project"
471 STAMP=$(date "+%Y%m%d%H%M")
472 mkdir -p ${scriptdir}/i18n/${STAMP}
473 cd ${scriptdir}/i18n/${STAMP}
474 dak control-suite -l stable > squeeze
475 dak control-suite -l testing > wheezy
476 dak control-suite -l unstable > sid
477 echo "${STAMP}" > timestamp
478 gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp
482 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
485 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
489 log "Updating stats data"
491 $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
492 R --slave --vanilla < $base/misc/ftpstats.R
493 dak stats arch-space > $webdir/arch-space
494 dak stats pkg-nums > $webdir/pkg-nums
497 function aptftpcleanup() {
498 log "Clean up apt-ftparchive's databases"
500 apt-ftparchive -q clean apt.conf
503 function cleantransactions() {
504 log "Cleanup transaction ids older than 3 months"
506 find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm
509 function logstats() {
510 $masterdir/tools/logs.py "$1"
513 # save timestamp when we start
514 function savetimestamp() {
515 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
516 echo ${NOW} > "${dbdir}/dinstallstart"
519 function maillogfile() {
520 cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
523 function renamelogfile() {
524 if [ -f "${dbdir}/dinstallstart" ]; then
525 NOW=$(cat "${dbdir}/dinstallstart")
527 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
528 logstats "$logdir/dinstall_${NOW}.log"
529 bzip2 -9 "$logdir/dinstall_${NOW}.log"
531 error "Problem, I don't know when dinstall started, unable to do log statistics."
532 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
534 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
535 bzip2 -9 "$logdir/dinstall_${NOW}.log"
539 function testingsourcelist() {
540 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
543 # do a last run of process-unchecked before dinstall is on.
544 function process_unchecked() {
545 log "Processing the unchecked queue"
546 UNCHECKED_WITHOUT_LOCK="-p"
551 # do a run of newstage only before dinstall is on.
552 function newstage() {
553 log "Processing the newstage queue"
554 UNCHECKED_WITHOUT_LOCK="-p"
558 # Function to update a "statefile" telling people what we are doing
561 # This should be called with the argument(s)
562 # - Status name we want to show.
565 RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")"
566 cat >"${DINSTALLSTATE}" <<EOF
567 Dinstall start: ${DINSTALLBEGIN}
569 Action start: ${RIGHTNOW}
573 # extract changelogs and stuff
574 function changelogs() {
575 log "Extracting changelogs"
576 dak make-changelog -e
577 mkdir -p ${exportpublic}/changelogs
578 cd ${exportpublic}/changelogs
579 rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
580 sudo -H -u archvsync /home/archvsync/runmirrors metaftpdo > ~dak/runmirrors-metadata.log 2>&1 &