2 # Timestamp. Used for dinstall stat graphs
4 echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
10 rm -f ${LOCK_ACCEPTED}
13 # If we error out this one is called, *FOLLOWED* by cleanup above
15 ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
17 subject="ATTENTION ATTENTION!"
18 if [ "${error}" = "false" ]; then
19 subject="${subject} (continued)"
21 subject="${subject} (interrupted)"
23 subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)"
25 if [ -r "${STAGEFILE}.log" ]; then
26 cat "${STAGEFILE}.log"
28 echo "file ${STAGEFILE}.log does not exist, sorry"
29 fi | mail -s "${subject}" -a "X-Debian: DAK" cron@backports.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
32 ########################################################################
33 # the actual dinstall functions follow #
34 ########################################################################
37 log "Telling QA user that we start dinstall"
38 ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1
41 # Updating various files
43 log "Updating Bugs docu, Mirror list and mailing-lists.txt"
45 $scriptsdir/update-bugdoctxt
46 $scriptsdir/update-mirrorlists
47 $scriptsdir/update-mailingliststxt
48 $scriptsdir/update-pseudopackages.sh
51 # Process (oldstable)-proposed-updates "NEW" queue
54 dak process-policy $1 | tee -a REPORT | mail -a "X-Debian: DAK" -e -s "NEW changes in $1" team@backports.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
58 log "Doing automated p-u-new processing"
59 cd "${queuedir}/p-u-new"
63 log "Doing automated o-p-u-new processing"
64 cd "${queuedir}/o-p-u-new"
68 # The first i18n one, syncing new descriptions
70 log "Synchronizing i18n package descriptions"
71 # First sync their newest data
72 cd ${scriptdir}/i18nsync
73 rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true
75 # Now check if we still know about the packages for which they created the files
76 # is the timestamp signed by us?
77 if $(gpgv --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then
78 # now read it. As its signed by us we are sure the content is what we expect, no need
79 # to do more here. And we only test -d a directory on it anyway.
80 TSTAMP=$(cat timestamp)
81 # do we have the dir still?
82 if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then
84 if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
85 # Yay, worked, lets copy around
86 for dir in wheezy sid; do
87 if [ -d dists/${dir}/ ]; then
88 cd dists/${dir}/main/i18n
89 rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/.
91 cd ${scriptdir}/i18nsync
94 echo "ARRRR, bad guys, wrong files, ARRR"
95 echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
98 echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
99 echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
102 echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
103 echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
108 log "Checking for cruft in overrides"
112 function dominate() {
113 log "Removing obsolete source and binary associations"
117 function filelist() {
118 log "Generating file lists for apt-ftparchive"
119 dak generate-filelist
122 function fingerprints() {
123 log "Updating fingerprints"
124 dak import-keyring -L /srv/backports-master.debian.org/keyrings/keyring.gpg
127 dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}"
129 if [ -s "${OUTFILE}" ]; then
130 /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org <<EOF
131 From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>
132 To: <debian-backports@lists.debian.org>
133 Subject: Debian Maintainers Keyring changes
134 Content-Type: text/plain; charset=utf-8
138 The following changes to the debian-maintainers keyring have just been activated:
142 Debian distribution maintenance software,
143 on behalf of the Keyring maintainers
150 function overrides() {
151 log "Writing overrides into text files"
156 rm -f override.sid.all3
157 for i in main contrib non-free main.debian-installer; do cat override.squeeze-backports.$i >> override.sid.all3; done
161 log "Generating package / file mapping"
162 dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
165 function packages() {
166 log "Generating Packages and Sources files"
168 apt-ftparchive generate apt.conf
169 #dak generate-packages-sources
173 log "Generating pdiff files"
174 dak generate-index-diffs
178 log "Generating Release files"
179 dak generate-releases
182 function dakcleanup() {
183 log "Cleanup old packages/files"
184 dak clean-suites -m 10000
188 function buildd_dir() {
189 # Rebuilt the buildd dir to avoid long times of 403
190 log "Regenerating the buildd incoming dir"
191 STAMP=$(date "+%Y%m%d%H%M")
200 log "Removing any core files ..."
201 find -type f -name core -print0 | xargs -0r rm -v
203 log "Checking permissions on files in the FTP tree ..."
204 find -type f \( \! -perm -444 -o -perm +002 \) -ls
205 find -type d \( \! -perm -555 -o -perm +002 \) -ls
207 log "Checking symlinks ..."
210 log "Creating recursive directory listing ... "
211 rm -f .${FILENAME}.new
212 TZ=UTC ls -lR > .${FILENAME}.new
214 if [ -r ${FILENAME}.gz ] ; then
215 mv -f ${FILENAME}.gz ${FILENAME}.old.gz
216 mv -f .${FILENAME}.new ${FILENAME}
217 rm -f ${FILENAME}.patch.gz
218 zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip -9cfn - >${FILENAME}.patch.gz
219 rm -f ${FILENAME}.old.gz
221 mv -f .${FILENAME}.new ${FILENAME}
224 gzip -9cfN ${FILENAME} >${FILENAME}.gz
228 function mkmaintainers() {
229 log 'Creating Maintainers index ... '
232 dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers
233 gzip -9v --rsyncable <Maintainers >Maintainers.gz
234 gzip -9v --rsyncable <Uploaders >Uploaders.gz
237 function copyoverrides() {
238 log 'Copying override files into public view ...'
240 for ofile in $copyoverrides ; do
242 chmod g+w override.$ofile
246 newofile=override.$ofile.gz
247 rm -f .newover-$ofile.gz
248 pc="`gzip 2>&1 -9nv <$overridedir/override.$ofile >.newover-$ofile.gz`"
249 if ! cmp -s .newover-$ofile.gz $newofile || [ ! -f $newofile ]; then
250 log " installing new $newofile $pc"
251 mv -f .newover-$ofile.gz $newofile
254 rm -f .newover-$ofile.gz
259 function mkfilesindices() {
261 cd $base/ftp/indices/files/components
265 log "Querying postgres"
267 SELECT CONCAT('./pool/', c.name, '/', f.filename) AS path, a.arch_string AS arch_string
269 JOIN files_archive_map af ON f.id = af.file_id
270 JOIN component c ON af.component_id = c.id
271 JOIN archive ON af.archive_id = archive.id
274 JOIN architecture a ON b.architecture = a.id)
276 WHERE archive.name = 'ftp-master'
277 ORDER BY path, arch_string
279 psql -At -c "$query" >$ARCHLIST
282 perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
285 perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);'
288 log "Generating sources list"
290 sed -n 's/|$//p' $ARCHLIST
292 find ./dists -maxdepth 1 \! -type d
293 find ./dists \! -type d | grep "/source/"
294 ) | sort -u | gzip -9 > source.list.gz
296 log "Generating arch lists"
298 ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u)
300 (sed -n "s/|$a$//p" $ARCHLIST
301 sed -n 's/|all$//p' $ARCHLIST
304 find ./dists -maxdepth 1 \! -type d
305 find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)"
306 ) | sort -u | gzip -9 > arch-$a.list.gz
309 log "Generating suite lists"
312 local suite_id="$(printf %d $1)"
315 SELECT DISTINCT CONCAT('./pool/', c.name, '/', f.filename)
317 (SELECT sa.source AS source
318 FROM src_associations sa
319 WHERE sa.suite = $suite_id
321 SELECT b.source AS source
322 FROM bin_associations ba
323 JOIN binaries b ON ba.bin = b.id WHERE ba.suite = $suite_id) s
324 JOIN dsc_files df ON s.source = df.source
325 JOIN files f ON df.file = f.id
326 JOIN files_archive_map af ON f.id = af.file_id
327 JOIN component c ON af.component_id = c.id
328 JOIN archive ON af.archive_id = archive.id
329 WHERE archive.name = 'ftp-master'
331 psql -F' ' -A -t -c "$query"
334 SELECT CONCAT('./pool/', c.name, '/', f.filename)
335 FROM bin_associations ba
336 JOIN binaries b ON ba.bin = b.id
337 JOIN files f ON b.file = f.id
338 JOIN files_archive_map af ON f.id = af.file_id
339 JOIN component c ON af.component_id = c.id
340 JOIN archive ON af.archive_id = archive.id
341 WHERE ba.suite = $suite_id AND archive.name = 'ftp-master'
343 psql -F' ' -A -t -c "$query"
346 psql -F' ' -At -c 'SELECT id, suite_name FROM suite' |
347 while read id suite; do
348 [ -e $base/ftp/dists/$suite ] || continue
351 distname=$(cd dists; readlink $suite || echo $suite)
352 find ./dists/$distname \! -type d
353 for distdir in ./dists/*; do
354 [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
358 ) | sort -u | gzip -9 > suite-${suite}.list.gz
361 log "Finding everything on the ftp site to generate sundries"
362 (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST
365 zcat *.list.gz | cat - *.list | sort -u |
366 diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list
368 log "Generating files list"
371 (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) |
372 cat - sundries.list dists.list project.list docs.list indices.list |
373 sort -u | poolfirst > ../arch-$a.files
377 for dist in sid wheezy; do
378 find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
382 (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-wheezy.list.gz) |
383 sort -u | poolfirst > ../typical.files
389 function mkchecksums() {
390 dsynclist=$dbdir/dsync.list
391 md5list=$indices/md5sums
393 log -n "Creating md5 / dsync index file ... "
396 ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
397 ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz
398 ${bindir}/dsync-flist -q link-dups $dsynclist || true
402 log "Regenerating \"public\" mirror/ hardlink fun"
403 DATE_SERIAL=$(date +"%Y%m%d01")
404 FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} )
405 if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then
406 SERIAL="$DATE_SERIAL"
408 SERIAL="$FILESOAPLUS1"
410 date -u > ${TRACEFILE}
411 echo "Using dak v1" >> ${TRACEFILE}
412 echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
413 echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
415 rsync -aH --link-dest ${ftpdir} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${ftpdir}/. .
419 log "Expiring old database dumps..."
421 $scriptsdir/expire_dumps -d . -p -f "dump_*"
424 function transitionsclean() {
425 log "Removing out of date transitions..."
427 dak transitions -c -a
431 log "Updating DM html page"
432 $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
436 log "Categorizing uncategorized bugs filed against ftp.debian.org"
440 function ddaccess() {
441 # Tell our dd accessible mirror to sync itself up. Including ftp dir.
442 log "Trigger dd accessible parts sync including ftp dir"
443 ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/pushddmirror dak@ries.debian.org pool
446 function mirrorpush() {
447 log "Starting the mirrorpush"
448 date -u > /srv/backports-web.debian.org/underlay/mirrorstart
449 echo "Using dak v1" >> /srv/backports-web.debian.org/underlay/mirrorstart
450 echo "Running on host $(hostname -f)" >> /srv/backports-web.debian.org/underlay/mirrorstart
451 sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
455 log "Exporting package data foo for i18n project"
456 STAMP=$(date "+%Y%m%d%H%M")
457 mkdir -p ${scriptdir}/i18n/${STAMP}
458 cd ${scriptdir}/i18n/${STAMP}
459 dak control-suite -l stable > squeeze
460 dak control-suite -l testing > wheezy
461 dak control-suite -l unstable > sid
462 echo "${STAMP}" > timestamp
463 gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp
467 ln -sfT ${scriptdir}/i18n/${STAMP} i18n
470 find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
474 log "Updating stats data"
476 #$scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
477 #R --slave --vanilla < $base/misc/ftpstats.R
478 dak stats arch-space > $webdir/arch-space
479 dak stats pkg-nums > $webdir/pkg-nums
482 function aptftpcleanup() {
483 log "Clean up apt-ftparchive's databases"
485 apt-ftparchive -q clean apt.conf
488 function cleantransactions() {
489 log "Cleanup transaction ids older than 3 months"
491 find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm
494 function logstats() {
495 $masterdir/tools/logs.py "$1"
498 # save timestamp when we start
499 function savetimestamp() {
500 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
501 echo ${NOW} > "${dbdir}/dinstallstart"
504 function maillogfile() {
505 cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
508 function renamelogfile() {
509 if [ -f "${dbdir}/dinstallstart" ]; then
510 NOW=$(cat "${dbdir}/dinstallstart")
512 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
513 # logstats "$logdir/dinstall_${NOW}.log"
514 bzip2 -9 "$logdir/dinstall_${NOW}.log"
516 error "Problem, I don't know when dinstall started, unable to do log statistics."
517 NOW=`date "+%Y.%m.%d-%H:%M:%S"`
519 mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
520 bzip2 -9 "$logdir/dinstall_${NOW}.log"
524 function testingsourcelist() {
525 dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
528 # do a last run of process-unchecked before dinstall is on.
529 function process_unchecked() {
530 log "Processing the unchecked queue"
531 UNCHECKED_WITHOUT_LOCK="-p"
536 # do a run of newstage only before dinstall is on.
537 function newstage() {
538 log "Processing the newstage queue"
539 UNCHECKED_WITHOUT_LOCK="-p"
543 # Function to update a "statefile" telling people what we are doing
546 # This should be called with the argument(s)
547 # - Status name we want to show.
550 RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")"
551 cat >"${DINSTALLSTATE}" <<EOF
552 Dinstall start: ${DINSTALLBEGIN}
554 Action start: ${RIGHTNOW}
558 # extract changelogs and stuff
559 function changelogs() {
560 log "Extracting changelogs"
561 dak make-changelog -e
562 mkdir -p ${exportpublic}/changelogs
563 cd ${exportpublic}/changelogs
564 rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
565 sudo -H -u archvsync /home/archvsync/runmirrors metabdo > ~dak/runmirrors-metadata.log 2>&1 &