From 636fd44b6b3b8f343eb53c4ef2dd053fb91bd4d6 Mon Sep 17 00:00:00 2001 From: Andreas Barth Date: Wed, 10 Mar 2010 22:12:41 +0000 Subject: [PATCH] add non-free files to quinn-diff (and to wanna-build) (wanna-build hands them only out if asked to do so) also inlines the loop-variants to only check the sources.list once --- triggers/common | 49 ++++++++++++------- triggers/filter-nonfree | 49 +++++++++++++++++++ triggers/trigger.debian | 2 +- triggers/trigger.debian-buildd | 10 ++-- ...trigger_wb_update_arch_loop_with_secondary | 4 +- .../trigger_wb_update_for_overlay_arch_loop | 4 +- 6 files changed, 90 insertions(+), 28 deletions(-) create mode 100755 triggers/filter-nonfree diff --git a/triggers/common b/triggers/common index 2b0062f..18408e6 100644 --- a/triggers/common +++ b/triggers/common @@ -39,13 +39,12 @@ get_architectures() { do_quinn() { SUITE="$1" ARCH="$2" - WHITELIST_FILE="$3" quinn-diff \ -A $ARCH \ -a "$PAS_BASE/$SUITE/Packages-arch-specific" \ -s Sources.$SUITE \ -p Packages.$SUITE.$ARCH \ - | filter-q-d "${WHITELIST_FILE}" > quinn-$SUITE.$ARCH 2>/dev/null + > quinn-$SUITE.$ARCH 2>/dev/null } # Call wanna-build's importing function. WB_SUITE might differ from SUITE @@ -106,11 +105,15 @@ trigger_wb_update() { echo "`date`: Processing ${ARCHIVE}/${SUITE} ..." - new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE} - parallel -l 5 -i trigger_wb_update_arch_loop "${SUITE}" "{}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" -- ${ARCHS} || true - #for ARCH in ${ARCHS}; do - #trigger_wb_update_arch_loop "${SUITE}" "${ARCH}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" - #done + new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE}.full + for ARCH in ${ARCHS}; do + new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}.full + done + filter-nonfree .full /org/wanna-build/etc/non-free-include-list Sources.${SUITE}.full $(for ARCH in ${ARCHS}; do echo Packages.${SUITE}.${ARCH}.full; done) + for ARCH in ${ARCHS}; do + do_quinn ${SUITE} ${ARCH} + do_wb_merge_all ${SUITE} ${ARCH} ${WB_SUITE} + done echo "`date`: ${ARCHIVE}/${SUITE} done" } @@ -132,13 +135,18 @@ trigger_wb_update_for_overlay() { echo "`date`: Processing ${ARCHIVE}/${SUITE} ..." - new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE} + new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE}.full new-keep-latest source $(eval echo Sources.${SUITE} ${SOURCES_BASE}) > Sources.${SUITE}-all - parallel -l 5 -i trigger_wb_update_for_overlay_arch_loop "${SUITE}" "{}" "${SOURCES_BASE}" "${PACKAGES_BASE}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" "${WHITELIST_FILE}" -- ${ARCHS} || true - #for ARCH in ${ARCHS}; do - #trigger_wb_update_for_overlay_arch_loop "${SUITE}" "${ARCH}" "${SOURCES_BASE}" "${PACKAGES_BASE}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" "${WHITELIST_FILE}" - #done + for ARCH in ${ARCHS}; do + new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}.full + new-keep-latest ${ARCH} Packages.${SUITE}.${ARCH} $(eval echo $(echo ${PACKAGES_BASE} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}-all.full + done + filter-nonfree .full /org/wanna-build/etc/non-free-include-list Sources.${SUITE}.full $(for ARCH in ${ARCHS}; do echo Packages.${SUITE}.${ARCH}.full; done) + for ARCH in ${ARCHS}; do + do_quinn ${SUITE} ${ARCH} + do_wb_merge_all ${SUITE} ${ARCH} ${WB_SUITE} overlay + done echo "`date`: ${ARCHIVE}/${SUITE} done" } @@ -164,12 +172,17 @@ trigger_wb_update_with_secondary() { echo "`date`: Processing ${ARCHIVE}/${SUITE} ..." - new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE} - - parallel -l 5 -i trigger_wb_update_arch_loop_with_secondary "${SUITE}" "{}" "${PACKAGES_BASE}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" "${WHITELIST_FILE}" -- ${ARCHS} || true - #for ARCH in ${ARCHS}; do - #trigger_wb_update_arch_loop_with_secondary "${SUITE}" "${ARCH}" "${PACKAGES_BASE}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" "${WHITELIST_FILE}" - #done + new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE}.full + + for ARCH in ${ARCHS}; do + new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}.full + new-keep-latest ${ARCH} Packages.${SUITE}.${ARCH} $(eval echo $(echo ${PACKAGES_BASE} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}-all.full + done + filter-nonfree .full /org/wanna-build/etc/non-free-include-list Sources.${SUITE}.full $(for ARCH in ${ARCHS}; do echo Packages.${SUITE}.${ARCH}.full Packages.${SUITE}.${ARCH}-all.full; done) + for ARCH in ${ARCHS}; do + do_quinn ${SUITE} ${ARCH} + do_wb_merge_all ${SUITE} ${ARCH} ${WB_SUITE} secondary + done echo "`date`: ${ARCHIVE}/${SUITE} done" } diff --git a/triggers/filter-nonfree b/triggers/filter-nonfree new file mode 100755 index 0000000..5402f50 --- /dev/null +++ b/triggers/filter-nonfree @@ -0,0 +1,49 @@ +#! /usr/bin/python + + +import sys, os.path +#from debian_bundle import deb822 +import apt_pkg + + +def check_source(ok, f, fw): + ret=[] + pkg = apt_pkg.ParseTagFile(f) + while pkg.Step(): + if pkg.Section['Section'].startswith('non-free'): + if pkg.Section.get('Autobuild') != 'yes': continue + if pkg.Section['Package'] not in ok: continue + ret += [pkg.Section['Package']] + fw.write(str(pkg.Section)) + fw.write("\n") + return ret + f.close() + fw.close() + +def check_binary(ok, f, fw): + pkg = apt_pkg.ParseTagFile(f) + while pkg.Step(): + if pkg.Section.get('Source', pkg.Section.get('Package')).split(" ")[0] not in ok: continue + fw.write(str(pkg.Section)) + fw.write("\n") + f.close() + fw.close() + +def outfile(name, file): + #return open(dir + "/" + os.path.basename(file), "w") + #return open(file + name, "w") + if file.endswith(name): return open(file[:-len(name)], "w") + +def main(): + if len(sys.argv) <= 4: + print "Error - too few args" + return + + name = sys.argv[1] + oklist = [ x[:-1].split(' ')[0] for x in open(sys.argv[2]) ] + okpkgs = check_source(oklist, open(sys.argv[3]), outfile(name, sys.argv[3])) + for file in sys.argv[4:]: + check_binary(okpkgs, open(file), outfile(name, file)) + +if __name__ == '__main__': + main() diff --git a/triggers/trigger.debian b/triggers/trigger.debian index 278a489..bccd883 100755 --- a/triggers/trigger.debian +++ b/triggers/trigger.debian @@ -43,7 +43,7 @@ main_sync() { for SUITE in $SUITES_FAST_AUTOBUILD do - SOURCES="${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib}/source/Sources.gz" + SOURCES="${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib,non-free}/source/Sources.gz" SOURCES="${SOURCES} ${ARCHIVE_BASE}/buildd-${SUITE}/Sources.gz" PACKAGES="${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib}/binary-%ARCH%/Packages.gz" PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib}/debian-installer/binary-%ARCH%/Packages.gz" diff --git a/triggers/trigger.debian-buildd b/triggers/trigger.debian-buildd index e7f9980..2f7e0e2 100755 --- a/triggers/trigger.debian-buildd +++ b/triggers/trigger.debian-buildd @@ -27,9 +27,9 @@ main() { # Handle unstable as a normal suite. suite=unstable - SOURCES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib}/source/Sources.gz" + SOURCES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib,non-free}/source/Sources.gz" SOURCES="${SOURCES} ${ARCHIVE_BASE}/buildd-${suite}/Sources.gz" - PACKAGES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib}/binary-%ARCH%/Packages.gz" + PACKAGES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib,non-free}/binary-%ARCH%/Packages.gz" PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/archive/${suite}/main/debian-installer/binary-%ARCH%/Packages.gz" PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/buildd-${suite}/Packages.gz" trigger_wb_update "$suite" "`get_architectures $suite`" "$SOURCES" "$PACKAGES" @@ -37,12 +37,12 @@ main() { # Handle experimental as an overlay suite. suite=experimental base_suite=unstable - SOURCES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib}/source/Sources.gz" + SOURCES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib,non-free}/source/Sources.gz" SOURCES="${SOURCES} ${ARCHIVE_BASE}/buildd-${suite}/Sources.gz" - PACKAGES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib}/binary-%ARCH%/Packages.gz" + PACKAGES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib,non-free}/binary-%ARCH%/Packages.gz" PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/buildd-${suite}/Packages.gz" - PACKAGES_BASE="${ARCHIVE_MAIN}/archive/${base_suite}/{main,contrib}/binary-%ARCH%/Packages.gz" + PACKAGES_BASE="${ARCHIVE_MAIN}/archive/${base_suite}/{main,contrib,non-free}/binary-%ARCH%/Packages.gz" PACKAGES_BASE="${PACKAGES_BASE} ${ARCHIVE_MAIN}/archive/${base_suite}/main/debian-installer/binary-%ARCH%/Packages.gz" #PACKAGES_BASE="${PACKAGES_BASE} ${ARCHIVE_MAIN}/buildd-${BASE_SUITE}/Packages.gz" trigger_wb_update_with_secondary "$suite" "`get_architectures $suite`" "$PACKAGES_BASE" "$SOURCES" "$PACKAGES" diff --git a/triggers/trigger_wb_update_arch_loop_with_secondary b/triggers/trigger_wb_update_arch_loop_with_secondary index c0d60ec..a3960ea 100755 --- a/triggers/trigger_wb_update_arch_loop_with_secondary +++ b/triggers/trigger_wb_update_arch_loop_with_secondary @@ -10,10 +10,10 @@ PACKAGES_BASE="$3" SOURCES="$4" PACKAGES="$5" WB_SUITE="$6" -WHITELIST_FILE="$7" +EXT="" new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH} new-keep-latest ${ARCH} Packages.${SUITE}.${ARCH} $(eval echo $(echo ${PACKAGES_BASE} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}-all -do_quinn ${SUITE} ${ARCH} "${WHITELIST_FILE}" +do_quinn ${SUITE} ${ARCH} "${EXT}" do_wb_merge_all ${SUITE} ${ARCH} ${WB_SUITE} secondary diff --git a/triggers/trigger_wb_update_for_overlay_arch_loop b/triggers/trigger_wb_update_for_overlay_arch_loop index 8981db1..2d054c8 100755 --- a/triggers/trigger_wb_update_for_overlay_arch_loop +++ b/triggers/trigger_wb_update_for_overlay_arch_loop @@ -11,10 +11,10 @@ PACKAGES_BASE="$4" SOURCES="$5" PACKAGES="$6" WB_SUITE="$7" -WHITELIST_FILE="$8" +EXT="" new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH} new-keep-latest ${ARCH} Packages.${SUITE}.${ARCH} $(eval echo $(echo ${PACKAGES_BASE} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}-all -do_quinn ${SUITE} ${ARCH} "${WHITELIST_FILE}" +do_quinn ${SUITE} ${ARCH} "${EXT}" do_wb_merge_all ${SUITE} ${ARCH} ${WB_SUITE} overlay -- 2.39.2