From 6ddbe440e53aaf379fa02b1308c2aec0bf9ecb87 Mon Sep 17 00:00:00 2001 From: Philipp Kern Date: Thu, 11 Mar 2010 19:08:16 +0000 Subject: [PATCH] Revert "add non-free files to quinn-diff (and to wanna-build)" This reverts commit 636fd44b6b3b8f343eb53c4ef2dd053fb91bd4d6. This commit breaks merging and performance. Traceback (most recent call last): File "/org/wanna-build/triggers/filter-nonfree", line 49, in main() File "/org/wanna-build/triggers/filter-nonfree", line 44, in main okpkgs = check_source(oklist, open(sys.argv[3]), outfile(name, sys.argv[3])) File "/org/wanna-build/triggers/filter-nonfree", line 13, in check_source if pkg.Section['Section'].startswith('non-free'): KeyError: 'Section' Signed-off-by: Philipp Kern --- triggers/common | 49 +++++++------------ triggers/filter-nonfree | 49 ------------------- triggers/trigger.debian | 2 +- triggers/trigger.debian-buildd | 10 ++-- ...trigger_wb_update_arch_loop_with_secondary | 4 +- .../trigger_wb_update_for_overlay_arch_loop | 4 +- 6 files changed, 28 insertions(+), 90 deletions(-) delete mode 100755 triggers/filter-nonfree diff --git a/triggers/common b/triggers/common index 18408e6..2b0062f 100644 --- a/triggers/common +++ b/triggers/common @@ -39,12 +39,13 @@ get_architectures() { do_quinn() { SUITE="$1" ARCH="$2" + WHITELIST_FILE="$3" quinn-diff \ -A $ARCH \ -a "$PAS_BASE/$SUITE/Packages-arch-specific" \ -s Sources.$SUITE \ -p Packages.$SUITE.$ARCH \ - > quinn-$SUITE.$ARCH 2>/dev/null + | filter-q-d "${WHITELIST_FILE}" > quinn-$SUITE.$ARCH 2>/dev/null } # Call wanna-build's importing function. WB_SUITE might differ from SUITE @@ -105,15 +106,11 @@ trigger_wb_update() { echo "`date`: Processing ${ARCHIVE}/${SUITE} ..." - new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE}.full - for ARCH in ${ARCHS}; do - new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}.full - done - filter-nonfree .full /org/wanna-build/etc/non-free-include-list Sources.${SUITE}.full $(for ARCH in ${ARCHS}; do echo Packages.${SUITE}.${ARCH}.full; done) - for ARCH in ${ARCHS}; do - do_quinn ${SUITE} ${ARCH} - do_wb_merge_all ${SUITE} ${ARCH} ${WB_SUITE} - done + new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE} + parallel -l 5 -i trigger_wb_update_arch_loop "${SUITE}" "{}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" -- ${ARCHS} || true + #for ARCH in ${ARCHS}; do + #trigger_wb_update_arch_loop "${SUITE}" "${ARCH}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" + #done echo "`date`: ${ARCHIVE}/${SUITE} done" } @@ -135,18 +132,13 @@ trigger_wb_update_for_overlay() { echo "`date`: Processing ${ARCHIVE}/${SUITE} ..." - new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE}.full + new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE} new-keep-latest source $(eval echo Sources.${SUITE} ${SOURCES_BASE}) > Sources.${SUITE}-all - for ARCH in ${ARCHS}; do - new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}.full - new-keep-latest ${ARCH} Packages.${SUITE}.${ARCH} $(eval echo $(echo ${PACKAGES_BASE} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}-all.full - done - filter-nonfree .full /org/wanna-build/etc/non-free-include-list Sources.${SUITE}.full $(for ARCH in ${ARCHS}; do echo Packages.${SUITE}.${ARCH}.full; done) - for ARCH in ${ARCHS}; do - do_quinn ${SUITE} ${ARCH} - do_wb_merge_all ${SUITE} ${ARCH} ${WB_SUITE} overlay - done + parallel -l 5 -i trigger_wb_update_for_overlay_arch_loop "${SUITE}" "{}" "${SOURCES_BASE}" "${PACKAGES_BASE}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" "${WHITELIST_FILE}" -- ${ARCHS} || true + #for ARCH in ${ARCHS}; do + #trigger_wb_update_for_overlay_arch_loop "${SUITE}" "${ARCH}" "${SOURCES_BASE}" "${PACKAGES_BASE}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" "${WHITELIST_FILE}" + #done echo "`date`: ${ARCHIVE}/${SUITE} done" } @@ -172,17 +164,12 @@ trigger_wb_update_with_secondary() { echo "`date`: Processing ${ARCHIVE}/${SUITE} ..." - new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE}.full - - for ARCH in ${ARCHS}; do - new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}.full - new-keep-latest ${ARCH} Packages.${SUITE}.${ARCH} $(eval echo $(echo ${PACKAGES_BASE} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}-all.full - done - filter-nonfree .full /org/wanna-build/etc/non-free-include-list Sources.${SUITE}.full $(for ARCH in ${ARCHS}; do echo Packages.${SUITE}.${ARCH}.full Packages.${SUITE}.${ARCH}-all.full; done) - for ARCH in ${ARCHS}; do - do_quinn ${SUITE} ${ARCH} - do_wb_merge_all ${SUITE} ${ARCH} ${WB_SUITE} secondary - done + new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE} + + parallel -l 5 -i trigger_wb_update_arch_loop_with_secondary "${SUITE}" "{}" "${PACKAGES_BASE}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" "${WHITELIST_FILE}" -- ${ARCHS} || true + #for ARCH in ${ARCHS}; do + #trigger_wb_update_arch_loop_with_secondary "${SUITE}" "${ARCH}" "${PACKAGES_BASE}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" "${WHITELIST_FILE}" + #done echo "`date`: ${ARCHIVE}/${SUITE} done" } diff --git a/triggers/filter-nonfree b/triggers/filter-nonfree deleted file mode 100755 index 5402f50..0000000 --- a/triggers/filter-nonfree +++ /dev/null @@ -1,49 +0,0 @@ -#! /usr/bin/python - - -import sys, os.path -#from debian_bundle import deb822 -import apt_pkg - - -def check_source(ok, f, fw): - ret=[] - pkg = apt_pkg.ParseTagFile(f) - while pkg.Step(): - if pkg.Section['Section'].startswith('non-free'): - if pkg.Section.get('Autobuild') != 'yes': continue - if pkg.Section['Package'] not in ok: continue - ret += [pkg.Section['Package']] - fw.write(str(pkg.Section)) - fw.write("\n") - return ret - f.close() - fw.close() - -def check_binary(ok, f, fw): - pkg = apt_pkg.ParseTagFile(f) - while pkg.Step(): - if pkg.Section.get('Source', pkg.Section.get('Package')).split(" ")[0] not in ok: continue - fw.write(str(pkg.Section)) - fw.write("\n") - f.close() - fw.close() - -def outfile(name, file): - #return open(dir + "/" + os.path.basename(file), "w") - #return open(file + name, "w") - if file.endswith(name): return open(file[:-len(name)], "w") - -def main(): - if len(sys.argv) <= 4: - print "Error - too few args" - return - - name = sys.argv[1] - oklist = [ x[:-1].split(' ')[0] for x in open(sys.argv[2]) ] - okpkgs = check_source(oklist, open(sys.argv[3]), outfile(name, sys.argv[3])) - for file in sys.argv[4:]: - check_binary(okpkgs, open(file), outfile(name, file)) - -if __name__ == '__main__': - main() diff --git a/triggers/trigger.debian b/triggers/trigger.debian index bccd883..278a489 100755 --- a/triggers/trigger.debian +++ b/triggers/trigger.debian @@ -43,7 +43,7 @@ main_sync() { for SUITE in $SUITES_FAST_AUTOBUILD do - SOURCES="${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib,non-free}/source/Sources.gz" + SOURCES="${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib}/source/Sources.gz" SOURCES="${SOURCES} ${ARCHIVE_BASE}/buildd-${SUITE}/Sources.gz" PACKAGES="${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib}/binary-%ARCH%/Packages.gz" PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib}/debian-installer/binary-%ARCH%/Packages.gz" diff --git a/triggers/trigger.debian-buildd b/triggers/trigger.debian-buildd index 2f7e0e2..e7f9980 100755 --- a/triggers/trigger.debian-buildd +++ b/triggers/trigger.debian-buildd @@ -27,9 +27,9 @@ main() { # Handle unstable as a normal suite. suite=unstable - SOURCES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib,non-free}/source/Sources.gz" + SOURCES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib}/source/Sources.gz" SOURCES="${SOURCES} ${ARCHIVE_BASE}/buildd-${suite}/Sources.gz" - PACKAGES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib,non-free}/binary-%ARCH%/Packages.gz" + PACKAGES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib}/binary-%ARCH%/Packages.gz" PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/archive/${suite}/main/debian-installer/binary-%ARCH%/Packages.gz" PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/buildd-${suite}/Packages.gz" trigger_wb_update "$suite" "`get_architectures $suite`" "$SOURCES" "$PACKAGES" @@ -37,12 +37,12 @@ main() { # Handle experimental as an overlay suite. suite=experimental base_suite=unstable - SOURCES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib,non-free}/source/Sources.gz" + SOURCES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib}/source/Sources.gz" SOURCES="${SOURCES} ${ARCHIVE_BASE}/buildd-${suite}/Sources.gz" - PACKAGES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib,non-free}/binary-%ARCH%/Packages.gz" + PACKAGES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib}/binary-%ARCH%/Packages.gz" PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/buildd-${suite}/Packages.gz" - PACKAGES_BASE="${ARCHIVE_MAIN}/archive/${base_suite}/{main,contrib,non-free}/binary-%ARCH%/Packages.gz" + PACKAGES_BASE="${ARCHIVE_MAIN}/archive/${base_suite}/{main,contrib}/binary-%ARCH%/Packages.gz" PACKAGES_BASE="${PACKAGES_BASE} ${ARCHIVE_MAIN}/archive/${base_suite}/main/debian-installer/binary-%ARCH%/Packages.gz" #PACKAGES_BASE="${PACKAGES_BASE} ${ARCHIVE_MAIN}/buildd-${BASE_SUITE}/Packages.gz" trigger_wb_update_with_secondary "$suite" "`get_architectures $suite`" "$PACKAGES_BASE" "$SOURCES" "$PACKAGES" diff --git a/triggers/trigger_wb_update_arch_loop_with_secondary b/triggers/trigger_wb_update_arch_loop_with_secondary index a3960ea..c0d60ec 100755 --- a/triggers/trigger_wb_update_arch_loop_with_secondary +++ b/triggers/trigger_wb_update_arch_loop_with_secondary @@ -10,10 +10,10 @@ PACKAGES_BASE="$3" SOURCES="$4" PACKAGES="$5" WB_SUITE="$6" -EXT="" +WHITELIST_FILE="$7" new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH} new-keep-latest ${ARCH} Packages.${SUITE}.${ARCH} $(eval echo $(echo ${PACKAGES_BASE} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}-all -do_quinn ${SUITE} ${ARCH} "${EXT}" +do_quinn ${SUITE} ${ARCH} "${WHITELIST_FILE}" do_wb_merge_all ${SUITE} ${ARCH} ${WB_SUITE} secondary diff --git a/triggers/trigger_wb_update_for_overlay_arch_loop b/triggers/trigger_wb_update_for_overlay_arch_loop index 2d054c8..8981db1 100755 --- a/triggers/trigger_wb_update_for_overlay_arch_loop +++ b/triggers/trigger_wb_update_for_overlay_arch_loop @@ -11,10 +11,10 @@ PACKAGES_BASE="$4" SOURCES="$5" PACKAGES="$6" WB_SUITE="$7" -EXT="" +WHITELIST_FILE="$8" new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH} new-keep-latest ${ARCH} Packages.${SUITE}.${ARCH} $(eval echo $(echo ${PACKAGES_BASE} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}-all -do_quinn ${SUITE} ${ARCH} "${EXT}" +do_quinn ${SUITE} ${ARCH} "${WHITELIST_FILE}" do_wb_merge_all ${SUITE} ${ARCH} ${WB_SUITE} overlay -- 2.39.2