Revert "add non-free files to quinn-diff (and to wanna-build)"

This reverts commit 636fd44b6b3b8f343eb53c4ef2dd053fb91bd4d6.

This commit breaks merging and performance.

Traceback (most recent call last):
  File "/org/wanna-build/triggers/filter-nonfree", line 49, in <module>
    main()
  File "/org/wanna-build/triggers/filter-nonfree", line 44, in main
    okpkgs = check_source(oklist, open(sys.argv[3]), outfile(name, sys.argv[3]))
  File "/org/wanna-build/triggers/filter-nonfree", line 13, in check_source
    if pkg.Section['Section'].startswith('non-free'):
KeyError: 'Section'

Signed-off-by: Philipp Kern <pkern@debian.org>

diff --git a/triggers/common b/triggers/common
index 18408e6ccc56359ead1ba63114b2677c721c1d87..2b0062f5673010be112893ac18b0d6128a0b832e 100644 (file)
--- a/triggers/common
+++ b/triggers/common
@@ -39,12 +39,13 @@ get_architectures() {
 do_quinn() {
     SUITE="$1"
     ARCH="$2"
+    WHITELIST_FILE="$3"
     quinn-diff \
         -A $ARCH \
         -a "$PAS_BASE/$SUITE/Packages-arch-specific" \
         -s Sources.$SUITE \
         -p Packages.$SUITE.$ARCH \
-        > quinn-$SUITE.$ARCH 2>/dev/null
+        | filter-q-d "${WHITELIST_FILE}" > quinn-$SUITE.$ARCH 2>/dev/null
 }
 
 # Call wanna-build's importing function.  WB_SUITE might differ from SUITE
@@ -105,15 +106,11 @@ trigger_wb_update() {
 
     echo "`date`: Processing ${ARCHIVE}/${SUITE} ..."
 
-    new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE}.full
-    for ARCH in ${ARCHS}; do
-        new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}.full
-    done
-    filter-nonfree .full /org/wanna-build/etc/non-free-include-list Sources.${SUITE}.full $(for ARCH in ${ARCHS}; do echo Packages.${SUITE}.${ARCH}.full; done)
-    for ARCH in ${ARCHS}; do
-        do_quinn ${SUITE} ${ARCH}
-        do_wb_merge_all ${SUITE} ${ARCH} ${WB_SUITE}
-    done
+    new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE}
+    parallel -l 5 -i trigger_wb_update_arch_loop "${SUITE}" "{}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" -- ${ARCHS} || true
+    #for ARCH in ${ARCHS}; do
+    #trigger_wb_update_arch_loop "${SUITE}" "${ARCH}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}"
+    #done
 
     echo "`date`: ${ARCHIVE}/${SUITE} done"
 }
@@ -135,18 +132,13 @@ trigger_wb_update_for_overlay() {
 
     echo "`date`: Processing ${ARCHIVE}/${SUITE} ..."
 
-    new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE}.full
+    new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE}
     new-keep-latest source $(eval echo Sources.${SUITE} ${SOURCES_BASE}) > Sources.${SUITE}-all
 
-    for ARCH in ${ARCHS}; do
-        new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}.full
-        new-keep-latest ${ARCH} Packages.${SUITE}.${ARCH} $(eval echo $(echo ${PACKAGES_BASE} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}-all.full
-    done
-    filter-nonfree .full /org/wanna-build/etc/non-free-include-list Sources.${SUITE}.full $(for ARCH in ${ARCHS}; do echo Packages.${SUITE}.${ARCH}.full; done)
-    for ARCH in ${ARCHS}; do
-        do_quinn ${SUITE} ${ARCH}
-        do_wb_merge_all ${SUITE} ${ARCH} ${WB_SUITE} overlay
-    done
+    parallel -l 5 -i trigger_wb_update_for_overlay_arch_loop "${SUITE}" "{}" "${SOURCES_BASE}" "${PACKAGES_BASE}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" "${WHITELIST_FILE}" -- ${ARCHS} || true
+    #for ARCH in ${ARCHS}; do
+    #trigger_wb_update_for_overlay_arch_loop "${SUITE}" "${ARCH}" "${SOURCES_BASE}" "${PACKAGES_BASE}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" "${WHITELIST_FILE}"
+    #done
 
     echo "`date`: ${ARCHIVE}/${SUITE} done"
 }
@@ -172,17 +164,12 @@ trigger_wb_update_with_secondary() {
 
     echo "`date`: Processing ${ARCHIVE}/${SUITE} ..."
 
-    new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE}.full
-
-    for ARCH in ${ARCHS}; do
-        new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}.full
-        new-keep-latest ${ARCH} Packages.${SUITE}.${ARCH} $(eval echo $(echo ${PACKAGES_BASE} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}-all.full
-    done
-    filter-nonfree .full /org/wanna-build/etc/non-free-include-list Sources.${SUITE}.full $(for ARCH in ${ARCHS}; do echo Packages.${SUITE}.${ARCH}.full Packages.${SUITE}.${ARCH}-all.full; done)
-    for ARCH in ${ARCHS}; do
-        do_quinn ${SUITE} ${ARCH}
-        do_wb_merge_all ${SUITE} ${ARCH} ${WB_SUITE} secondary
-    done
+    new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE}
+
+    parallel -l 5 -i trigger_wb_update_arch_loop_with_secondary "${SUITE}" "{}" "${PACKAGES_BASE}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" "${WHITELIST_FILE}" -- ${ARCHS} || true
+    #for ARCH in ${ARCHS}; do
+    #trigger_wb_update_arch_loop_with_secondary "${SUITE}" "${ARCH}" "${PACKAGES_BASE}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" "${WHITELIST_FILE}"
+    #done
 
     echo "`date`: ${ARCHIVE}/${SUITE} done"
 }

diff --git a/triggers/filter-nonfree b/triggers/filter-nonfree
deleted file mode 100755 (executable)
index 5402f50..0000000
--- a/triggers/filter-nonfree
+++ /dev/null
@@ -1,49 +0,0 @@
-#! /usr/bin/python
-
-
-import sys, os.path
-#from debian_bundle import deb822
-import apt_pkg
-
-
-def check_source(ok, f, fw):
-    ret=[]
-    pkg = apt_pkg.ParseTagFile(f)
-    while pkg.Step():
-        if pkg.Section['Section'].startswith('non-free'):
-            if pkg.Section.get('Autobuild') != 'yes': continue
-            if pkg.Section['Package'] not in ok: continue
-        ret += [pkg.Section['Package']]
-        fw.write(str(pkg.Section))
-        fw.write("\n")
-    return ret
-    f.close()
-    fw.close()
-
-def check_binary(ok, f, fw):
-    pkg = apt_pkg.ParseTagFile(f)
-    while pkg.Step():
-        if pkg.Section.get('Source', pkg.Section.get('Package')).split(" ")[0] not in ok: continue
-        fw.write(str(pkg.Section))
-        fw.write("\n")
-    f.close()
-    fw.close()
-
-def outfile(name, file):
-    #return open(dir + "/" + os.path.basename(file), "w")
-    #return open(file + name, "w")
-    if file.endswith(name): return open(file[:-len(name)], "w")
-
-def main():
-    if len(sys.argv) <= 4:
-        print "Error - too few args"
-        return
-
-    name = sys.argv[1]
-    oklist = [ x[:-1].split(' ')[0] for x in open(sys.argv[2]) ]
-    okpkgs = check_source(oklist, open(sys.argv[3]), outfile(name, sys.argv[3]))
-    for file in sys.argv[4:]:
-        check_binary(okpkgs, open(file), outfile(name, file))
-
-if __name__ == '__main__':
-        main()

diff --git a/triggers/trigger.debian b/triggers/trigger.debian
index bccd883e2b81913b7003c6a554a84afee28f2d0b..278a489db6de44ff32c9bd379e3959edc261bb94 100755 (executable)
--- a/triggers/trigger.debian
+++ b/triggers/trigger.debian
@@ -43,7 +43,7 @@ main_sync() {
 
     for SUITE in $SUITES_FAST_AUTOBUILD
     do
-        SOURCES="${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib,non-free}/source/Sources.gz"
+        SOURCES="${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib}/source/Sources.gz"
         SOURCES="${SOURCES} ${ARCHIVE_BASE}/buildd-${SUITE}/Sources.gz"
         PACKAGES="${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib}/binary-%ARCH%/Packages.gz"
         PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib}/debian-installer/binary-%ARCH%/Packages.gz"

diff --git a/triggers/trigger.debian-buildd b/triggers/trigger.debian-buildd
index 2f7e0e2417be97444c20a7266dc4531494e39a25..e7f9980ef2c25bc2a87657dcade7d3e902f16511 100755 (executable)
--- a/triggers/trigger.debian-buildd
+++ b/triggers/trigger.debian-buildd
@@ -27,9 +27,9 @@ main() {
 
     # Handle unstable as a normal suite.
     suite=unstable
-    SOURCES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib,non-free}/source/Sources.gz"
+    SOURCES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib}/source/Sources.gz"
     SOURCES="${SOURCES} ${ARCHIVE_BASE}/buildd-${suite}/Sources.gz"
-    PACKAGES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib,non-free}/binary-%ARCH%/Packages.gz"
+    PACKAGES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib}/binary-%ARCH%/Packages.gz"
     PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/archive/${suite}/main/debian-installer/binary-%ARCH%/Packages.gz"
     PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/buildd-${suite}/Packages.gz"
     trigger_wb_update "$suite" "`get_architectures $suite`" "$SOURCES" "$PACKAGES"
@@ -37,12 +37,12 @@ main() {
     # Handle experimental as an overlay suite.
     suite=experimental
     base_suite=unstable
-    SOURCES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib,non-free}/source/Sources.gz"
+    SOURCES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib}/source/Sources.gz"
     SOURCES="${SOURCES} ${ARCHIVE_BASE}/buildd-${suite}/Sources.gz"
-    PACKAGES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib,non-free}/binary-%ARCH%/Packages.gz"
+    PACKAGES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib}/binary-%ARCH%/Packages.gz"
     PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/buildd-${suite}/Packages.gz"
 
-    PACKAGES_BASE="${ARCHIVE_MAIN}/archive/${base_suite}/{main,contrib,non-free}/binary-%ARCH%/Packages.gz"
+    PACKAGES_BASE="${ARCHIVE_MAIN}/archive/${base_suite}/{main,contrib}/binary-%ARCH%/Packages.gz"
     PACKAGES_BASE="${PACKAGES_BASE} ${ARCHIVE_MAIN}/archive/${base_suite}/main/debian-installer/binary-%ARCH%/Packages.gz"
     #PACKAGES_BASE="${PACKAGES_BASE} ${ARCHIVE_MAIN}/buildd-${BASE_SUITE}/Packages.gz"
     trigger_wb_update_with_secondary "$suite" "`get_architectures $suite`" "$PACKAGES_BASE" "$SOURCES" "$PACKAGES"

diff --git a/triggers/trigger_wb_update_arch_loop_with_secondary b/triggers/trigger_wb_update_arch_loop_with_secondary
index a3960ea3f808375b147928aa231d82acff623796..c0d60eca4b6e4fc539a92afbf83902dff13f9427 100755 (executable)
--- a/triggers/trigger_wb_update_arch_loop_with_secondary
+++ b/triggers/trigger_wb_update_arch_loop_with_secondary
@@ -10,10 +10,10 @@ PACKAGES_BASE="$3"
 SOURCES="$4"
 PACKAGES="$5"
 WB_SUITE="$6"
-EXT=""
+WHITELIST_FILE="$7"
 
 new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}
 new-keep-latest ${ARCH} Packages.${SUITE}.${ARCH} $(eval echo $(echo ${PACKAGES_BASE} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}-all
-do_quinn ${SUITE} ${ARCH} "${EXT}"
+do_quinn ${SUITE} ${ARCH} "${WHITELIST_FILE}"
 do_wb_merge_all ${SUITE} ${ARCH} ${WB_SUITE} secondary
 

diff --git a/triggers/trigger_wb_update_for_overlay_arch_loop b/triggers/trigger_wb_update_for_overlay_arch_loop
index 2d054c8d88f3298dda488485bbef6ac64967b488..8981db16643763ad1e63601285590cbb35a1c412 100755 (executable)
--- a/triggers/trigger_wb_update_for_overlay_arch_loop
+++ b/triggers/trigger_wb_update_for_overlay_arch_loop
@@ -11,10 +11,10 @@ PACKAGES_BASE="$4"
 SOURCES="$5"
 PACKAGES="$6"
 WB_SUITE="$7"
-EXT=""
+WHITELIST_FILE="$8"
 
 new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}
 new-keep-latest ${ARCH} Packages.${SUITE}.${ARCH} $(eval echo $(echo ${PACKAGES_BASE} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}-all
-do_quinn ${SUITE} ${ARCH} "${EXT}"
+do_quinn ${SUITE} ${ARCH} "${WHITELIST_FILE}"
 do_wb_merge_all ${SUITE} ${ARCH} ${WB_SUITE} overlay