add non-free files to quinn-diff (and to wanna-build)

(wanna-build hands them only out if asked to do so)
also inlines the loop-variants to only check the sources.list once

diff --git a/triggers/common b/triggers/common
index 2b0062f5673010be112893ac18b0d6128a0b832e..18408e6ccc56359ead1ba63114b2677c721c1d87 100644 (file)
--- a/triggers/common
+++ b/triggers/common
@@ -39,13 +39,12 @@ get_architectures() {
 do_quinn() {
     SUITE="$1"
     ARCH="$2"
-    WHITELIST_FILE="$3"
     quinn-diff \
         -A $ARCH \
         -a "$PAS_BASE/$SUITE/Packages-arch-specific" \
         -s Sources.$SUITE \
         -p Packages.$SUITE.$ARCH \
-        | filter-q-d "${WHITELIST_FILE}" > quinn-$SUITE.$ARCH 2>/dev/null
+        > quinn-$SUITE.$ARCH 2>/dev/null
 }
 
 # Call wanna-build's importing function.  WB_SUITE might differ from SUITE
@@ -106,11 +105,15 @@ trigger_wb_update() {
 
     echo "`date`: Processing ${ARCHIVE}/${SUITE} ..."
 
-    new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE}
-    parallel -l 5 -i trigger_wb_update_arch_loop "${SUITE}" "{}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" -- ${ARCHS} || true
-    #for ARCH in ${ARCHS}; do
-    #trigger_wb_update_arch_loop "${SUITE}" "${ARCH}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}"
-    #done
+    new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE}.full
+    for ARCH in ${ARCHS}; do
+        new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}.full
+    done
+    filter-nonfree .full /org/wanna-build/etc/non-free-include-list Sources.${SUITE}.full $(for ARCH in ${ARCHS}; do echo Packages.${SUITE}.${ARCH}.full; done)
+    for ARCH in ${ARCHS}; do
+        do_quinn ${SUITE} ${ARCH}
+        do_wb_merge_all ${SUITE} ${ARCH} ${WB_SUITE}
+    done
 
     echo "`date`: ${ARCHIVE}/${SUITE} done"
 }
@@ -132,13 +135,18 @@ trigger_wb_update_for_overlay() {
 
     echo "`date`: Processing ${ARCHIVE}/${SUITE} ..."
 
-    new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE}
+    new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE}.full
     new-keep-latest source $(eval echo Sources.${SUITE} ${SOURCES_BASE}) > Sources.${SUITE}-all
 
-    parallel -l 5 -i trigger_wb_update_for_overlay_arch_loop "${SUITE}" "{}" "${SOURCES_BASE}" "${PACKAGES_BASE}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" "${WHITELIST_FILE}" -- ${ARCHS} || true
-    #for ARCH in ${ARCHS}; do
-    #trigger_wb_update_for_overlay_arch_loop "${SUITE}" "${ARCH}" "${SOURCES_BASE}" "${PACKAGES_BASE}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" "${WHITELIST_FILE}"
-    #done
+    for ARCH in ${ARCHS}; do
+        new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}.full
+        new-keep-latest ${ARCH} Packages.${SUITE}.${ARCH} $(eval echo $(echo ${PACKAGES_BASE} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}-all.full
+    done
+    filter-nonfree .full /org/wanna-build/etc/non-free-include-list Sources.${SUITE}.full $(for ARCH in ${ARCHS}; do echo Packages.${SUITE}.${ARCH}.full; done)
+    for ARCH in ${ARCHS}; do
+        do_quinn ${SUITE} ${ARCH}
+        do_wb_merge_all ${SUITE} ${ARCH} ${WB_SUITE} overlay
+    done
 
     echo "`date`: ${ARCHIVE}/${SUITE} done"
 }
@@ -164,12 +172,17 @@ trigger_wb_update_with_secondary() {
 
     echo "`date`: Processing ${ARCHIVE}/${SUITE} ..."
 
-    new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE}
-
-    parallel -l 5 -i trigger_wb_update_arch_loop_with_secondary "${SUITE}" "{}" "${PACKAGES_BASE}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" "${WHITELIST_FILE}" -- ${ARCHS} || true
-    #for ARCH in ${ARCHS}; do
-    #trigger_wb_update_arch_loop_with_secondary "${SUITE}" "${ARCH}" "${PACKAGES_BASE}" "${SOURCES}" "${PACKAGES}" "${WB_SUITE}" "${WHITELIST_FILE}"
-    #done
+    new-keep-latest source $(eval echo ${SOURCES}) > Sources.${SUITE}.full
+
+    for ARCH in ${ARCHS}; do
+        new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}.full
+        new-keep-latest ${ARCH} Packages.${SUITE}.${ARCH} $(eval echo $(echo ${PACKAGES_BASE} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}-all.full
+    done
+    filter-nonfree .full /org/wanna-build/etc/non-free-include-list Sources.${SUITE}.full $(for ARCH in ${ARCHS}; do echo Packages.${SUITE}.${ARCH}.full Packages.${SUITE}.${ARCH}-all.full; done)
+    for ARCH in ${ARCHS}; do
+        do_quinn ${SUITE} ${ARCH}
+        do_wb_merge_all ${SUITE} ${ARCH} ${WB_SUITE} secondary
+    done
 
     echo "`date`: ${ARCHIVE}/${SUITE} done"
 }

diff --git a/triggers/filter-nonfree b/triggers/filter-nonfree
new file mode 100755 (executable)
index 0000000..5402f50
--- /dev/null
+++ b/triggers/filter-nonfree
@@ -0,0 +1,49 @@
+#! /usr/bin/python
+
+
+import sys, os.path
+#from debian_bundle import deb822
+import apt_pkg
+
+
+def check_source(ok, f, fw):
+    ret=[]
+    pkg = apt_pkg.ParseTagFile(f)
+    while pkg.Step():
+        if pkg.Section['Section'].startswith('non-free'):
+            if pkg.Section.get('Autobuild') != 'yes': continue
+            if pkg.Section['Package'] not in ok: continue
+        ret += [pkg.Section['Package']]
+        fw.write(str(pkg.Section))
+        fw.write("\n")
+    return ret
+    f.close()
+    fw.close()
+
+def check_binary(ok, f, fw):
+    pkg = apt_pkg.ParseTagFile(f)
+    while pkg.Step():
+        if pkg.Section.get('Source', pkg.Section.get('Package')).split(" ")[0] not in ok: continue
+        fw.write(str(pkg.Section))
+        fw.write("\n")
+    f.close()
+    fw.close()
+
+def outfile(name, file):
+    #return open(dir + "/" + os.path.basename(file), "w")
+    #return open(file + name, "w")
+    if file.endswith(name): return open(file[:-len(name)], "w")
+
+def main():
+    if len(sys.argv) <= 4:
+        print "Error - too few args"
+        return
+
+    name = sys.argv[1]
+    oklist = [ x[:-1].split(' ')[0] for x in open(sys.argv[2]) ]
+    okpkgs = check_source(oklist, open(sys.argv[3]), outfile(name, sys.argv[3]))
+    for file in sys.argv[4:]:
+        check_binary(okpkgs, open(file), outfile(name, file))
+
+if __name__ == '__main__':
+        main()

diff --git a/triggers/trigger.debian b/triggers/trigger.debian
index 278a489db6de44ff32c9bd379e3959edc261bb94..bccd883e2b81913b7003c6a554a84afee28f2d0b 100755 (executable)
--- a/triggers/trigger.debian
+++ b/triggers/trigger.debian
@@ -43,7 +43,7 @@ main_sync() {
 
     for SUITE in $SUITES_FAST_AUTOBUILD
     do
-        SOURCES="${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib}/source/Sources.gz"
+        SOURCES="${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib,non-free}/source/Sources.gz"
         SOURCES="${SOURCES} ${ARCHIVE_BASE}/buildd-${SUITE}/Sources.gz"
         PACKAGES="${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib}/binary-%ARCH%/Packages.gz"
         PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib}/debian-installer/binary-%ARCH%/Packages.gz"

diff --git a/triggers/trigger.debian-buildd b/triggers/trigger.debian-buildd
index e7f9980ef2c25bc2a87657dcade7d3e902f16511..2f7e0e2417be97444c20a7266dc4531494e39a25 100755 (executable)
--- a/triggers/trigger.debian-buildd
+++ b/triggers/trigger.debian-buildd
@@ -27,9 +27,9 @@ main() {
 
     # Handle unstable as a normal suite.
     suite=unstable
-    SOURCES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib}/source/Sources.gz"
+    SOURCES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib,non-free}/source/Sources.gz"
     SOURCES="${SOURCES} ${ARCHIVE_BASE}/buildd-${suite}/Sources.gz"
-    PACKAGES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib}/binary-%ARCH%/Packages.gz"
+    PACKAGES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib,non-free}/binary-%ARCH%/Packages.gz"
     PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/archive/${suite}/main/debian-installer/binary-%ARCH%/Packages.gz"
     PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/buildd-${suite}/Packages.gz"
     trigger_wb_update "$suite" "`get_architectures $suite`" "$SOURCES" "$PACKAGES"
@@ -37,12 +37,12 @@ main() {
     # Handle experimental as an overlay suite.
     suite=experimental
     base_suite=unstable
-    SOURCES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib}/source/Sources.gz"
+    SOURCES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib,non-free}/source/Sources.gz"
     SOURCES="${SOURCES} ${ARCHIVE_BASE}/buildd-${suite}/Sources.gz"
-    PACKAGES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib}/binary-%ARCH%/Packages.gz"
+    PACKAGES="${ARCHIVE_BASE}/archive/${suite}/{main,contrib,non-free}/binary-%ARCH%/Packages.gz"
     PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/buildd-${suite}/Packages.gz"
 
-    PACKAGES_BASE="${ARCHIVE_MAIN}/archive/${base_suite}/{main,contrib}/binary-%ARCH%/Packages.gz"
+    PACKAGES_BASE="${ARCHIVE_MAIN}/archive/${base_suite}/{main,contrib,non-free}/binary-%ARCH%/Packages.gz"
     PACKAGES_BASE="${PACKAGES_BASE} ${ARCHIVE_MAIN}/archive/${base_suite}/main/debian-installer/binary-%ARCH%/Packages.gz"
     #PACKAGES_BASE="${PACKAGES_BASE} ${ARCHIVE_MAIN}/buildd-${BASE_SUITE}/Packages.gz"
     trigger_wb_update_with_secondary "$suite" "`get_architectures $suite`" "$PACKAGES_BASE" "$SOURCES" "$PACKAGES"

diff --git a/triggers/trigger_wb_update_arch_loop_with_secondary b/triggers/trigger_wb_update_arch_loop_with_secondary
index c0d60eca4b6e4fc539a92afbf83902dff13f9427..a3960ea3f808375b147928aa231d82acff623796 100755 (executable)
--- a/triggers/trigger_wb_update_arch_loop_with_secondary
+++ b/triggers/trigger_wb_update_arch_loop_with_secondary
@@ -10,10 +10,10 @@ PACKAGES_BASE="$3"
 SOURCES="$4"
 PACKAGES="$5"
 WB_SUITE="$6"
-WHITELIST_FILE="$7"
+EXT=""
 
 new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}
 new-keep-latest ${ARCH} Packages.${SUITE}.${ARCH} $(eval echo $(echo ${PACKAGES_BASE} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}-all
-do_quinn ${SUITE} ${ARCH} "${WHITELIST_FILE}"
+do_quinn ${SUITE} ${ARCH} "${EXT}"
 do_wb_merge_all ${SUITE} ${ARCH} ${WB_SUITE} secondary
 

diff --git a/triggers/trigger_wb_update_for_overlay_arch_loop b/triggers/trigger_wb_update_for_overlay_arch_loop
index 8981db16643763ad1e63601285590cbb35a1c412..2d054c8d88f3298dda488485bbef6ac64967b488 100755 (executable)
--- a/triggers/trigger_wb_update_for_overlay_arch_loop
+++ b/triggers/trigger_wb_update_for_overlay_arch_loop
@@ -11,10 +11,10 @@ PACKAGES_BASE="$4"
 SOURCES="$5"
 PACKAGES="$6"
 WB_SUITE="$7"
-WHITELIST_FILE="$8"
+EXT=""
 
 new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}
 new-keep-latest ${ARCH} Packages.${SUITE}.${ARCH} $(eval echo $(echo ${PACKAGES_BASE} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}-all
-do_quinn ${SUITE} ${ARCH} "${WHITELIST_FILE}"
+do_quinn ${SUITE} ${ARCH} "${EXT}"
 do_wb_merge_all ${SUITE} ${ARCH} ${WB_SUITE} overlay