From: Andreas Barth <aba@not.so.argh.org>
Date: Mon, 7 Jun 2010 20:30:08 +0000 (+0000)
Subject: wanna-build/merge-v3: fix handling of multiple Packages/Sources files
X-Git-Url: https://git.donarmstrong.com/?p=wannabuild.git;a=commitdiff_plain;h=30cbae16042fc0bc6bf34fcc7d5bc34079b08ceb

wanna-build/merge-v3: fix handling of multiple Packages/Sources files
wanna-build/merge-v3, WB::QD: ignore binaries of other arches
wanna-build/merge-v3: packages in Not-For-Us Failed-Removed don't go to Needs-build
wanna-build/merge-v3: fix skip packages as obsolete
triggers/common: fix for SUITE != WB_SUITE
triggers/filter-nonfree: change parameters
triggers/trigger.debian: build non-free packages / sources for unstable / experimental
---

diff --git a/bin/wanna-build b/bin/wanna-build
index ffb3348..3775de8 100755
--- a/bin/wanna-build
+++ b/bin/wanna-build
@@ -520,7 +520,7 @@ sub process {
                         }
                         parse_all_v3($$srcs, {'arch' => $arch, 'suite' => $distribution, 'time' => $curr_date});
                         @bpkgs = @ipkgs unless @bpkgs;
-                        call_edos_depcheck( {'arch' => $arch, 'pkgs' => @bpkgs, 'srcs' => $$srcs, 'depwait' => 1 });
+                        call_edos_depcheck( {'arch' => $arch, 'pkgs' => \@bpkgs, 'srcs' => $$srcs, 'depwait' => 1 });
                         last SWITCH;
 		};
 		/^import/ && do {
@@ -2433,11 +2433,11 @@ sub wb_edos_builddebcheck {
     my $sourceprefix="source---";
     my $architecture=$args->{'arch'};
     my $edosoptions = "-failures -explain -quiet";
-    my @packagefiles = $args->{'pkgs'};
+    my $packagefiles = $args->{'pkgs'};
     my $sourcesfile = $args->{'src'};
 
     my $packagearch="";
-    foreach my $packagefile (@packagefiles) {
+    foreach my $packagefile (@$packagefiles) {
         open(P,$packagefile);
         while (<P>) {
             next unless /^Architecture/;
@@ -2466,9 +2466,9 @@ sub wb_edos_builddebcheck {
         }   
     }
 
-    print "calling: edos-debcheck $edosoptions < $sourcesfile ".join('', map {" '-base FILE' ".$_ } @packagefiles)."\n";
+    print "calling: edos-debcheck $edosoptions < $sourcesfile ".join('', map {" '-base FILE' ".$_ } @$packagefiles)."\n";
     open(RESULT, '-|',
-        "edos-debcheck $edosoptions < $sourcesfile ".join('', map {" '-base FILE' ".$_ } @packagefiles));
+        "edos-debcheck $edosoptions < $sourcesfile ".join('', map {" '-base FILE' ".$_ } @$packagefiles));
 
     my $explanation="";
     my $result={};
@@ -2940,7 +2940,7 @@ sub parse_all_v3() {
 
             # does at least one binary exist in the database and is more recent - if so, we're probably just outdated, ignore the source package
             for my $bin (@{$pkgs->{'binary'}}) {
-                if ($binary->{$bin} and vercmp($pkgs->{'version'}, $binary->{$bin}) < 0) {
+                if ($binary->{$bin} and vercmp($pkgs->{'version'}, $binary->{$bin}->{'version'}) < 0) {
                     print "$logstr skipped because binaries (assumed to be) overwritten\n" if $verbose || $simulate;
                     next SRCS;
                 }
@@ -2999,6 +2999,7 @@ sub parse_all_v3() {
             next SRCS;
         }
         next if $pkgs->{'version'} eq $pkg->{'version'};
+        next if isin( $pkg->{'state'}, qw(Not-For-Us Failed-Removed));
 
         if (defined( $pkg->{'state'} ) && isin( $pkg->{'state'}, qw(Building Built Build-Attempted))) {
             send_mail( $pkg->{'builder'},
diff --git a/lib/WB/QD.pm b/lib/WB/QD.pm
index 059d6b6..f4b1f5f 100644
--- a/lib/WB/QD.pm
+++ b/lib/WB/QD.pm
@@ -67,7 +67,8 @@ sub readsourcebins {
             /^Source:\s*(\S+)\s+\((\S+)\)$/mi and $p->{'source'} = $1 and $p->{'version'} = $2;
             $p->{'version'} =~ /(\S+)\+b([0-9]+)/ and $p->{'version'} = $1 and $p->{'binnmu'} = $2;
 
-            $binary->{$p->{'binary'}} = { 'version' => $p->{'version'}, 'arch' => $p->{'arch'}} unless $binary->{$p->{'binary'}} and vercmp($binary->{$p->{'binary'}->{'version'}}, $p->{'version'}) < 0;
+            next unless $p->{'arch'} eq 'all' || $p->{'arch'} eq ${arch};
+            $binary->{$p->{'binary'}} = { 'version' => $p->{'version'}, 'arch' => $p->{'arch'}} unless $binary->{$p->{'binary'}} and vercmp($binary->{$p->{'binary'}}->{'version'}, $p->{'version'}) < 0;
 
             #next if $pas->{$p->{'binary'}} && pasignore($pas->{$p->{'binary'}}, $arch);
             next if $p->{'arch'} eq 'all';
diff --git a/triggers/common b/triggers/common
index 9a1553a..e8eec79 100644
--- a/triggers/common
+++ b/triggers/common
@@ -66,7 +66,7 @@ trigger_wb_update() {
     for ARCH in ${ARCHS}; do
         new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}
     done
-    parallel -l 5 -i wanna-build -v --Pas ${PAS_FILE} --merge-v3 -A "{}" --dist=$WB_SUITE Packages.%SUITE%.%ARCH% . Sources.%SUITE% -- ${ARCHS} || true
+    parallel -l 5 -i wanna-build -v --Pas ${PAS_FILE} --merge-v3 -A "{}" --dist=$WB_SUITE Packages.${SUITE}.%ARCH% . Sources.${SUITE} -- ${ARCHS} || true
 
     echo "`date`: ${ARCHIVE}/${SUITE} done"
 }
@@ -94,8 +94,8 @@ trigger_wb_update_for_overlay() {
         new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}
         new-keep-latest ${ARCH} Packages.${SUITE}.${ARCH} $(eval echo $(echo ${PACKAGES_BASE} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}-all
     done
-    parallel -l 5 -i wanna-build -v --Pas ${PAS_FILE} --merge-v3 -A "{}" --dist=$WB_SUITE Packages.%SUITE%.%ARCH% . Sources.%SUITE% . \
-        Packages.%SUITE%.%ARCH%-all . Sources.%SUITE%-all -- ${ARCHS} || true
+    parallel -l 5 -i wanna-build -v --Pas ${PAS_FILE} --merge-v3 -A "{}" --dist=$WB_SUITE Packages.${SUITE}.%ARCH% . Sources.${SUITE} . \
+        Packages.${SUITE}.%ARCH%-all . Sources.${SUITE}-all -- ${ARCHS} || true
 
     echo "`date`: ${ARCHIVE}/${SUITE} done"
 }
@@ -127,8 +127,8 @@ trigger_wb_update_with_secondary() {
         new-keep-latest ${ARCH} $(eval echo $(echo ${PACKAGES} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}
         ucat Packages.${SUITE}.${ARCH} $(eval echo $(echo ${PACKAGES_BASE} | sed -e 's/%ARCH%/${ARCH}/g')) > Packages.${SUITE}.${ARCH}-all
     done
-    parallel -l 5 -i wanna-build -v --Pas ${PAS_FILE} --merge-v3 -A "{}" --dist=$WB_SUITE Packages.%SUITE%.%ARCH% . Sources.%SUITE% . \
-        Packages.%SUITE%.%ARCH%-all -- ${ARCHS} || true
+    parallel -l 5 -i wanna-build -v --Pas ${PAS_FILE} --merge-v3 -A "{}" --dist=$WB_SUITE Packages.${SUITE}.%ARCH% . Sources.${SUITE} . \
+        Packages.${SUITE}.%ARCH%-all -- ${ARCHS} || true
 
     echo "`date`: ${ARCHIVE}/${SUITE} done"
 }
diff --git a/triggers/filter-nonfree b/triggers/filter-nonfree
old mode 100644
new mode 100755
index 074b767..feb4e16
--- a/triggers/filter-nonfree
+++ b/triggers/filter-nonfree
@@ -1,33 +1,38 @@
 #! /usr/bin/python
 
 
-import sys, os.path
+import sys, os.path, gzip, subprocess
 #from debian_bundle import deb822
 import apt_pkg
 
 
-def check_source(ok, f, fw):
+def check_source(ok, srcs, fw):
     ret=[]
-    pkg = apt_pkg.ParseTagFile(f)
-    while pkg.Step():
-        if not pkg.Section.get('Package'): continue
-        if pkg.Section.get('Section','').startswith('non-free'):
-            if pkg.Section.get('Autobuild') != 'yes': continue
-            if pkg.Section['Package'] not in ok: continue
-        ret += [pkg.Section['Package']]
-        fw.write(str(pkg.Section))
-        fw.write("\n")
-    return ret
-    f.close()
+    for f in srcs:
+        #f = open(f, 'r')
+        f = subprocess.Popen("zcat %s" % (f), shell=True, stdout=subprocess.PIPE).stdout
+        pkg = apt_pkg.ParseTagFile(f)
+        while pkg.Step():
+            if not pkg.Section.get('Package'): continue
+            if pkg.Section.get('Section','').startswith('non-free'):
+                if pkg.Section.get('Autobuild') != 'yes': continue
+                if pkg.Section['Package'] not in ok: continue
+            ret += [pkg.Section['Package']]
+            fw.write(str(pkg.Section))
+            fw.write("\n")
+        f.close()
     fw.close()
+    return ret
 
-def check_binary(ok, f, fw):
-    pkg = apt_pkg.ParseTagFile(f)
-    while pkg.Step():
-        if pkg.Section.get('Source', pkg.Section.get('Package')).split(" ")[0] not in ok: continue
-        fw.write(str(pkg.Section))
-        fw.write("\n")
-    f.close()
+def check_binary(ok, bins, fw):
+    for f in bins:
+        f = subprocess.Popen("zcat %s" % (f), shell=True, stdout=subprocess.PIPE).stdout
+        pkg = apt_pkg.ParseTagFile(f)
+        while pkg.Step():
+            if pkg.Section.get('Source', pkg.Section.get('Package')).split(" ")[0] not in ok: continue
+            fw.write(str(pkg.Section))
+            fw.write("\n")
+        f.close()
     fw.close()
 
 def outfile(name, file):
@@ -35,16 +40,32 @@ def outfile(name, file):
     #return open(file + name, "w")
     if file.endswith(name): return open(file[:-len(name)], "w")
 
+def replarch(arch, pkgs):
+    return [ x.replace('%ARCH%', arch) for x in pkgs ]
+
 def main():
+    # call me:
+    # /org/wanna-build/etc/non-free-include-list "arch1 arch2" write-sources write-packages Sources* . Packages*
     if len(sys.argv) <= 4:
         print "Error - too few args"
         return
 
-    name = sys.argv[1]
-    oklist = [ x[:-1].split(' ')[0] for x in open(sys.argv[2]) ]
-    okpkgs = check_source(oklist, open(sys.argv[3]), outfile(name, sys.argv[3]))
-    for file in sys.argv[4:]:
-        check_binary(okpkgs, open(file), outfile(name, file))
+    oklist = [ x[:-1].split(' ')[0] for x in open(sys.argv[1]) ]
+    #outsrcs = open(sys.argv[3], "w")
+    #outpkgs = open(sys
+    rem = sys.argv[5:]
+    if '.' not in rem:
+        print "need both Sources and Packages"
+        return
+    src = rem[:rem.index('.')]
+    bin = rem[rem.index('.')+1:]
+    if not src or not bin:
+        print "need non-empty Sources and Packages"
+        return
+    okpkgs = check_source(oklist, src, open(sys.argv[3], "w"))
+    print okpkgs
+    for arch in sys.argv[2].split(" "):
+        check_binary(okpkgs, replarch(arch, bin), open(replarch(arch, [sys.argv[4]])[0], "w"))
 
 if __name__ == '__main__':
         main()
diff --git a/triggers/trigger.debian b/triggers/trigger.debian
index dd6eaed..6b6b8d5 100755
--- a/triggers/trigger.debian
+++ b/triggers/trigger.debian
@@ -45,9 +45,13 @@ main_sync() {
     do
         SOURCES="${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib}/source/Sources.gz"
         SOURCES="${SOURCES} ${ARCHIVE_BASE}/buildd-${SUITE}/Sources.gz"
+        NFSOURCES="${ARCHIVE_BASE}/archive/${SUITE}/non-free/source/Sources.gz"
         PACKAGES="${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib}/binary-%ARCH%/Packages.gz"
         PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/archive/${SUITE}/{main,contrib}/debian-installer/binary-%ARCH%/Packages.gz"
         PACKAGES="${PACKAGES} ${ARCHIVE_BASE}/buildd-${SUITE}/Packages.gz"
+        NFPACKAGES="${ARCHIVE_BASE}/archive/${SUITE}/non-free/binary-%ARCH%/Packages.gz"
+        NFPACKAGES="${NFPACKAGES} ${ARCHIVE_BASE}/archive/${SUITE}/non-free/debian-installer/binary-%ARCH%/Packages.gz"
+        /org/wanna-build/triggers/filter-nonfree /org/wanna-build/etc/non-free-include-list "`get_architectures $SUITE`" Sources.${SUITE}.non-free Packages.${SUITE}.%ARCH%.non-free ${NFSOURCES} . ${NFPACKAGES} || true
 
         if [ "$SUITE" != "experimental" ]
         then