Also for stats report which repo and which job number use our setup

[neurodebian.git] / tools / blends-inject

diff --git a/tools/blends-inject b/tools/blends-inject
index 6477526e7774d52adbf091c861945064cf4583fe..d62d3ba7dee23e06f5228b3fa5d6123479e58f11 100755 (executable)
--- a/tools/blends-inject
+++ b/tools/blends-inject
@@ -16,10 +16,12 @@ possibly of different blends.  This script allows to automate:
 Possible TODOs:
 ---------------
 
 Possible TODOs:
 ---------------
 

-Whenever processing multiple files, figure out topdir automatically,
-so we could do
-
-blends-inject */debian/blends
+* For every package the same task file might be re-read/written (if
+  entry changed/added) from disk.
+  That allows to replace easily original entry for 'source' package
+  (listed as Suggests:) with actual first listed binary package.
+  This should be taken into consideration if current per-package
+  handling gets changed

 
 """
 
 
 """
 


@@ -31,24 +33,88 @@ Paths to the blends top directories, containing tasks directories are
 specified in ~/.blends-inject.cfg file, e.g.::
 
  [debian-med]
 specified in ~/.blends-inject.cfg file, e.g.::
 
  [debian-med]

- path = /home/yoh/deb/debian-med/
+ path = ~/deb/debian-med/

 
  [debian-science]
 
  [debian-science]

- path = /home/yoh/deb/debian-science/
-
+ path = ~/deb/debian-science/

 
 Definition of the fields for task files by default are looked up
 
 Definition of the fields for task files by default are looked up

-within debian/blends, or files provided in the command line.
+within debian/blends, or files provided in the command line.  Also for "-a"
+mode of operation you should define list of globs to match your debian/blends
+files::
+
+ [paths]
+ all=~/deb/gits/pkg-exppsy/neurodebian/future/blends/*
+     ~/deb/gits/*/debian/blends
+     ~/deb/gits/pkg-exppsy/*/debian/blends
+ # Python regular expression on which files to skip
+ # Default is listed below
+ #skip=.*[~#]$
+

 
 Format of debian/blends
 -----------------------
 
 
 Format of debian/blends
 -----------------------
 

-TODO:
+Example::
+
+ ; If originally filed using project source name, and it is different
+ ; from the primary (first) binary package name, keep 'Source' to be
+ ; able to adopt previously included tasks entry
+Source: brian
+
+ ; Define the format on how entries should be handled.
+ ; Possible values:
+ ;   extended -- whenever package is not in Debian and additional
+ ;               fields should be obtained from debian/*:
+ ;               * License
+ ;               * WNPP
+ ;               * Pkg-Description
+ ;               * Responsible
+ ;               * Homepage
+ ;               * Vcs-*
+ ;   plain [default] -- only fields listed here should be mentioned.
+ ;               Common use -- whenever package is already known to UDD.
+ ;
+ ;  By default, all fields specified previously propagate into following
+ ;  packages as well.  If that is not desired, add suffix '-clean' to
+ ;  the Format
+Format: extended
+
+Tasks: debian-science/neuroscience-modeling
+
+ ; Could have Depends/Recommends/Suggests and Ignore
+ ; All those define Pkg-Name field which is not included
+ ; in the final "rendering" but is available as Pkg-Name item
+Depends: python-brian
+Pkg-URL: http://neuro.debian.net/pkgs/%(Pkg-Name)s.html
+Language: Python, C
+Published-Authors: Goodman D.F. and Brette R.
+Published-Title: Brian: a simulator for spiking neural networks in Python
+Published-In: Front. Neuroinform
+Published-Year: 2008
+Published-DOI: 10.3389/neuro.11.005.2008
+
+ ; May be some previous entry should be removed, thus say so
+Remove: python-brian-doc
+
+ ;Tasks: debian-med/imaging-dev
+ ;Why: Allows interactive development/scripting
+
+ ; ; It should be possible to switch between formats,
+ ; ; e.g. if some component is not yet in Debian
+ ;Format: extended
+ ;
+ ; ; Now some bogus one but with customizations
+ ;Tasks: debian-med/documentation
+ ;Recommends: python-brian-doc
+ ;Language:
+ ;Remark: some remark
+ ;

 
 """
 
 
 
 """
 
 

-import re, os, sys
+import re, os, sys, tempfile, glob

 from os.path import join, exists, expanduser, dirname, basename
 
 from ConfigParser import ConfigParser
 from os.path import join, exists, expanduser, dirname, basename
 
 from ConfigParser import ConfigParser


@@ -57,32 +123,49 @@ from optparse import OptionParser, Option
 from copy import deepcopy
 #from debian_bundle import deb822
 from debian import deb822
 from copy import deepcopy
 #from debian_bundle import deb822
 from debian import deb822

+#import deb822

 from debian.changelog import Changelog
 
 from debian.changelog import Changelog
 

+# all files we are dealing with should be UTF8, thus
+# lets override
+import codecs
+
+def open(f, *args):
+    return codecs.open(f, *args, encoding='utf-8')
+

 __author__ = 'Yaroslav Halchenko'
 __prog__ = os.path.basename(sys.argv[0])
 __author__ = 'Yaroslav Halchenko'
 __prog__ = os.path.basename(sys.argv[0])

-__version__ = '0.0.1'
+__version__ = '0.0.7'

 __copyright__ = 'Copyright (c) 2010 Yaroslav Halchenko'
 __license__ = 'GPL'
 
 # What fields initiate new package description
 __copyright__ = 'Copyright (c) 2010 Yaroslav Halchenko'
 __license__ = 'GPL'
 
 # What fields initiate new package description

-PKG_FIELDS = ('depends', 'recommends', 'suggests', 'ignore', 'removed')
+PKG_FIELDS = ('depends', 'recommends', 'suggests', 'ignore', 'remove')

 
 # We might need to resort to assure some what a canonical order
 
 # We might need to resort to assure some what a canonical order

-FIELDS_ORDER = ('depends', 'recommends', 'suggests', 'ignore',
-                'homepage', 'language', 'wnpp', 'responsible', 'license',
+# Prefixes for "standard" blends/tasks fields.  Others do not get embedded
+# into tasks files
+BLENDS_FIELDS_PREFIXES = ('depends', 'recommends', 'suggests', 'ignore',
+                'why', 'homepage', 'language', 'wnpp', 'responsible', 'license',

                 'vcs-', 'pkg-url', 'pkg-description',
                 'published-', 'x-', 'registration', 'remark')
                 'vcs-', 'pkg-url', 'pkg-description',
                 'published-', 'x-', 'registration', 'remark')

+# Additional fields which might come useful (e.g. for filing wnpp bugs)
+# but are not "standard" thus should be in the trailer
+CUSTOM_FIELDS_PREFIXES = ('author', 'pkg-name', 'pkg-source',
+                          'version', 'remove')
+# Other fields should cause Error for consistency
+
+FIELDS_ORDER = BLENDS_FIELDS_PREFIXES + CUSTOM_FIELDS_PREFIXES

 
 verbosity = None
 
 
 verbosity = None
 

-def error(msg, exit_code):
+def error(msg, exit_code=1):

     sys.stderr.write(msg + '\n')
     sys.exit(exit_code)
 
 def verbose(level, msg):
     if level <= verbosity:
     sys.stderr.write(msg + '\n')
     sys.exit(exit_code)
 
 def verbose(level, msg):
     if level <= verbosity:

-        print " "*level, msg
+        sys.stderr.write(" "*level + msg + '\n')

 
 
 def parse_debian_blends(f='debian/blends'):
 
 
 def parse_debian_blends(f='debian/blends'):


@@ -95,6 +178,7 @@ def parse_debian_blends(f='debian/blends'):
     for p in deb822.Deb822.iter_paragraphs(open(f)):
         items += p.items()
 
     for p in deb822.Deb822.iter_paragraphs(open(f)):
         items += p.items()
 

+    verbose(6, "Got items %s" % items)

     # Traverse and collect things
     format_ = 'plain'
     format_clean = False # do not propagate fields into a new pkg if True
     # Traverse and collect things
     format_ = 'plain'
     format_clean = False # do not propagate fields into a new pkg if True


@@ -102,8 +186,28 @@ def parse_debian_blends(f='debian/blends'):
     pkgs = []
     tasks = []
 
     pkgs = []
     tasks = []
 

+    def new_pkg(prev_pkg, bname, sname, tasks):
+        """Helper function to create a new package
+        """
+        if format_clean or prev_pkg is None:
+            pkg = deb822.Deb822()
+        else:
+            pkg = deepcopy(prev_pkg)
+            for k_ in PKG_FIELDS:   # prune older depends
+                pkg.pop(k_, None)
+        pkg['Pkg-Name'] = pkg[k] = bname.lower()
+        if sname is not None:
+            sname = sname.lower()
+        pkg['Pkg-Source'] = sname
+        pkgs.append(pkg)
+        pkg.tasks = dict( (t.strip(), deb822.Deb822Dict()) for t in tasks )
+        pkg.format = format_
+        return pkg
+

     for k, v in items:
     for k, v in items:

+

         kl = k.lower()
         kl = k.lower()

+

         if kl == 'source':
             source = v.strip()
         elif kl == 'format':
         if kl == 'source':
             source = v.strip()
         elif kl == 'format':


@@ -112,25 +216,25 @@ def parse_debian_blends(f='debian/blends'):
             if format_clean:
                 format_ = format_[:-6]
         elif kl == 'tasks':
             if format_clean:
                 format_ = format_[:-6]
         elif kl == 'tasks':

-            tasks = v.split(',')
-            newtasks = True                 # either we need to provide tune-ups
+            tasks = [x.strip() for x in v.split(',')]
+            newtasks = pkg is not None      # either we need to provide tune-ups

                                             # for current package
         elif kl in PKG_FIELDS: # new package
                                             # for current package
         elif kl in PKG_FIELDS: # new package

-            if format_clean or pkg is None:
-                pkg = deb822.Deb822()
-            else:
-                pkg = deepcopy(pkg)
-                for k_ in PKG_FIELDS:   # prune older depends
-                    pkg.pop(k_, None)
-            pkg['Pkg-Name'] = pkg[k] = v
-            if source is None:
+            if source is None and not format_ in ['extended']:

                 source = v
                 source = v

-            pkg['Pkg-Source'] = source
-            pkgs.append(pkg)
-            pkg.tasks = dict( (t.strip(), deb822.OrderedSet()) for t in tasks )
-            pkg.format = format_
+            pkg = new_pkg(pkg, v, source, tasks)

             newtasks = False
         else:
             newtasks = False
         else:

+                       if pkg is None:
+                               # So we had just source?
+                               if source is None:
+                                       error("No package or source is known where to add %s" % (k,), 1)
+                                       # TODO: just deduce source from DebianMaterials
+                               pkg = new_pkg(pkg, source, source, tasks)
+                               # Since only source is available, it should be only Suggest:-ed
+                               pkg['Suggests'] = source.lower()
+                               newtasks = False
+

             if newtasks:
                 # Add customization
                 for t in tasks:
             if newtasks:
                 # Add customization
                 for t in tasks:


@@ -140,6 +244,7 @@ def parse_debian_blends(f='debian/blends'):
             else:
                 # just store the key in the pkg itself
                 pkg[k] = v
             else:
                 # just store the key in the pkg itself
                 pkg[k] = v

+

     return pkgs
 
 
     return pkgs
 
 


@@ -147,7 +252,9 @@ def expand_pkgs(pkgs, topdir='.'):
     """In-place modification of pkgs taking if necessary additional
     information from Debian materials, and pruning empty fields
     """
     """In-place modification of pkgs taking if necessary additional
     information from Debian materials, and pruning empty fields
     """

+    verbose(4, "Expanding content for %d packages" % len(pkgs))

     debianm = None
     debianm = None

+

     # Expand packages which format is extended
     for pkg in pkgs:
         if pkg.format == 'extended':
     # Expand packages which format is extended
     for pkg in pkgs:
         if pkg.format == 'extended':


@@ -156,10 +263,12 @@ def expand_pkgs(pkgs, topdir='.'):
                 debianm = DebianMaterials(topdir)
             for k, m in (('License', lambda: debianm.get_license(pkg['Pkg-Name'])),
                          ('WNPP', debianm.get_wnpp),
                 debianm = DebianMaterials(topdir)
             for k, m in (('License', lambda: debianm.get_license(pkg['Pkg-Name'])),
                          ('WNPP', debianm.get_wnpp),

-                         ('Pkg-description',
-                          lambda: debianm.binaries[pkg['Pkg-Name']]['Description']),
+                         ('Pkg-Description',
+                          lambda: debianm.get_description(pkg['Pkg-Name'])),

                          ('Responsible', debianm.get_responsible),
                          ('Responsible', debianm.get_responsible),

-                         ('Homepage', lambda: debianm.source.get('Homepage', None))):
+                         ('Homepage', lambda: debianm.source.get('Homepage', None)),
+                         ('Pkg-source', lambda: debianm.source.get('Source', None)),
+                         ):

                 if pkg.get(k, None):
                     continue
                 v = m()
                 if pkg.get(k, None):
                     continue
                 v = m()


@@ -168,7 +277,24 @@ def expand_pkgs(pkgs, topdir='.'):
             # VCS fields
             pkg.update(debianm.get_vcsfields())
 
             # VCS fields
             pkg.update(debianm.get_vcsfields())
 

-def key_prefix_compare(x, y, order, strict=False, case=False):
+
+def prefix_index(x, entries, strict=True, case=False, default=10000):
+    """Returns an index for the x in entries
+    """
+    if not case:
+        x = x.lower()
+    for i, v in enumerate(entries):
+        if x.startswith(v):
+            return i
+
+    if strict:
+        raise IndexError(
+            "Could not find location for %s as specified by %s" %
+            (x, entries))
+    return default
+
+
+def key_prefix_compare(x, y, order, strict=True, case=False):

     """Little helper to help with sorting
 
     Sorts according to the order of string prefixes as given by
     """Little helper to help with sorting
 
     Sorts according to the order of string prefixes as given by


@@ -179,22 +305,8 @@ def key_prefix_compare(x, y, order, strict=False, case=False):
     if not case:
         order = [v.lower() for v in order]
 
     if not case:
         order = [v.lower() for v in order]
 

-    def prefix_index(t, order, strict=True, case=False):
-        x = t[0]
-        if not case:
-            x = x.lower()
-        for i, v in enumerate(order):
-            if x.startswith(v):
-                return i
-
-        if strict:
-            raise IndexError(
-                "Could not find location for %s as specified by %s" %
-                (x, order))
-        return 10000                    #  some large number ;)
-
-    cmp_res =  cmp(prefix_index(x, order, strict, case),
-                   prefix_index(y, order, strict, case))
+    cmp_res =  cmp(prefix_index(x[0], order, strict, case),
+                   prefix_index(y[0], order, strict, case))

     if not cmp_res:                     # still unknown
         return cmp(x, y)
     return cmp_res
     if not cmp_res:                     # still unknown
         return cmp(x, y)
     return cmp_res


@@ -230,10 +342,16 @@ def group_packages_into_tasks(pkgs):
 
             # Move Pkg-source/name into attributes
             pkg__.source = pkg__.pop('Pkg-Source')
 
             # Move Pkg-source/name into attributes
             pkg__.source = pkg__.pop('Pkg-Source')

-            pkg__.name = pkg__.pop('Pkg-name')
+            pkg__.name = pkg__.pop('Pkg-Name')
+            # Store the action taken on the package for later on actions
+            for f in PKG_FIELDS:
+                if f in pkg__:
+                    pkg__.action = f
+                    break

 
             tasks[task] = tasks.get(task, []) + [pkg__]
 
             tasks[task] = tasks.get(task, []) + [pkg__]

-
+    verbose(4, "Grouped %d packages into %d tasks: %s" %
+            (len(pkgs), len(tasks), ', '.join(tasks.keys())))

     return tasks
 
 def inject_tasks(tasks, config):
     return tasks
 
 def inject_tasks(tasks, config):


@@ -241,23 +359,36 @@ def inject_tasks(tasks, config):
     for task, pkgs in tasks.iteritems():
         verbose(2, "Task %s with %d packages" % (task, len(pkgs)))
         blend, puretask = task.split('/')
     for task, pkgs in tasks.iteritems():
         verbose(2, "Task %s with %d packages" % (task, len(pkgs)))
         blend, puretask = task.split('/')

-        taskfile = join(config.get(blend, 'path'), 'tasks', puretask)
+        taskfile = expanduser(join(config.get(blend, 'path'), 'tasks', puretask))

 
         # Load the file
         stats = dict(Added=[], Modified=[])
         for pkg in pkgs:
             msgs = {'Name': pkg.name.strip(), 'Action': None}
 
         # Load the file
         stats = dict(Added=[], Modified=[])
         for pkg in pkgs:
             msgs = {'Name': pkg.name.strip(), 'Action': None}

+
+            # Create a copy of the pkg with only valid tasks
+            # fields:
+            # TODO: make it configurable?
+            pkg = deepcopy(pkg)
+            for k in pkg:
+                if prefix_index(k, BLENDS_FIELDS_PREFIXES,
+                                strict=False, default=None) is None:
+                    pkg.pop(k) # remove it from becoming present in
+                               # the taskfile
+

             # Find either it is known to the task file already
 
             # Load entirely so we could simply manipulate
             entries = open(taskfile).readlines()
             known = False
             # We need to search by name and by source
             # Find either it is known to the task file already
 
             # Load entirely so we could simply manipulate
             entries = open(taskfile).readlines()
             known = False
             # We need to search by name and by source

-            # We need to search for every possible type of dependecy
-            regexp = re.compile('^ *(%s) *: *(%s) *$' %
-                                ('|'.join(PKG_FIELDS),
-                                '|'.join((pkg.name, pkg.source))),
-                                re.I)
+            # We need to search for every possible type of dependency
+            regexp_str = '^ *(%s) *: *(%s) *$' \
+                         % ('|'.join(PKG_FIELDS),
+                            '|'.join((pkg.name, pkg.source)).replace('+', '\+'))
+            verbose(4, "Searching for presence in %s using regexp: '%s'"
+                    % (taskfile, regexp_str))
+            regexp = re.compile(regexp_str, re.I)

             for istart, e in enumerate(entries):
                 if regexp.search(e):
                     verbose(4, "Found %s in position %i: %s" %
             for istart, e in enumerate(entries):
                 if regexp.search(e):
                     verbose(4, "Found %s in position %i: %s" %


@@ -265,9 +396,11 @@ def inject_tasks(tasks, config):
                     known = True
                     break
 
                     known = True
                     break
 

-            descr = ' ; Added by %s %s.  Modified manually: False\n' % \
+            descr = ' ; Added by %s %s. [Please note here if modified manually]\n' % \

                     (__prog__,  __version__)
                     (__prog__,  __version__)

-            # Replace existing entry
+
+            entry = pkg.dump()
+            # Replace existing entry?

             if known:
                 # TODO: Check if previous copy does not have our preceding comment
                 # Find the previous end
             if known:
                 # TODO: Check if previous copy does not have our preceding comment
                 # Find the previous end


@@ -280,31 +413,53 @@ def inject_tasks(tasks, config):
 
                 # Lets not change file without necessity, if entry is identical --
                 # do nothing
 
                 # Lets not change file without necessity, if entry is identical --
                 # do nothing

-                entry = pkg.dump()

                 old_entry = entries[istart:istart+icount]
 
                 if u''.join(old_entry) == entry:
                 old_entry = entries[istart:istart+icount]
 
                 if u''.join(old_entry) == entry:

-                   pass
+                    # no changes -- just go to the next one
+                    continue

                 else: # Rewrite the entry
                    if __prog__ in entries[istart-1]:
                        istart -= 1
                        icount += 2
                 else: # Rewrite the entry
                    if __prog__ in entries[istart-1]:
                        istart -= 1
                        icount += 2

-                   if not 'Removed' in pkg.keys():
-                       entries = entries[:istart] + [descr + entry] + entries[istart+icount:]
+                   if 'remove' != pkg.action:
+                       entry = descr + entry

                        msgs['Action'] = 'Changed'
                    else:
                        while entries[istart-1].strip() == '':
                            istart -=1
                            icount +=2
                        msgs['Action'] = 'Changed'
                    else:
                        while entries[istart-1].strip() == '':
                            istart -=1
                            icount +=2

-                       entries = entries[:istart] + entries[istart+icount:]
+                       entry = ''

                        msgs['Action'] = 'Removed'
                        msgs['Action'] = 'Removed'

-                   open(taskfile, 'w').write(''.join(entries))
-            elif not 'removed' in pkg:  # or Append one
+                   entries_prior = entries[:istart]
+                   entries_post = entries[istart+icount:]
+            elif not 'remove' == pkg.action:  # or Append one

                 msgs['Action'] = 'Added'
                 msgs['Action'] = 'Added'

+                entries_prior = entries
+                entry = descr + entry
+                entries_post = []

                 # could be as simple as
                 # could be as simple as

-                open(taskfile, 'a').write('\n%s%s' % (descr, pkg.dump(),))
+                # Lets do 'in full' for consistent handling of empty lines
+                # around
+                #output = '\n%s%s' % (descr, pkg.dump(),)
+                #open(taskfile, 'a').write(output)

 
             if msgs['Action']:
 
             if msgs['Action']:

+                # Prepare for dumping
+                # Prune spaces before
+                while len(entries_prior) and entries_prior[-1].strip() == '':
+                    entries_prior = entries_prior[:-1]
+                if len(entries_prior) and not entries_prior[-1].endswith('\n'):
+                    entries_prior[-1] += '\n' # assure present trailing newline
+                # Prune spaces after
+                while len(entries_post) and entries_post[0].strip() == '':
+                    entries_post = entries_post[1:]
+                if len(entries_post) and len(entry):
+                    # only then trailing empty line
+                    entry += '\n'
+                output = ''.join(entries_prior + [ '\n' + entry ] + entries_post)
+                open(taskfile, 'w').write(output) # then only overwrite
+

                 verbose(3, "%(Action)s %(Name)s" % msgs)
 
 
                 verbose(3, "%(Action)s %(Name)s" % msgs)
 
 


@@ -348,7 +503,9 @@ class DebianMaterials(object):
             if v.get('Source', None):
                 self._source = v
             else:
             if v.get('Source', None):
                 self._source = v
             else:

-                self._binaries[v['Package']] = v
+                # Since it might be hash-commented out
+                if 'Package' in v:
+                    self._binaries[v['Package']] = v

 
     def get_license(self, package=None, first_only=True):
         """Return a license(s). Parsed out from debian/copyright if it is
 
     def get_license(self, package=None, first_only=True):
         """Return a license(s). Parsed out from debian/copyright if it is


@@ -401,6 +558,72 @@ class DebianMaterials(object):
                 vcs[f] = v
         return vcs
 
                 vcs[f] = v
         return vcs
 

+    def get_description(self, pkg_name):
+        """Some logic to extract description.
+
+           If binary package matching pkg_name is found -- gets it description.
+           If no binary package with such name, and name matches source name,
+           obtain description of the first binary package.
+        """
+        if pkg_name in self.binaries:
+            pass
+        elif pkg_name.lower() == self.source['Source'].lower():
+            pkg_name = self.binaries.keys()[0]
+        else:
+            error("Name %s does not match any binary, nor source package in %s"
+                  % (pkg_name, self))
+        return self.binaries[pkg_name]['Description']
+
+def print_wnpp(pkgs, config, wnpp_type="ITP"):
+    """Little helper to spit out formatted entry for WNPP bugreport
+
+    TODO: It would puke atm if any field is missing
+    """
+
+    pkg = pkgs[0]                       # everything is based on the 1st one
+    opts = dict(pkg.items())
+    opts['WNPP-Type'] = wnpp_type.upper()
+    opts['Pkg-Description-Short'] = re.sub('\n.*', '', pkg['Pkg-Description'])
+
+    subject = "%(WNPP-Type)s: %(Pkg-Name)s -- %(Pkg-Description-Short)s" % opts
+    body = """*** Please type your report below this line ***
+
+* Package name    : %(Pkg-Name)s
+  Version         : %(Version)s
+  Upstream Author : %(Author)s
+* URL             : %(Homepage)s
+* License         : %(License)s
+  Programming Lang: %(Language)s
+  Description     : %(Pkg-Description)s
+
+""" % opts
+
+    # Unfortunately could not figure out how to set the owner, so I will just print it out
+    if False:
+        tmpfile = tempfile.NamedTemporaryFile()
+        tmpfile.write(body)
+        tmpfile.flush()
+        cmd = "reportbug -b --paranoid --subject='%s' --severity=wishlist --body-file='%s' -o /tmp/o.txt wnpp" \
+              % (subject, tmpfile.name)
+        verbose(2, "Running %s" %cmd)
+        os.system(cmd)
+    else:
+        print "Subject: %s\n\n%s" % (subject, body)
+
+
+def is_template(p):
+    """Helper to return true if pkg definition looks like a template
+       and should not be processed
+    """
+    # We might want to skip some which define a skeleton
+    # (no source/homepage/etc although fields are there)
+    for f in ['vcs-browser', 'pkg-url', 'pkg-description',
+              'published-Title', 'pkg-name', 'homepage',
+              'author']:
+        if f in p and p[f] != "":
+            return False
+    return True
+

 
 def main():
 
 
 def main():
 


@@ -424,38 +647,75 @@ def main():
         Option("-v", "--verbosity", action="store", type="int",
                dest="verbosity", default=1, help="Noise level."))
 
         Option("-v", "--verbosity", action="store", type="int",
                dest="verbosity", default=1, help="Noise level."))
 

+    # We might like to create a separate 'group' of options for commands
+    p.add_option(
+        Option("-w", action="store_true",
+               dest="wnpp", default=False,
+               help="Operate in WNPP mode: dumps cut-paste-able entry for WNPP bugreport"))
+
+    p.add_option(
+        Option("--wnpp", action="store",
+               dest="wnpp_mode", default=None,
+               help="Operate in WNPP mode: dumps cut-paste-able entry for WNPP bugreport"))
+
+    p.add_option(
+        Option("-a", action="store_true",
+               dest="all_mode", default=False,
+               help="Process all files listed in paths.all"))
+
+

     (options, infiles) = p.parse_args()
     global verbosity; verbosity = options.verbosity
 
     (options, infiles) = p.parse_args()
     global verbosity; verbosity = options.verbosity
 

-    if not len(infiles):
-        infiles = [join(options.topdir or './', 'debian/blends')]     #  default one
+       if options.wnpp and options.wnpp_mode is None:
+            options.wnpp_mode = 'ITP'

 
     # Load configuration
 
     # Load configuration

-    config = ConfigParser()
+    config = ConfigParser(defaults={'skip': '.*[~#]$'})

     config.read(options.config_file)
 
     config.read(options.config_file)
 

+    if options.all_mode:
+        if len(infiles):
+            raise ValueError("Do not specify any files in -a mode.  Use configuration file, section paths, option all")
+        globs = config.get('paths', 'all', None).split()
+        infiles = reduce(list.__add__, (glob.glob(expanduser(f)) for f in globs))
+        verbose(1, "Found %d files in specified paths" % len(infiles))
+
+    if not len(infiles):
+        infiles = [join(options.topdir or './', 'debian/blends')]     #  default one
+
+    skip_re = re.compile(config.get('paths', 'skip', None))
+

     for blends_file in infiles:
         verbose(1, "Processing %s" % blends_file)
         if not exists(blends_file):
             error("Cannot find a file %s.  Either provide a file or specify top "
                   "debian directory with -d." % blends_file, 1)
     for blends_file in infiles:
         verbose(1, "Processing %s" % blends_file)
         if not exists(blends_file):
             error("Cannot find a file %s.  Either provide a file or specify top "
                   "debian directory with -d." % blends_file, 1)

+        if skip_re.match(blends_file):
+            verbose(2, "W: Skipped since matches paths.skip regexp")
+            continue

         pkgs = parse_debian_blends(blends_file)
         if options.topdir is None:
             if dirname(blends_file).endswith('/debian'):
                 topdir = dirname(dirname(blends_file))
             else:
         pkgs = parse_debian_blends(blends_file)
         if options.topdir is None:
             if dirname(blends_file).endswith('/debian'):
                 topdir = dirname(dirname(blends_file))
             else:

-                topdir = '.'            # and hope for the best
+                topdir = '.'            # and hope for the best ;)

         else:
             topdir = options.topdir
         else:
             topdir = options.topdir

-        expand_pkgs(pkgs, topdir=topdir)
-        tasks = group_packages_into_tasks(pkgs)
-        inject_tasks(tasks, config)
-
-    #for t,v in tasks.iteritems():
-    #    print "-------TASK: ", t
-    #    print ''.join(str(t_) for t_ in v)
-    #print pkgs[0]
-    #print tasks['debian-med/documentation'][0]
+
+               expand_pkgs(pkgs, topdir=topdir)
+
+        pkgs = [p for p in pkgs if not is_template(p)]
+        if not len(pkgs):
+            verbose(2, "W: Skipping since seems to contain templates only")
+            continue
+        if options.wnpp_mode is not None:
+                   print_wnpp(pkgs, config, options.wnpp_mode)
+        else:
+            # by default -- operate on blends/tasks files
+            tasks = group_packages_into_tasks(pkgs)
+            inject_tasks(tasks, config)
+

 
 if __name__ == '__main__':
     main()
 
 if __name__ == '__main__':
     main()