X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=tools%2Fblends-inject;h=d62d3ba7dee23e06f5228b3fa5d6123479e58f11;hb=HEAD;hp=6477526e7774d52adbf091c861945064cf4583fe;hpb=286fcd8724d8fb19f4ee94441990d5f41f9554c0;p=neurodebian.git diff --git a/tools/blends-inject b/tools/blends-inject index 6477526..d62d3ba 100755 --- a/tools/blends-inject +++ b/tools/blends-inject @@ -16,10 +16,12 @@ possibly of different blends. This script allows to automate: Possible TODOs: --------------- -Whenever processing multiple files, figure out topdir automatically, -so we could do - -blends-inject */debian/blends +* For every package the same task file might be re-read/written (if + entry changed/added) from disk. + That allows to replace easily original entry for 'source' package + (listed as Suggests:) with actual first listed binary package. + This should be taken into consideration if current per-package + handling gets changed """ @@ -31,24 +33,88 @@ Paths to the blends top directories, containing tasks directories are specified in ~/.blends-inject.cfg file, e.g.:: [debian-med] - path = /home/yoh/deb/debian-med/ + path = ~/deb/debian-med/ [debian-science] - path = /home/yoh/deb/debian-science/ - + path = ~/deb/debian-science/ Definition of the fields for task files by default are looked up -within debian/blends, or files provided in the command line. +within debian/blends, or files provided in the command line. Also for "-a" +mode of operation you should define list of globs to match your debian/blends +files:: + + [paths] + all=~/deb/gits/pkg-exppsy/neurodebian/future/blends/* + ~/deb/gits/*/debian/blends + ~/deb/gits/pkg-exppsy/*/debian/blends + # Python regular expression on which files to skip + # Default is listed below + #skip=.*[~#]$ + Format of debian/blends ----------------------- -TODO: +Example:: + + ; If originally filed using project source name, and it is different + ; from the primary (first) binary package name, keep 'Source' to be + ; able to adopt previously included tasks entry +Source: brian + + ; Define the format on how entries should be handled. + ; Possible values: + ; extended -- whenever package is not in Debian and additional + ; fields should be obtained from debian/*: + ; * License + ; * WNPP + ; * Pkg-Description + ; * Responsible + ; * Homepage + ; * Vcs-* + ; plain [default] -- only fields listed here should be mentioned. + ; Common use -- whenever package is already known to UDD. + ; + ; By default, all fields specified previously propagate into following + ; packages as well. If that is not desired, add suffix '-clean' to + ; the Format +Format: extended + +Tasks: debian-science/neuroscience-modeling + + ; Could have Depends/Recommends/Suggests and Ignore + ; All those define Pkg-Name field which is not included + ; in the final "rendering" but is available as Pkg-Name item +Depends: python-brian +Pkg-URL: http://neuro.debian.net/pkgs/%(Pkg-Name)s.html +Language: Python, C +Published-Authors: Goodman D.F. and Brette R. +Published-Title: Brian: a simulator for spiking neural networks in Python +Published-In: Front. Neuroinform +Published-Year: 2008 +Published-DOI: 10.3389/neuro.11.005.2008 + + ; May be some previous entry should be removed, thus say so +Remove: python-brian-doc + + ;Tasks: debian-med/imaging-dev + ;Why: Allows interactive development/scripting + + ; ; It should be possible to switch between formats, + ; ; e.g. if some component is not yet in Debian + ;Format: extended + ; + ; ; Now some bogus one but with customizations + ;Tasks: debian-med/documentation + ;Recommends: python-brian-doc + ;Language: + ;Remark: some remark + ; """ -import re, os, sys +import re, os, sys, tempfile, glob from os.path import join, exists, expanduser, dirname, basename from ConfigParser import ConfigParser @@ -57,32 +123,49 @@ from optparse import OptionParser, Option from copy import deepcopy #from debian_bundle import deb822 from debian import deb822 +#import deb822 from debian.changelog import Changelog +# all files we are dealing with should be UTF8, thus +# lets override +import codecs + +def open(f, *args): + return codecs.open(f, *args, encoding='utf-8') + __author__ = 'Yaroslav Halchenko' __prog__ = os.path.basename(sys.argv[0]) -__version__ = '0.0.1' +__version__ = '0.0.7' __copyright__ = 'Copyright (c) 2010 Yaroslav Halchenko' __license__ = 'GPL' # What fields initiate new package description -PKG_FIELDS = ('depends', 'recommends', 'suggests', 'ignore', 'removed') +PKG_FIELDS = ('depends', 'recommends', 'suggests', 'ignore', 'remove') # We might need to resort to assure some what a canonical order -FIELDS_ORDER = ('depends', 'recommends', 'suggests', 'ignore', - 'homepage', 'language', 'wnpp', 'responsible', 'license', +# Prefixes for "standard" blends/tasks fields. Others do not get embedded +# into tasks files +BLENDS_FIELDS_PREFIXES = ('depends', 'recommends', 'suggests', 'ignore', + 'why', 'homepage', 'language', 'wnpp', 'responsible', 'license', 'vcs-', 'pkg-url', 'pkg-description', 'published-', 'x-', 'registration', 'remark') +# Additional fields which might come useful (e.g. for filing wnpp bugs) +# but are not "standard" thus should be in the trailer +CUSTOM_FIELDS_PREFIXES = ('author', 'pkg-name', 'pkg-source', + 'version', 'remove') +# Other fields should cause Error for consistency + +FIELDS_ORDER = BLENDS_FIELDS_PREFIXES + CUSTOM_FIELDS_PREFIXES verbosity = None -def error(msg, exit_code): +def error(msg, exit_code=1): sys.stderr.write(msg + '\n') sys.exit(exit_code) def verbose(level, msg): if level <= verbosity: - print " "*level, msg + sys.stderr.write(" "*level + msg + '\n') def parse_debian_blends(f='debian/blends'): @@ -95,6 +178,7 @@ def parse_debian_blends(f='debian/blends'): for p in deb822.Deb822.iter_paragraphs(open(f)): items += p.items() + verbose(6, "Got items %s" % items) # Traverse and collect things format_ = 'plain' format_clean = False # do not propagate fields into a new pkg if True @@ -102,8 +186,28 @@ def parse_debian_blends(f='debian/blends'): pkgs = [] tasks = [] + def new_pkg(prev_pkg, bname, sname, tasks): + """Helper function to create a new package + """ + if format_clean or prev_pkg is None: + pkg = deb822.Deb822() + else: + pkg = deepcopy(prev_pkg) + for k_ in PKG_FIELDS: # prune older depends + pkg.pop(k_, None) + pkg['Pkg-Name'] = pkg[k] = bname.lower() + if sname is not None: + sname = sname.lower() + pkg['Pkg-Source'] = sname + pkgs.append(pkg) + pkg.tasks = dict( (t.strip(), deb822.Deb822Dict()) for t in tasks ) + pkg.format = format_ + return pkg + for k, v in items: + kl = k.lower() + if kl == 'source': source = v.strip() elif kl == 'format': @@ -112,25 +216,25 @@ def parse_debian_blends(f='debian/blends'): if format_clean: format_ = format_[:-6] elif kl == 'tasks': - tasks = v.split(',') - newtasks = True # either we need to provide tune-ups + tasks = [x.strip() for x in v.split(',')] + newtasks = pkg is not None # either we need to provide tune-ups # for current package elif kl in PKG_FIELDS: # new package - if format_clean or pkg is None: - pkg = deb822.Deb822() - else: - pkg = deepcopy(pkg) - for k_ in PKG_FIELDS: # prune older depends - pkg.pop(k_, None) - pkg['Pkg-Name'] = pkg[k] = v - if source is None: + if source is None and not format_ in ['extended']: source = v - pkg['Pkg-Source'] = source - pkgs.append(pkg) - pkg.tasks = dict( (t.strip(), deb822.OrderedSet()) for t in tasks ) - pkg.format = format_ + pkg = new_pkg(pkg, v, source, tasks) newtasks = False else: + if pkg is None: + # So we had just source? + if source is None: + error("No package or source is known where to add %s" % (k,), 1) + # TODO: just deduce source from DebianMaterials + pkg = new_pkg(pkg, source, source, tasks) + # Since only source is available, it should be only Suggest:-ed + pkg['Suggests'] = source.lower() + newtasks = False + if newtasks: # Add customization for t in tasks: @@ -140,6 +244,7 @@ def parse_debian_blends(f='debian/blends'): else: # just store the key in the pkg itself pkg[k] = v + return pkgs @@ -147,7 +252,9 @@ def expand_pkgs(pkgs, topdir='.'): """In-place modification of pkgs taking if necessary additional information from Debian materials, and pruning empty fields """ + verbose(4, "Expanding content for %d packages" % len(pkgs)) debianm = None + # Expand packages which format is extended for pkg in pkgs: if pkg.format == 'extended': @@ -156,10 +263,12 @@ def expand_pkgs(pkgs, topdir='.'): debianm = DebianMaterials(topdir) for k, m in (('License', lambda: debianm.get_license(pkg['Pkg-Name'])), ('WNPP', debianm.get_wnpp), - ('Pkg-description', - lambda: debianm.binaries[pkg['Pkg-Name']]['Description']), + ('Pkg-Description', + lambda: debianm.get_description(pkg['Pkg-Name'])), ('Responsible', debianm.get_responsible), - ('Homepage', lambda: debianm.source.get('Homepage', None))): + ('Homepage', lambda: debianm.source.get('Homepage', None)), + ('Pkg-source', lambda: debianm.source.get('Source', None)), + ): if pkg.get(k, None): continue v = m() @@ -168,7 +277,24 @@ def expand_pkgs(pkgs, topdir='.'): # VCS fields pkg.update(debianm.get_vcsfields()) -def key_prefix_compare(x, y, order, strict=False, case=False): + +def prefix_index(x, entries, strict=True, case=False, default=10000): + """Returns an index for the x in entries + """ + if not case: + x = x.lower() + for i, v in enumerate(entries): + if x.startswith(v): + return i + + if strict: + raise IndexError( + "Could not find location for %s as specified by %s" % + (x, entries)) + return default + + +def key_prefix_compare(x, y, order, strict=True, case=False): """Little helper to help with sorting Sorts according to the order of string prefixes as given by @@ -179,22 +305,8 @@ def key_prefix_compare(x, y, order, strict=False, case=False): if not case: order = [v.lower() for v in order] - def prefix_index(t, order, strict=True, case=False): - x = t[0] - if not case: - x = x.lower() - for i, v in enumerate(order): - if x.startswith(v): - return i - - if strict: - raise IndexError( - "Could not find location for %s as specified by %s" % - (x, order)) - return 10000 # some large number ;) - - cmp_res = cmp(prefix_index(x, order, strict, case), - prefix_index(y, order, strict, case)) + cmp_res = cmp(prefix_index(x[0], order, strict, case), + prefix_index(y[0], order, strict, case)) if not cmp_res: # still unknown return cmp(x, y) return cmp_res @@ -230,10 +342,16 @@ def group_packages_into_tasks(pkgs): # Move Pkg-source/name into attributes pkg__.source = pkg__.pop('Pkg-Source') - pkg__.name = pkg__.pop('Pkg-name') + pkg__.name = pkg__.pop('Pkg-Name') + # Store the action taken on the package for later on actions + for f in PKG_FIELDS: + if f in pkg__: + pkg__.action = f + break tasks[task] = tasks.get(task, []) + [pkg__] - + verbose(4, "Grouped %d packages into %d tasks: %s" % + (len(pkgs), len(tasks), ', '.join(tasks.keys()))) return tasks def inject_tasks(tasks, config): @@ -241,23 +359,36 @@ def inject_tasks(tasks, config): for task, pkgs in tasks.iteritems(): verbose(2, "Task %s with %d packages" % (task, len(pkgs))) blend, puretask = task.split('/') - taskfile = join(config.get(blend, 'path'), 'tasks', puretask) + taskfile = expanduser(join(config.get(blend, 'path'), 'tasks', puretask)) # Load the file stats = dict(Added=[], Modified=[]) for pkg in pkgs: msgs = {'Name': pkg.name.strip(), 'Action': None} + + # Create a copy of the pkg with only valid tasks + # fields: + # TODO: make it configurable? + pkg = deepcopy(pkg) + for k in pkg: + if prefix_index(k, BLENDS_FIELDS_PREFIXES, + strict=False, default=None) is None: + pkg.pop(k) # remove it from becoming present in + # the taskfile + # Find either it is known to the task file already # Load entirely so we could simply manipulate entries = open(taskfile).readlines() known = False # We need to search by name and by source - # We need to search for every possible type of dependecy - regexp = re.compile('^ *(%s) *: *(%s) *$' % - ('|'.join(PKG_FIELDS), - '|'.join((pkg.name, pkg.source))), - re.I) + # We need to search for every possible type of dependency + regexp_str = '^ *(%s) *: *(%s) *$' \ + % ('|'.join(PKG_FIELDS), + '|'.join((pkg.name, pkg.source)).replace('+', '\+')) + verbose(4, "Searching for presence in %s using regexp: '%s'" + % (taskfile, regexp_str)) + regexp = re.compile(regexp_str, re.I) for istart, e in enumerate(entries): if regexp.search(e): verbose(4, "Found %s in position %i: %s" % @@ -265,9 +396,11 @@ def inject_tasks(tasks, config): known = True break - descr = ' ; Added by %s %s. Modified manually: False\n' % \ + descr = ' ; Added by %s %s. [Please note here if modified manually]\n' % \ (__prog__, __version__) - # Replace existing entry + + entry = pkg.dump() + # Replace existing entry? if known: # TODO: Check if previous copy does not have our preceding comment # Find the previous end @@ -280,31 +413,53 @@ def inject_tasks(tasks, config): # Lets not change file without necessity, if entry is identical -- # do nothing - entry = pkg.dump() old_entry = entries[istart:istart+icount] if u''.join(old_entry) == entry: - pass + # no changes -- just go to the next one + continue else: # Rewrite the entry if __prog__ in entries[istart-1]: istart -= 1 icount += 2 - if not 'Removed' in pkg.keys(): - entries = entries[:istart] + [descr + entry] + entries[istart+icount:] + if 'remove' != pkg.action: + entry = descr + entry msgs['Action'] = 'Changed' else: while entries[istart-1].strip() == '': istart -=1 icount +=2 - entries = entries[:istart] + entries[istart+icount:] + entry = '' msgs['Action'] = 'Removed' - open(taskfile, 'w').write(''.join(entries)) - elif not 'removed' in pkg: # or Append one + entries_prior = entries[:istart] + entries_post = entries[istart+icount:] + elif not 'remove' == pkg.action: # or Append one msgs['Action'] = 'Added' + entries_prior = entries + entry = descr + entry + entries_post = [] # could be as simple as - open(taskfile, 'a').write('\n%s%s' % (descr, pkg.dump(),)) + # Lets do 'in full' for consistent handling of empty lines + # around + #output = '\n%s%s' % (descr, pkg.dump(),) + #open(taskfile, 'a').write(output) if msgs['Action']: + # Prepare for dumping + # Prune spaces before + while len(entries_prior) and entries_prior[-1].strip() == '': + entries_prior = entries_prior[:-1] + if len(entries_prior) and not entries_prior[-1].endswith('\n'): + entries_prior[-1] += '\n' # assure present trailing newline + # Prune spaces after + while len(entries_post) and entries_post[0].strip() == '': + entries_post = entries_post[1:] + if len(entries_post) and len(entry): + # only then trailing empty line + entry += '\n' + output = ''.join(entries_prior + [ '\n' + entry ] + entries_post) + open(taskfile, 'w').write(output) # then only overwrite + verbose(3, "%(Action)s %(Name)s" % msgs) @@ -348,7 +503,9 @@ class DebianMaterials(object): if v.get('Source', None): self._source = v else: - self._binaries[v['Package']] = v + # Since it might be hash-commented out + if 'Package' in v: + self._binaries[v['Package']] = v def get_license(self, package=None, first_only=True): """Return a license(s). Parsed out from debian/copyright if it is @@ -401,6 +558,72 @@ class DebianMaterials(object): vcs[f] = v return vcs + def get_description(self, pkg_name): + """Some logic to extract description. + + If binary package matching pkg_name is found -- gets it description. + If no binary package with such name, and name matches source name, + obtain description of the first binary package. + """ + if pkg_name in self.binaries: + pass + elif pkg_name.lower() == self.source['Source'].lower(): + pkg_name = self.binaries.keys()[0] + else: + error("Name %s does not match any binary, nor source package in %s" + % (pkg_name, self)) + return self.binaries[pkg_name]['Description'] + +def print_wnpp(pkgs, config, wnpp_type="ITP"): + """Little helper to spit out formatted entry for WNPP bugreport + + TODO: It would puke atm if any field is missing + """ + + pkg = pkgs[0] # everything is based on the 1st one + opts = dict(pkg.items()) + opts['WNPP-Type'] = wnpp_type.upper() + opts['Pkg-Description-Short'] = re.sub('\n.*', '', pkg['Pkg-Description']) + + subject = "%(WNPP-Type)s: %(Pkg-Name)s -- %(Pkg-Description-Short)s" % opts + body = """*** Please type your report below this line *** + +* Package name : %(Pkg-Name)s + Version : %(Version)s + Upstream Author : %(Author)s +* URL : %(Homepage)s +* License : %(License)s + Programming Lang: %(Language)s + Description : %(Pkg-Description)s + +""" % opts + + # Unfortunately could not figure out how to set the owner, so I will just print it out + if False: + tmpfile = tempfile.NamedTemporaryFile() + tmpfile.write(body) + tmpfile.flush() + cmd = "reportbug -b --paranoid --subject='%s' --severity=wishlist --body-file='%s' -o /tmp/o.txt wnpp" \ + % (subject, tmpfile.name) + verbose(2, "Running %s" %cmd) + os.system(cmd) + else: + print "Subject: %s\n\n%s" % (subject, body) + + +def is_template(p): + """Helper to return true if pkg definition looks like a template + and should not be processed + """ + # We might want to skip some which define a skeleton + # (no source/homepage/etc although fields are there) + for f in ['vcs-browser', 'pkg-url', 'pkg-description', + 'published-Title', 'pkg-name', 'homepage', + 'author']: + if f in p and p[f] != "": + return False + return True + def main(): @@ -424,38 +647,75 @@ def main(): Option("-v", "--verbosity", action="store", type="int", dest="verbosity", default=1, help="Noise level.")) + # We might like to create a separate 'group' of options for commands + p.add_option( + Option("-w", action="store_true", + dest="wnpp", default=False, + help="Operate in WNPP mode: dumps cut-paste-able entry for WNPP bugreport")) + + p.add_option( + Option("--wnpp", action="store", + dest="wnpp_mode", default=None, + help="Operate in WNPP mode: dumps cut-paste-able entry for WNPP bugreport")) + + p.add_option( + Option("-a", action="store_true", + dest="all_mode", default=False, + help="Process all files listed in paths.all")) + + (options, infiles) = p.parse_args() global verbosity; verbosity = options.verbosity - if not len(infiles): - infiles = [join(options.topdir or './', 'debian/blends')] # default one + if options.wnpp and options.wnpp_mode is None: + options.wnpp_mode = 'ITP' # Load configuration - config = ConfigParser() + config = ConfigParser(defaults={'skip': '.*[~#]$'}) config.read(options.config_file) + if options.all_mode: + if len(infiles): + raise ValueError("Do not specify any files in -a mode. Use configuration file, section paths, option all") + globs = config.get('paths', 'all', None).split() + infiles = reduce(list.__add__, (glob.glob(expanduser(f)) for f in globs)) + verbose(1, "Found %d files in specified paths" % len(infiles)) + + if not len(infiles): + infiles = [join(options.topdir or './', 'debian/blends')] # default one + + skip_re = re.compile(config.get('paths', 'skip', None)) + for blends_file in infiles: verbose(1, "Processing %s" % blends_file) if not exists(blends_file): error("Cannot find a file %s. Either provide a file or specify top " "debian directory with -d." % blends_file, 1) + if skip_re.match(blends_file): + verbose(2, "W: Skipped since matches paths.skip regexp") + continue pkgs = parse_debian_blends(blends_file) if options.topdir is None: if dirname(blends_file).endswith('/debian'): topdir = dirname(dirname(blends_file)) else: - topdir = '.' # and hope for the best + topdir = '.' # and hope for the best ;) else: topdir = options.topdir - expand_pkgs(pkgs, topdir=topdir) - tasks = group_packages_into_tasks(pkgs) - inject_tasks(tasks, config) - - #for t,v in tasks.iteritems(): - # print "-------TASK: ", t - # print ''.join(str(t_) for t_ in v) - #print pkgs[0] - #print tasks['debian-med/documentation'][0] + + expand_pkgs(pkgs, topdir=topdir) + + pkgs = [p for p in pkgs if not is_template(p)] + if not len(pkgs): + verbose(2, "W: Skipping since seems to contain templates only") + continue + if options.wnpp_mode is not None: + print_wnpp(pkgs, config, options.wnpp_mode) + else: + # by default -- operate on blends/tasks files + tasks = group_packages_into_tasks(pkgs) + inject_tasks(tasks, config) + if __name__ == '__main__': main()