2 #emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
3 #ex: set sts=4 ts=4 sw=4 noet:
4 """Script to help maintaining package definitions within Debian blends
7 Often it becomes necessary to duplicate the same information about a
8 package within debian packaging and multiple blends task files,
9 possibly of different blends. This script allows to automate:
11 - construction of entries
12 - injection into task files
13 - modification of existing entries if things changed
14 - removal of previously injected entries
19 * For every package the same task file might be re-read/written (if
20 entry changed/added) from disk.
21 That allows to replace easily original entry for 'source' package
22 (listed as Suggests:) with actual first listed binary package.
23 This should be taken into consideration if current per-package
32 Paths to the blends top directories, containing tasks directories are
33 specified in ~/.blends-inject.cfg file, e.g.::
36 path = /home/yoh/deb/debian-med/
39 path = /home/yoh/deb/debian-science/
41 Definition of the fields for task files by default are looked up
42 within debian/blends, or files provided in the command line. Also for "-a"
43 mode of operation you should define list of globs to match your debian/blends
47 all=/home/yoh/deb/gits/pkg-exppsy/neurodebian/future/blends/*
48 /home/yoh/deb/gits/*/debian/blends
49 /home/yoh/deb/gits/pkg-exppsy/*/debian/blends
52 Format of debian/blends
53 -----------------------
57 ; If originally filed using project source name, and it is different
58 ; from the primary (first) binary package name, keep 'Source' to be
59 ; able to adopt previously included tasks entry
62 ; Define the format on how entries should be handled.
64 ; extended -- whenever package is not in Debian and additional
65 ; fields should be obtained from debian/*:
72 ; plain [default] -- only fields listed here should be mentioned.
73 ; Common use -- whenever package is already known to UDD.
75 ; By default, all fields specified previously propagate into following
76 ; packages as well. If that is not desired, add suffix '-clean' to
80 Tasks: debian-science/neuroscience-modeling
82 ; Could have Depends/Recommends/Suggests and Ignore
83 ; All those define Pkg-Name field which is not included
84 ; in the final "rendering" but is available as Pkg-Name item
86 Pkg-URL: http://neuro.debian.net/pkgs/%(Pkg-Name)s.html
88 Published-Authors: Goodman D.F. and Brette R.
89 Published-Title: Brian: a simulator for spiking neural networks in Python
90 Published-In: Front. Neuroinform
92 Published-DOI: 10.3389/neuro.11.005.2008
94 ; May be some previous entry should be removed, thus say so
95 Remove: python-brian-doc
97 ;Tasks: debian-med/imaging-dev
98 ;Why: Allows interactive development/scripting
100 ; ; It should be possible to switch between formats,
101 ; ; e.g. if some component is not yet in Debian
104 ; ; Now some bogus one but with customizations
105 ;Tasks: debian-med/documentation
106 ;Recommends: python-brian-doc
114 import re, os, sys, tempfile, glob
115 from os.path import join, exists, expanduser, dirname, basename
117 from ConfigParser import ConfigParser
118 from optparse import OptionParser, Option
120 from copy import deepcopy
121 #from debian_bundle import deb822
122 from debian import deb822
124 from debian.changelog import Changelog
126 # all files we are dealing with should be UTF8, thus
131 return codecs.open(f, *args, encoding='utf-8')
133 __author__ = 'Yaroslav Halchenko'
134 __prog__ = os.path.basename(sys.argv[0])
135 __version__ = '0.0.5'
136 __copyright__ = 'Copyright (c) 2010 Yaroslav Halchenko'
139 # What fields initiate new package description
140 PKG_FIELDS = ('depends', 'recommends', 'suggests', 'ignore', 'remove')
142 # We might need to resort to assure some what a canonical order
143 # Prefixes for "standard" blends/tasks fields. Others do not get embedded
145 BLENDS_FIELDS_PREFIXES = ('depends', 'recommends', 'suggests', 'ignore',
146 'why', 'homepage', 'language', 'wnpp', 'responsible', 'license',
147 'vcs-', 'pkg-url', 'pkg-description',
148 'published-', 'x-', 'registration', 'remark')
149 # Additional fields which might come useful (e.g. for filing wnpp bugs)
150 # but are not "standard" thus should be in the trailer
151 CUSTOM_FIELDS_PREFIXES = ('author', 'pkg-name', 'pkg-source',
153 # Other fields should cause Error for consistency
155 FIELDS_ORDER = BLENDS_FIELDS_PREFIXES + CUSTOM_FIELDS_PREFIXES
159 def error(msg, exit_code=1):
160 sys.stderr.write(msg + '\n')
163 def verbose(level, msg):
164 if level <= verbosity:
165 sys.stderr.write(" "*level + msg + '\n')
168 def parse_debian_blends(f='debian/blends'):
169 """Parses debian/blends file
171 Returns unprocessed list of customized Deb822 entries
173 # Linearize all the paragraphs since we are not using them
175 for p in deb822.Deb822.iter_paragraphs(open(f)):
178 verbose(6, "Got items %s" % items)
179 # Traverse and collect things
181 format_clean = False # do not propagate fields into a new pkg if True
182 pkg, source = None, None
186 def new_pkg(prev_pkg, bname, sname, tasks):
187 """Helper function to create a new package
189 if format_clean or prev_pkg is None:
190 pkg = deb822.Deb822()
192 pkg = deepcopy(prev_pkg)
193 for k_ in PKG_FIELDS: # prune older depends
195 pkg['Pkg-Name'] = pkg[k] = bname.lower()
196 pkg['Pkg-Source'] = sname.lower()
198 pkg.tasks = dict( (t.strip(), deb822.Deb822Dict()) for t in tasks )
210 format_clean = format_.endswith('-clean')
212 format_ = format_[:-6]
215 newtasks = pkg is not None # either we need to provide tune-ups
216 # for current package
217 elif kl in PKG_FIELDS: # new package
220 pkg = new_pkg(pkg, v, source, tasks)
224 # So we had just source?
226 error("No package or source is known where to add %s" % (k,), 1)
227 # TODO: just deduce source from DebianMaterials
228 pkg = new_pkg(pkg, source, source, tasks)
229 # Since only source is available, it should be only Suggest:-ed
230 pkg['Suggests'] = source.lower()
236 if not t in pkg.tasks:
237 pkg.tasks[t] = deb822.Deb822Dict()
240 # just store the key in the pkg itself
246 def expand_pkgs(pkgs, topdir='.'):
247 """In-place modification of pkgs taking if necessary additional
248 information from Debian materials, and pruning empty fields
250 verbose(4, "Expanding content for %d packages" % len(pkgs))
253 # Expand packages which format is extended
255 if pkg.format == 'extended':
256 # expanding, for that we need debian/control
258 debianm = DebianMaterials(topdir)
259 for k, m in (('License', lambda: debianm.get_license(pkg['Pkg-Name'])),
260 ('WNPP', debianm.get_wnpp),
262 lambda: debianm.get_description(pkg['Pkg-Name'])),
263 ('Responsible', debianm.get_responsible),
264 ('Homepage', lambda: debianm.source.get('Homepage', None))):
271 pkg.update(debianm.get_vcsfields())
274 def prefix_index(x, entries, strict=True, case=False, default=10000):
275 """Returns an index for the x in entries
279 for i, v in enumerate(entries):
285 "Could not find location for %s as specified by %s" %
290 def key_prefix_compare(x, y, order, strict=True, case=False):
291 """Little helper to help with sorting
293 Sorts according to the order of string prefixes as given by
294 `order`. If `strict`, then if no matching prefix found, would
295 raise KeyError; otherwise provides least priority to those keys
296 which were not found in `order`
299 order = [v.lower() for v in order]
301 cmp_res = cmp(prefix_index(x[0], order, strict, case),
302 prefix_index(y[0], order, strict, case))
303 if not cmp_res: # still unknown
308 def group_packages_into_tasks(pkgs):
309 """Given a list of packages (with .tasks) group them per each
310 task and perform necessary customizations stored in .tasks
312 # Time to take care about packages and tasks
313 # Unroll pkgs into a collection of pkgs per known task
316 # Lets just create deepcopies with tune-ups for each task
317 for itask, (task, fields) in enumerate(pkg.tasks.iteritems()):
321 # Perform string completions and removals
322 for k,v in pkg_.iteritems():
324 if v is None or not len(v.strip()):
327 # Sort the fields according to FIELDS_ORDER. Unfortunately
328 # silly Deb822* cannot create from list of tuples, so will do
330 pkg__ = deb822.Deb822()
331 for k,v in sorted(pkg_.items(),
333 key_prefix_compare(x, y, order=FIELDS_ORDER)):
336 # Move Pkg-source/name into attributes
337 pkg__.source = pkg__.pop('Pkg-Source')
338 pkg__.name = pkg__.pop('Pkg-Name')
339 # Store the action taken on the package for later on actions
345 tasks[task] = tasks.get(task, []) + [pkg__]
346 verbose(4, "Grouped %d packages into %d tasks: %s" %
347 (len(pkgs), len(tasks), ', '.join(tasks.keys())))
350 def inject_tasks(tasks, config):
351 # Now go through task files and replace/add entries
352 for task, pkgs in tasks.iteritems():
353 verbose(2, "Task %s with %d packages" % (task, len(pkgs)))
354 blend, puretask = task.split('/')
355 taskfile = join(config.get(blend, 'path'), 'tasks', puretask)
358 stats = dict(Added=[], Modified=[])
360 msgs = {'Name': pkg.name.strip(), 'Action': None}
362 # Create a copy of the pkg with only valid tasks
364 # TODO: make it configurable?
367 if prefix_index(k, BLENDS_FIELDS_PREFIXES,
368 strict=False, default=None) is None:
369 pkg.pop(k) # remove it from becoming present in
372 # Find either it is known to the task file already
374 # Load entirely so we could simply manipulate
375 entries = open(taskfile).readlines()
377 # We need to search by name and by source
378 # We need to search for every possible type of dependecy
379 regexp = re.compile('^ *(%s) *: *(%s) *$' %
380 ('|'.join(PKG_FIELDS),
381 '|'.join((pkg.name, pkg.source))),
383 for istart, e in enumerate(entries):
385 verbose(4, "Found %s in position %i: %s" %
386 (pkg.name, istart, e.rstrip()))
390 descr = ' ; Added by %s %s. [Please note here if modified manually]\n' % \
391 (__prog__, __version__)
394 # Replace existing entry?
396 # TODO: Check if previous copy does not have our preceding comment
397 # Find the previous end
400 while entries[istart+icount].strip() != '':
402 except IndexError, e:
403 pass # if we go beyond
405 # Lets not change file without necessity, if entry is identical --
407 old_entry = entries[istart:istart+icount]
409 if u''.join(old_entry) == entry:
410 # no changes -- just go to the next one
412 else: # Rewrite the entry
413 if __prog__ in entries[istart-1]:
416 if 'remove' != pkg.action:
417 entry = descr + entry
418 msgs['Action'] = 'Changed'
420 while entries[istart-1].strip() == '':
424 msgs['Action'] = 'Removed'
425 entries_prior = entries[:istart]
426 entries_post = entries[istart+icount:]
427 elif not 'remove' == pkg.action: # or Append one
428 msgs['Action'] = 'Added'
429 entries_prior = entries
430 entry = descr + entry
432 # could be as simple as
433 # Lets do 'in full' for consistent handling of empty lines
435 #output = '\n%s%s' % (descr, pkg.dump(),)
436 #open(taskfile, 'a').write(output)
439 # Prepare for dumping
440 # Prune spaces before
441 while len(entries_prior) and entries_prior[-1].strip() == '':
442 entries_prior = entries_prior[:-1]
443 if len(entries_prior) and not entries_prior[-1].endswith('\n'):
444 entries_prior[-1] += '\n' # assure present trailing newline
446 while len(entries_post) and entries_post[0].strip() == '':
447 entries_post = entries_post[1:]
448 if len(entries_post) and len(entry):
449 # only then trailing empty line
451 output = ''.join(entries_prior + [ '\n' + entry ] + entries_post)
452 open(taskfile, 'w').write(output) # then only overwrite
454 verbose(3, "%(Action)s %(Name)s" % msgs)
457 class DebianMaterials(object):
458 """Extract selected information from an existing debian/
460 _WNPP_RE = re.compile('^ *\* *Initial release.*closes:? #(?P<bug>[0-9]*).*', re.I)
462 def __init__(self, topdir):
463 #self.topdir = topdir
464 self._debiandir = join(topdir, 'debian')
466 self._binaries = None
470 if self._source is None:
471 self._assign_packages()
476 if self._binaries is None:
477 self._assign_packages()
478 return self._binaries
480 def fpath(self, name):
481 return join(self._debiandir, name)
483 def _assign_packages(self):
485 control = deb822.Deb822.iter_paragraphs(
486 open(self.fpath('control')))
489 "Cannot parse %s file necessary for the %s package entry. Error: %s"
490 % (control_file, pkg['Pkg-Name'], str(e)))
494 if v.get('Source', None):
497 self._binaries[v['Package']] = v
499 def get_license(self, package=None, first_only=True):
500 """Return a license(s). Parsed out from debian/copyright if it is
501 in machine readable format
504 # may be package should carry custom copyright file
505 copyright_file_ = self.fpath('%s.copyright' % package)
506 if package and exists(copyright_file_):
507 copyright_file = copyright_file_
509 copyright_file = self.fpath('copyright')
512 for p in deb822.Deb822.iter_paragraphs(open(copyright_file)):
513 if not 'Files' in p or p['Files'].strip().startswith('debian/'):
516 # Take only the short version of first line
517 l = re.sub('\n.*', '', l).strip()
520 if not l in licenses:
527 return ', '.join(licenses)
530 """Search for a template changelog entry closing "Initial bug
532 for l in open(self.fpath('changelog')):
533 rr = self._WNPP_RE.match(l)
535 return rr.groupdict()['bug']
538 def get_responsible(self):
539 """Returns responsible, atm -- maintainer
541 return self.source['Maintainer']
543 def get_vcsfields(self):
544 vcs = deb822.Deb822()
545 for f,v in self._source.iteritems():
546 if f.lower().startswith('vcs-'):
550 def get_description(self, pkg_name):
551 """Some logic to extract description.
553 If binary package matching pkg_name is found -- gets it description.
554 If no binary package with such name, and name matches source name,
555 obtain description of the first binary package.
557 if pkg_name in self.binaries:
559 elif pkg_name.lower() == self.source['Source'].lower():
560 pkg_name = self.binaries.keys()[0]
562 error("Name %s does not match any binary, nor source package in %s"
564 return self.binaries[pkg_name]['Description']
566 def print_wnpp(pkgs, config, wnpp_type="ITP"):
567 """Little helper to spit out formatted entry for WNPP bugreport
569 TODO: It would puke atm if any field is missing
572 pkg = pkgs[0] # everything is based on the 1st one
573 opts = dict(pkg.items())
574 opts['WNPP-Type'] = wnpp_type.upper()
575 opts['Pkg-Description-Short'] = re.sub('\n.*', '', pkg['Pkg-Description'])
577 subject = "%(WNPP-Type)s: %(Pkg-Name)s -- %(Pkg-Description-Short)s" % opts
578 body = """*** Please type your report below this line ***
580 * Package name : %(Pkg-Name)s
581 Version : %(Version)s
582 Upstream Author : %(Author)s
584 * License : %(License)s
585 Programming Lang: %(Language)s
586 Description : %(Pkg-Description)s
590 # Unfortunately could not figure out how to set the owner, so I will just print it out
592 tmpfile = tempfile.NamedTemporaryFile()
595 cmd = "reportbug -b --paranoid --subject='%s' --severity=wishlist --body-file='%s' -o /tmp/o.txt wnpp" \
596 % (subject, tmpfile.name)
597 verbose(2, "Running %s" %cmd)
600 print "Subject: %s\n\n%s" % (subject, body)
606 usage="%s [OPTIONS] [blends_files]\n\n" % __prog__ + __doc__,
607 version="%prog " + __version__)
610 Option("-d", "--topdir", action="store",
611 dest="topdir", default=None,
612 help="Top directory of a Debian package. It is used to locate "
613 "'debian/blends' if none is specified, and where to look for "
614 "extended information."))
617 Option("-c", "--config-file", action="store",
618 dest="config_file", default=join(expanduser('~'), '.%s.cfg' % __prog__),
619 help="Noise level."))
622 Option("-v", "--verbosity", action="store", type="int",
623 dest="verbosity", default=1, help="Noise level."))
625 # We might like to create a separate 'group' of options for commands
627 Option("-w", action="store_true",
628 dest="wnpp", default=False,
629 help="Operate in WNPP mode: dumps cut-paste-able entry for WNPP bugreport"))
632 Option("--wnpp", action="store",
633 dest="wnpp_mode", default=None,
634 help="Operate in WNPP mode: dumps cut-paste-able entry for WNPP bugreport"))
637 Option("-a", action="store_true",
638 dest="all_mode", default=False,
639 help="Process all files listed in paths.all"))
642 (options, infiles) = p.parse_args()
643 global verbosity; verbosity = options.verbosity
645 if options.wnpp and options.wnpp_mode is None:
646 options.wnpp_mode = 'ITP'
649 config = ConfigParser()
650 config.read(options.config_file)
654 raise ValueError("Do not specify any files in -a mode. Use configuration file, section paths, option all")
655 globs = config.get('paths', 'all', None).split()
656 infiles = reduce(list.__add__, (glob.glob(f) for f in globs))
657 verbose(1, "Found %d files in specified paths" % len(infiles))
660 infiles = [join(options.topdir or './', 'debian/blends')] # default one
662 for blends_file in infiles:
663 verbose(1, "Processing %s" % blends_file)
664 if not exists(blends_file):
665 error("Cannot find a file %s. Either provide a file or specify top "
666 "debian directory with -d." % blends_file, 1)
668 pkgs = parse_debian_blends(blends_file)
669 if options.topdir is None:
670 if dirname(blends_file).endswith('/debian'):
671 topdir = dirname(dirname(blends_file))
673 topdir = '.' # and hope for the best ;)
675 topdir = options.topdir
677 expand_pkgs(pkgs, topdir=topdir)
678 if options.wnpp_mode is not None:
679 print_wnpp(pkgs, config, options.wnpp_mode)
681 # by default -- operate on blends/tasks files
682 tasks = group_packages_into_tasks(pkgs)
683 inject_tasks(tasks, config)
686 if __name__ == '__main__':