X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=buildscripts%2Foutput-distance.py;h=8586d2481ab9cb9beb45bcd06c4b64685eee63b7;hb=c7e9415c57c69658706dd6f6aeb744403275ba0f;hp=8f1b68633c89e0959cccd24b1f7e6184770ce9b7;hpb=2f19ea6bdfd30afec339c5695b9e2df4df755896;p=lilypond.git
diff --git a/buildscripts/output-distance.py b/buildscripts/output-distance.py
index 8f1b68633c..8586d2481a 100644
--- a/buildscripts/output-distance.py
+++ b/buildscripts/output-distance.py
@@ -2,13 +2,12 @@
import sys
import optparse
import os
+import math
## so we can call directly as buildscripts/output-distance.py
me_path = os.path.abspath (os.path.split (sys.argv[0])[0])
sys.path.insert (0, me_path + '/../python/')
-
-
-import safeeval
+sys.path.insert (0, me_path + '/../python/out/')
X_AXIS = 0
@@ -17,7 +16,45 @@ INFTY = 1e6
OUTPUT_EXPRESSION_PENALTY = 1
ORPHAN_GROB_PENALTY = 1
-inspect_max_count = 0
+options = None
+
+################################################################
+# system interface.
+temp_dir = None
+class TempDirectory:
+ def __init__ (self):
+ import tempfile
+ self.dir = tempfile.mkdtemp ()
+ print 'dir is', self.dir
+ def __del__ (self):
+ print 'rm -rf %s' % self.dir
+ os.system ('rm -rf %s' % self.dir)
+ def __call__ (self):
+ return self.dir
+
+
+def get_temp_dir ():
+ global temp_dir
+ if not temp_dir:
+ temp_dir = TempDirectory ()
+ return temp_dir ()
+
+def read_pipe (c):
+ print 'pipe' , c
+ return os.popen (c).read ()
+
+def system (c):
+ print 'system' , c
+ s = os.system (c)
+ if s :
+ raise Exception ("failed")
+ return
+
+def shorten_string (s):
+ threshold = 15
+ if len (s) > 2*threshold:
+ s = s[:threshold] + '..' + s[-threshold:]
+ return s
def max_distance (x1, x2):
dist = 0.0
@@ -28,6 +65,38 @@ def max_distance (x1, x2):
return dist
+def compare_png_images (old, new, dest_dir):
+ def png_dims (f):
+ m = re.search ('([0-9]+) x ([0-9]+)', read_pipe ('file %s' % f))
+
+ return tuple (map (int, m.groups ()))
+
+ dest = os.path.join (dest_dir, new.replace ('.png', '.compare.jpeg'))
+ try:
+ dims1 = png_dims (old)
+ dims2 = png_dims (new)
+ except AttributeError:
+ ## hmmm. what to do?
+ system ('touch %(dest)s' % locals ())
+ return
+
+ dims = (min (dims1[0], dims2[0]),
+ min (dims1[1], dims2[1]))
+
+ dir = get_temp_dir ()
+ system ('convert -depth 8 -crop %dx%d+0+0 %s %s/crop1.png' % (dims + (old, dir)))
+ system ('convert -depth 8 -crop %dx%d+0+0 %s %s/crop2.png' % (dims + (new, dir)))
+
+ system ('compare -depth 8 %(dir)s/crop1.png %(dir)s/crop2.png %(dir)s/diff.png' % locals ())
+
+ system ("convert -depth 8 %(dir)s/diff.png -blur 0x3 -negate -channel alpha,blue -type TrueColorMatte -fx 'intensity' %(dir)s/matte.png" % locals ())
+
+ system ("composite -compose atop -quality 65 %(dir)s/matte.png %(new)s %(dest)s" % locals ())
+
+
+################################################################
+# interval/bbox arithmetic.
+
empty_interval = (INFTY, -INFTY)
empty_bbox = (empty_interval, empty_interval)
@@ -268,46 +337,235 @@ def read_signature_file (name):
################################################################
# different systems of a .ly file.
-def read_pipe (c):
- print 'pipe' , c
- return os.popen (c).read ()
-def system (c):
- print 'system' , c
- s = os.system (c)
- if s :
- raise Exception ("failed")
- return
+hash_to_original_name = {}
-def compare_png_images (old, new, dir):
- def png_dims (f):
- m = re.search ('([0-9]+) x ([0-9]+)', read_pipe ('file %s' % f))
+class FileLink:
+ def __init__ (self, f1, f2):
+ self._distance = None
+ self.file_names = (f1, f2)
- return tuple (map (int, m.groups ()))
+ def text_record_string (self):
+ return '%-30f %-20s\n' % (self.distance (),
+ self.name ()
+ + os.path.splitext (self.file_names[1])[1]
+ )
+
+ def calc_distance (self):
+ return 0.0
- dims1 = png_dims (old)
- dims2 = png_dims (new)
+ def distance (self):
+ if self._distance == None:
+ self._distance = self.calc_distance ()
- dims = (min (dims1[0], dims2[0]),
- min (dims1[1], dims2[1]))
+ return self._distance
+
+ def source_file (self):
+ for ext in ('.ly', '.ly.txt'):
+ base = os.path.splitext (self.file_names[1])[0]
+ f = base + ext
+ if os.path.exists (f):
+ return f
+
+ return ''
+
+ def name (self):
+ base = os.path.basename (self.file_names[1])
+ base = os.path.splitext (base)[0]
+ base = hash_to_original_name.get (base, base)
+ base = os.path.splitext (base)[0]
+ return base
+
+ def extension (self):
+ return os.path.splitext (self.file_names[1])[1]
- system ('convert -crop %dx%d+0+0 %s crop1.png' % (dims + (old,)))
- system ('convert -crop %dx%d+0+0 %s crop2.png' % (dims + (new,)))
+ def link_files_for_html (self, dest_dir):
+ for f in self.file_names:
+ link_file (f, os.path.join (dest_dir, f))
- system ('compare crop1.png crop2.png diff.png')
+ def get_distance_details (self):
+ return ''
- system ("convert diff.png -border 2 -blur 0x3 -negate -channel alpha,blue -type TrueColorMatte -fx 'intensity' matte.png")
+ def get_cell (self, oldnew):
+ return ''
+
+ def get_file (self, oldnew):
+ return self.file_names[oldnew]
+
+ def html_record_string (self, dest_dir):
+ dist = self.distance()
+
+ details = self.get_distance_details ()
+ if details:
+ details_base = os.path.splitext (self.file_names[1])[0]
+ details_base += '.details.html'
+ fn = dest_dir + '/' + details_base
+ open_write_file (fn).write (details)
- dest = os.path.join (dir, new.replace ('.png', '.compare.png'))
- system ("composite matte.png %(new)s %(dest)s" % locals ())
+ details = '
(details)' % locals ()
-class FileLink:
- def __init__ (self):
- self.original_name = ''
- self.base_names = ('','')
- self.system_links = {}
- self._distance = None
+ cell1 = self.get_cell (0)
+ cell2 = self.get_cell (1)
+
+ name = self.name () + self.extension ()
+ file1 = self.get_file (0)
+ file2 = self.get_file (1)
+
+ return '''
%s' % str + return str + + def calc_distance (self): + if self.contents[0] == self.contents[1]: + d = 0.0 + else: + d = 1.0001 *options.threshold + + return d + + +class TextFileCompareLink (FileCompareLink): + def calc_distance (self): + import difflib + diff = difflib.unified_diff (self.contents[0].strip().split ('\n'), + self.contents[1].strip().split ('\n'), + fromfiledate = self.file_names[0], + tofiledate = self.file_names[1] + ) + + self.diff_lines = [l for l in diff] + self.diff_lines = self.diff_lines[2:] + + return math.sqrt (float (len ([l for l in self.diff_lines if l[0] in '-+']))) + + def get_cell (self, oldnew): + str = '' + if oldnew == 1: + str = '\n'.join ([d.replace ('\n','') for d in self.diff_lines]) + str = '
%s' % str + return str + +class LogFileCompareLink (TextFileCompareLink): + def get_content (self, f): + c = TextFileCompareLink.get_content (self, f) + c = re.sub ("\nProcessing `[^\n]+'\n", '', c) + return c + +class ProfileFileLink (FileCompareLink): + def __init__ (self, f1, f2): + FileCompareLink.__init__ (self, f1, f2) + self.results = [{}, {}] + + def get_cell (self, oldnew): + str = '' + for k in ('time', 'cells'): + if oldnew==0: + str += '%-8s: %d\n' % (k, int (self.results[oldnew][k])) + else: + str += '%-8s: %8d (%5.3f)\n' % (k, int (self.results[oldnew][k]), + self.get_ratio (k)) + + return '
%s' % str + + def get_ratio (self, key): + (v1,v2) = (self.results[0].get (key, -1), + self.results[1].get (key, -1)) + + if v1 <= 0 or v2 <= 0: + return 0.0 + + return (v1 - v2) / float (v1+v2) + + def calc_distance (self): + for oldnew in (0,1): + def note_info (m): + self.results[oldnew][m.group(1)] = float (m.group (2)) + + re.sub ('([a-z]+): ([-0-9.]+)\n', + note_info, self.contents[oldnew]) + + dist = 0.0 + factor = { + 'time': 0.1, + 'cells': 5.0, + } + + for k in ('time', 'cells'): + real_val = math.tan (self.get_ratio (k) * 0.5 * math.pi) + dist += math.exp (math.fabs (real_val) * factor[k]) - 1 + + dist = min (dist, 100) + return dist + + +class MidiFileLink (TextFileCompareLink): + def get_content (self, oldnew): + import midi + data = FileCompareLink.get_content (self, oldnew) + midi = midi.parse (data) + tracks = midi[1] + + str = '' + j = 0 + for t in tracks: + str += 'track %d' % j + j += 1 + + for e in t: + ev_str = repr (e) + if re.search ('LilyPond [0-9.]+', ev_str): + continue + + str += ' ev %s\n' % `e` + return str + + + +class SignatureFileLink (FileLink): + def __init__ (self, f1, f2 ): + FileLink.__init__ (self, f1, f2) + self.system_links = {} + def add_system_link (self, link, number): self.system_links[number] = link @@ -321,22 +579,6 @@ class FileLink: return d + orphan_distance - def distance (self): - if type (self._distance) != type (0.0): - return self.calc_distance () - - return self._distance - - def text_record_string (self): - return '%-30f %-20s\n' % (self.distance (), - self.original_name) - - def source_file (self): - for ext in ('.ly', '.ly.txt'): - if os.path.exists (self.base_names[1] + ext): - return self.base_names[1] + ext - return '' - def add_file_compare (self, f1, f2): system_index = [] @@ -350,21 +592,6 @@ class FileLink: self.base_names = (os.path.normpath (base1), os.path.normpath (base2)) - def note_original (match): - self.original_name = match.group (1) - return '' - - if not self.original_name: - self.original_name = os.path.split (base1)[1] - - ## ugh: drop the .ly.txt - for ext in ('.ly', '.ly.txt'): - try: - re.sub (r'\\sourcefilename "([^"]+)"', - note_original, open (base1 + ext).read ()) - except IOError: - pass - s1 = read_signature_file (f1) s2 = read_signature_file (f2) @@ -372,22 +599,58 @@ class FileLink: self.add_system_link (link, system_index[0]) - def link_files_for_html (self, old_dir, new_dir, dest_dir): - png_linked = [[], []] - for ext in ('.png', '.ly', '-page*png'): - + + def create_images (self, dest_dir): + + files_created = [[], []] + for oldnew in (0, 1): + pat = self.base_names[oldnew] + '.eps' + + for f in glob.glob (pat): + infile = f + outfile = (dest_dir + '/' + f).replace ('.eps', '.png') + data_option = '' + if options.local_data_dir: + data_option = ('-slilypond-datadir=%s/../share/lilypond/current ' + % os.path.dirname(infile)) + + mkdir (os.path.split (outfile)[0]) + cmd = ('gs -sDEVICE=png16m -dGraphicsAlphaBits=4 -dTextAlphaBits=4 ' + ' %(data_option)s ' + ' -r101 ' + ' -sOutputFile=%(outfile)s -dNOSAFER -dEPSCrop -q -dNOPAUSE ' + ' %(infile)s -c quit ') % locals () + + files_created[oldnew].append (outfile) + system (cmd) + + return files_created + + def link_files_for_html (self, dest_dir): + FileLink.link_files_for_html (self, dest_dir) + to_compare = [[], []] + + exts = [] + if options.create_images: + to_compare = self.create_images (dest_dir) + else: + exts += ['.png', '-page*png'] + + for ext in exts: for oldnew in (0,1): for f in glob.glob (self.base_names[oldnew] + ext): dst = dest_dir + '/' + f link_file (f, dst) if f.endswith ('.png'): - png_linked[oldnew].append (f) + to_compare[oldnew].append (f) + + if options.compare_images: + for (old, new) in zip (to_compare[0], to_compare[1]): + compare_png_images (old, new, dest_dir) - for (old,new) in zip (png_linked[0], png_linked[1]): - compare_png_images (old, new, dest_dir) - def html_record_string (self, old_dir, new_dir): + def get_cell (self, oldnew): def img_cell (ly, img, name): if not name: name = 'source' @@ -395,15 +658,10 @@ class FileLink: name = '%s' % name return ''' -
distance | -%(dir1)s | -%(dir2)s | +%(short_dir1)s | +%(short_dir2)s |
---|
') - below_count =len ([1 for s,l in results - if threshold >= s > 0.0]) + below_count = len (below) if below_count: html += ('
%d below threshold
' % below_count) - - html += ('%d unchanged
' - % len ([1 for (s,l) in results if s == 0.0])) - + + html += ('%d unchanged
' % len (unchanged)) dest_file = dest_dir + '/index.html' open_write_file (dest_file).write (html) + + + for link in changed: + link.link_files_for_html (dest_dir) + def print_results (self, threshold): self.write_text_result_page ('', threshold) def compare_trees (dir1, dir2, dest_dir, threshold): data = ComparisonData () data.compare_trees (dir1, dir2) + data.read_sources () + + data.print_results (threshold) if os.path.isdir (dest_dir): system ('rm -rf %s '% dest_dir) + data.write_changed (dest_dir, threshold) data.create_html_result_page (dir1, dir2, dest_dir, threshold) data.create_text_result_page (dir1, dir2, dest_dir, threshold) @@ -693,6 +990,7 @@ def mkdir (x): def link_file (x, y): mkdir (os.path.split (y)[0]) try: + print x, '->', y os.link (x, y) except OSError, z: print 'OSError', x, y, z @@ -719,14 +1017,17 @@ def test_paired_files (): def test_compare_trees (): system ('rm -rf dir1 dir2') system ('mkdir dir1 dir2') - system ('cp 20{-*.signature,.ly,.png} dir1') - system ('cp 20{-*.signature,.ly,.png} dir2') - system ('cp 20expr{-*.signature,.ly,.png} dir1') - system ('cp 19{-*.signature,.ly,.png} dir2/') - system ('cp 19{-*.signature,.ly,.png} dir1/') - system ('cp 19-1.signature 19-sub-1.signature') - system ('cp 19.ly 19-sub.ly') - system ('cp 19.png 19-sub.png') + system ('cp 20{-*.signature,.ly,.png,.eps,.log,.profile} dir1') + system ('cp 20{-*.signature,.ly,.png,.eps,.log,.profile} dir2') + system ('cp 20expr{-*.signature,.ly,.png,.eps,.log,.profile} dir1') + system ('cp 19{-*.signature,.ly,.png,.eps,.log,.profile} dir2/') + system ('cp 19{-*.signature,.ly,.png,.eps,.log,.profile} dir1/') + system ('cp 19-1.signature 19.sub-1.signature') + system ('cp 19.ly 19.sub.ly') + system ('cp 19.profile 19.sub.profile') + system ('cp 19.log 19.sub.log') + system ('cp 19.png 19.sub.png') + system ('cp 19.eps 19.sub.eps') system ('cp 20multipage* dir1') system ('cp 20multipage* dir2') @@ -734,38 +1035,47 @@ def test_compare_trees (): system ('mkdir -p dir1/subdir/ dir2/subdir/') - system ('cp 19-sub{-*.signature,.ly,.png} dir1/subdir/') - system ('cp 19-sub{-*.signature,.ly,.png} dir2/subdir/') - system ('cp 20grob{-*.signature,.ly,.png} dir2/') - system ('cp 20grob{-*.signature,.ly,.png} dir1/') + system ('cp 19.sub{-*.signature,.ly,.png,.eps,.log,.profile} dir1/subdir/') + system ('cp 19.sub{-*.signature,.ly,.png,.eps,.log,.profile} dir2/subdir/') + system ('cp 20grob{-*.signature,.ly,.png,.eps,.log,.profile} dir2/') + system ('cp 20grob{-*.signature,.ly,.png,.eps,.log,.profile} dir1/') + system ('echo HEAD is 1 > dir1/tree.gittxt') + system ('echo HEAD is 2 > dir2/tree.gittxt') ## introduce differences system ('cp 19-1.signature dir2/20-1.signature') + system ('cp 19.profile dir2/20.profile') system ('cp 19.png dir2/20.png') system ('cp 19multipage-page1.png dir2/20multipage-page1.png') - system ('cp 20-1.signature dir2/subdir/19-sub-1.signature') - system ('cp 20.png dir2/subdir/19-sub.png') + system ('cp 20-1.signature dir2/subdir/19.sub-1.signature') + system ('cp 20.png dir2/subdir/19.sub.png') + system ("sed 's/: /: 1/g' 20.profile > dir2/subdir/19.sub.profile") ## radical diffs. system ('cp 19-1.signature dir2/20grob-1.signature') system ('cp 19-1.signature dir2/20grob-2.signature') + system ('cp 19multipage.midi dir1/midi-differ.midi') + system ('cp 20multipage.midi dir2/midi-differ.midi') + system ('cp 19multipage.log dir1/log-differ.log') + system ('cp 19multipage.log dir2/log-differ.log && echo different >> dir2/log-differ.log && echo different >> dir2/log-differ.log') - compare_trees ('dir1', 'dir2', 'compare-dir1dir2', 0.5) + compare_trees ('dir1', 'dir2', 'compare-dir1dir2', options.threshold) def test_basic_compare (): ly_template = r""" -#(set! toplevel-score-handler print-score-with-defaults) - #(set! toplevel-music-handler - (lambda (p m) - (if (not (eq? (ly:music-property m 'void) #t)) - (print-score-with-defaults - p (scorify-music m p))))) + +\version "2.10.0" +#(define default-toplevel-book-handler + print-book-with-defaults-as-systems ) + +#(ly:set-option (quote no-point-and-click)) \sourcefilename "my-source.ly" %(papermod)s -\header { tagline = ##f } +\header { tagline = ##f } +\score { << \new Staff \relative c { c4^"%(userstring)s" %(extragrob)s @@ -774,6 +1084,9 @@ def test_basic_compare (): c4^"%(userstring)s" %(extragrob)s } >> +\layout{} +} + """ dicts = [{ 'papermod' : '', @@ -792,30 +1105,32 @@ def test_basic_compare (): 'name' : '20grob', 'extragrob': 'r2. \\break c1', 'userstring': 'test' }, - ] for d in dicts: open (d['name'] + '.ly','w').write (ly_template % d) names = [d['name'] for d in dicts] - - system ('lilypond -ddump-signatures --png -b eps ' + ' '.join (names)) + + system ('lilypond -ddump-profile -dseparate-log-files -ddump-signatures --png -dbackend=eps ' + ' '.join (names)) multipage_str = r''' #(set-default-paper-size "a6") - {c1 \pageBreak c1 } + \score { + \relative {c1 \pageBreak c1 } + \layout {} + \midi {} + } ''' - open ('20multipage', 'w').write (multipage_str) - open ('19multipage', 'w').write ('#(set-global-staff-size 19.5)\n' + multipage_str) - system ('lilypond -ddump-signatures --png 19multipage 20multipage ') + open ('20multipage.ly', 'w').write (multipage_str.replace ('c1', 'd1')) + open ('19multipage.ly', 'w').write ('#(set-global-staff-size 19.5)\n' + multipage_str) + system ('lilypond -dseparate-log-files -ddump-signatures --png 19multipage 20multipage ') test_compare_signatures (names) def test_compare_signatures (names, timing=False): - import time times = 1 @@ -898,21 +1213,49 @@ def main (): type="float", help='threshold for geometric distance') - (o,a) = p.parse_args () + p.add_option ('--no-compare-images', + dest="compare_images", + default=True, + action="store_false", + help="Don't run graphical comparisons") + + p.add_option ('--create-images', + dest="create_images", + default=False, + action="store_true", + help="Create PNGs from EPSes") + - if o.run_test: + p.add_option ('--local-datadir', + dest="local_data_dir", + default=False, + action="store_true", + help='whether to use the share/lilypond/ directory in the test directory') + + p.add_option ('-o', '--output-dir', + dest="output_dir", + default=None, + action="store", + type="string", + help='where to put the test results [tree2/compare-tree1tree2]') + + global options + (options, args) = p.parse_args () + + if options.run_test: run_tests () sys.exit (0) - if len (a) != 2: + if len (args) != 2: p.print_usage() sys.exit (2) - global inspect_max_count - inspect_max_count = o.max_count - - compare_trees (a[0], a[1], os.path.join (a[1], 'compare-' + a[0]), - o.threshold) + name = options.output_dir + if not name: + name = args[0].replace ('/', '') + name = os.path.join (args[1], 'compare-' + shorten_string (name)) + + compare_trees (args[0], args[1], name, options.threshold) if __name__ == '__main__': main()