X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=buildscripts%2Foutput-distance.py;h=a19380f1f28b4c9cfbd4aad84749d89e3c733954;hb=9f3572d98bb948c9689cd1f75401a029451fa001;hp=7c7a914d567dd23a4426c5d8e1c3b77f678b58f3;hpb=04265f11d1f21416ccebd2dcaa1d903dc781b36e;p=lilypond.git diff --git a/buildscripts/output-distance.py b/buildscripts/output-distance.py index 7c7a914d56..a19380f1f2 100644 --- a/buildscripts/output-distance.py +++ b/buildscripts/output-distance.py @@ -1,12 +1,6 @@ #!@TARGET_PYTHON@ import sys import optparse -import os - -## so we can call directly as buildscripts/output-distance.py -me_path = os.path.abspath (os.path.split (sys.argv[0])[0]) -sys.path.insert (0, me_path + '/../python/') - import safeeval @@ -15,9 +9,8 @@ X_AXIS = 0 Y_AXIS = 1 INFTY = 1e6 -OUTPUT_EXPRESSION_PENALTY = 1 -ORPHAN_GROB_PENALTY = 1 -inspect_max_count = 0 +OUTPUT_EXPRESSION_PENALTY = 100 +ORPHAN_GROB_PENALTY = 1000 def max_distance (x1, x2): dist = 0.0 @@ -31,9 +24,6 @@ def max_distance (x1, x2): empty_interval = (INFTY, -INFTY) empty_bbox = (empty_interval, empty_interval) -def interval_is_empty (i): - return i[0] > i[1] - def interval_length (i): return max (i[1]-i[0], 0) @@ -45,10 +35,6 @@ def interval_intersect (i1, i2): return (max (i1[0], i2[0]), min (i1[1], i2[1])) -def bbox_is_empty (b): - return (interval_is_empty (b[0]) - or interval_is_empty (b[1])) - def bbox_union (b1, b2): return (interval_union (b1[X_AXIS], b2[X_AXIS]), interval_union (b2[Y_AXIS], b2[Y_AXIS])) @@ -78,10 +64,10 @@ class GrobSignature: def __repr__ (self): return '%s: (%.2f,%.2f), (%.2f,%.2f)\n' % (self.name, - self.bbox[0][0], - self.bbox[0][1], - self.bbox[1][0], - self.bbox[1][1]) + self.bbox[0][0], + self.bbox[0][1], + self.bbox[1][0], + self.bbox[1][1]) def axis_centroid (self, axis): return apply (sum, self.bbox[axis]) / 2 @@ -100,13 +86,15 @@ class GrobSignature: def expression_distance (self, other): if self.output_expression == other.output_expression: - return 0 + return 0.0 else: - return 1 - -################################################################ -# single System. + return OUTPUT_EXPRESSION_PENALTY + def distance(self, other, max_distance): + return (self.expression_distance (other) + + self.centroid_distance (other, max_distance) + + self.bbox_distance (other)) + class SystemSignature: def __init__ (self, grob_sigs): d = {} @@ -142,9 +130,6 @@ class SystemSignature: def grobs (self): return reduce (lambda x,y: x+y, self.grob_dict.values(), []) -################################################################ -## comparison of systems. - class SystemLink: def __init__ (self, system1, system2): self.system1 = system1 @@ -153,301 +138,48 @@ class SystemLink: self.link_list_dict = {} self.back_link_dict = {} - - ## pairs - self.orphans = [] - - ## pair -> distance - self.geo_distances = {} - - ## pairs - self.expression_changed = [] - - self._geometric_distance = None - self._expression_change_count = None - self._orphan_count = None - for g in system1.grobs (): - - ## skip empty bboxes. - if bbox_is_empty (g.bbox): - continue - closest = system2.closest (g.name, g.centroid) self.link_list_dict.setdefault (closest, []) self.link_list_dict[closest].append (g) self.back_link_dict[g] = closest + def distance (self): + d = 0.0 - def calc_geometric_distance (self): - total = 0.0 + scale = max (bbox_diameter (self.system1.bbox), + bbox_diameter (self.system2.bbox)) + for (g1,g2) in self.back_link_dict.items (): - if g2: - d = g1.bbox_distance (g2) - if d: - self.geo_distances[(g1,g2)] = d - - total += d - - self._geometric_distance = total - - def calc_orphan_count (self): - count = 0 - for (g1, g2) in self.back_link_dict.items (): if g2 == None: - self.orphans.append ((g1, None)) - - count += 1 + d += ORPHAN_GROB_PENALTY + else: + d += g1.distance (g2, scale) - self._orphan_count = count - - def calc_output_exp_distance (self): - d = 0 - for (g1,g2) in self.back_link_dict.items (): - if g2: - d += g1.expression_distance (g2) + for (g1,g2s) in self.link_list_dict.items (): + if len (g2s) != 1: + d += ORPHAN_GROB_PENALTY - self._expression_change_count = d + return d - def output_expression_details_string (self): - return ', '.join ([g1.name for g1 in self.expression_changed]) - - def geo_details_string (self): - results = [(d, g1,g2) for ((g1, g2), d) in self.geo_distances.items()] - results.sort () - results.reverse () - - return ', '.join (['%s: %f' % (g1.name, d) for (d, g1, g2) in results]) +################################################################ +# Files/directories - def orphan_details_string (self): - return ', '.join (['%s-None' % g1.name for (g1,g2) in self.orphans if g2==None]) +import glob +import shutil +import re - def geometric_distance (self): - if self._geometric_distance == None: - self.calc_geometric_distance () - return self._geometric_distance - - def orphan_count (self): - if self._orphan_count == None: - self.calc_orphan_count () - - return self._orphan_count - - def output_expression_change_count (self): - if self._expression_change_count == None: - self.calc_output_exp_distance () - return self._expression_change_count - - def distance (self): - return (self.output_expression_change_count (), - self.orphan_count (), - self.geometric_distance ()) - def read_signature_file (name): print 'reading', name - - entries = open (name).read ().split ('\n') - def string_to_tup (s): - return tuple (map (float, s.split (' '))) - - def string_to_entry (s): - fields = s.split('@') - fields[2] = string_to_tup (fields[2]) - fields[3] = string_to_tup (fields[3]) - - return tuple (fields) - - entries = [string_to_entry (e) for e in entries - if e and not e.startswith ('#')] + exp_str = ("[%s]" % open (name).read ()) + entries = safeeval.safe_eval (exp_str) grob_sigs = [GrobSignature (e) for e in entries] sig = SystemSignature (grob_sigs) return sig -################################################################ -# different systems of a .ly file. - -class FileLink: - def __init__ (self): - self.original_name = '' - self.base_names = ('','') - self.system_links = {} - self._distance = None - - def add_system_link (self, link, number): - self.system_links[number] = link - - def calc_distance (self): - d = 0.0 - - orphan_distance = 0.0 - for l in self.system_links.values (): - d = max (d, l.geometric_distance ()) - orphan_distance += l.orphan_count () - - return d + orphan_distance - - def distance (self): - if type (self._distance) != type (0.0): - return self.calc_distance () - - return self._distance - - def text_record_string (self): - return '%-30f %-20s\n' % (self.distance (), - self.original_name) - - def source_file (self): - for ext in ('.ly', '.ly.txt'): - if os.path.exists (self.base_names[1] + ext): - return self.base_names[1] + ext - return '' - - def add_file_compare (self, f1, f2): - system_index = [] - - def note_system_index (m): - system_index.append (int (m.group (1))) - return '' - - base1 = re.sub ("-([0-9]+).signature", note_system_index, f1) - base2 = re.sub ("-([0-9]+).signature", note_system_index, f2) - - self.base_names = (os.path.normpath (base1), - os.path.normpath (base2)) - - def note_original (match): - self.original_name = match.group (1) - return '' - - if not self.original_name: - self.original_name = os.path.split (base1)[1] - - ## ugh: drop the .ly.txt - for ext in ('.ly', '.ly.txt'): - try: - re.sub (r'\\sourcefilename "([^"]+)"', - note_original, open (base1 + ext).read ()) - except IOError: - pass - - s1 = read_signature_file (f1) - s2 = read_signature_file (f2) - - link = SystemLink (s1, s2) - - self.add_system_link (link, system_index[0]) - - def link_files_for_html (self, old_dir, new_dir, dest_dir): - for ext in ('.png', '.ly'): - for oldnew in (0,1): - link_file (self.base_names[oldnew] + ext, - dest_dir + '/' + self.base_names[oldnew] + ext) - - def html_record_string (self, old_dir, new_dir): - def img_cell (ly, img, name): - if not name: - name = 'source' - else: - name = '%s' % name - - return ''' - - - -
-(%(name)s) - - -''' % locals () - - - img_1 = self.base_names[0] + '.png' - ly_1 = self.base_names[0] + '.ly' - img_2 = self.base_names[1] + '.png' - ly_2 = self.base_names[1] + '.ly' - html_2 = self.base_names[1] + '.html' - name = self.original_name - - html_entry = ''' - - -%f
-(details) - - -%s -%s - -''' % (self.distance (), html_2, - img_cell (ly_1, img_1, name), img_cell (ly_2, img_2, name)) - - - return html_entry - - - def html_system_details_string (self): - systems = self.system_links.items () - systems.sort () - - html = "" - for (c, link) in systems: - e = '%d' % c - for d in link.distance (): - e += '%f' % d - - e = '%s' % e - - html += e - - e = '%d' % c - for s in (link.output_expression_details_string (), - link.orphan_details_string (), - link.geo_details_string ()): - e += "%s" % s - - - e = '%s' % e - html += e - - original = self.original_name - html = ''' - -comparison details for %(original)s - - - - - - - - - - -%(html)s -
systemoutputorphangeo
- - - -''' % locals () - return html - - def write_html_system_details (self, dir1, dir2, dest_dir): - dest_file = os.path.join (dest_dir, self.base_names[1] + '.html') - - details = open_write_file (dest_file) - details.write (self.html_system_details_string ()) - -################################################################ -# Files/directories - -import glob -import re - - - def compare_signature_files (f1, f2): s1 = read_signature_file (f1) s2 = read_signature_file (f2) @@ -481,160 +213,138 @@ class ComparisonData: self.result_dict = {} self.missing = [] self.added = [] - self.file_links = {} - + def compare_trees (self, dir1, dir2): self.compare_directories (dir1, dir2) - (root, dirs, files) = os.walk (dir1).next () + (root, files, dirs) = os.walk (dir1).next () for d in dirs: d1 = os.path.join (dir1, d) d2 = os.path.join (dir2, d) - - if os.path.islink (d1) or os.path.islink (d2): - continue if os.path.isdir (d2): self.compare_trees (d1, d2) def compare_directories (self, dir1, dir2): - + (paired, m1, m2) = paired_files (dir1, dir2, '*.signature') self.missing += [(dir1, m) for m in m1] self.added += [(dir2, m) for m in m2] for p in paired: - if (inspect_max_count - and len (self.file_links) > inspect_max_count): - - continue - f2 = dir2 + '/' + p f1 = dir1 + '/' + p - self.compare_files (f1, f2) - - def compare_files (self, f1, f2): - name = os.path.split (f1)[1] - name = re.sub ('-[0-9]+.signature', '', name) + distance = compare_signature_files (f1, f2) + self.result_dict[f2] = (distance, f1) + + def create_text_result_page (self, dir1, dir2): + self.write_text_result_page (dir2 + '/' + os.path.split (dir1)[1] + '.txt') - file_link = None - try: - file_link = self.file_links[name] - except KeyError: - file_link = FileLink () - self.file_links[name] = file_link - - file_link.add_file_compare (f1,f2) - - def write_text_result_page (self, filename, threshold): + def write_text_result_page (self, filename): print 'writing "%s"' % filename out = None if filename == '': out = sys.stdout else: - out = open_write_file (filename) - - ## todo: support more scores. - results = [(link.distance(), link) - for link in self.file_links.values ()] + out = open (filename, 'w') + + results = [(score, oldfile, file) for (file, (score, oldfile)) in self.result_dict.items ()] results.sort () results.reverse () + for (s, oldfile, f) in results: + out.write ('%-30f %-20s\n' % (s, f)) + + for (dir, file) in self.missing: + out.write ('%10s%-20s %s\n' % ('', 'missing',os.path.join (dir, file))) + for (dir, file) in self.added: + out.write ('%20s%-10s %s\n' % ('','added', os.path.join (dir, file))) + + def print_results (self): + self.write_text_result_page ('') - for (score, link) in results: - if score > threshold: - out.write (link.text_record_string ()) - - out.write ('\n\n') - out.write ('%d below threshold\n' % len ([1 for s,l in results - if threshold >= s > 0.0])) - out.write ('%d unchanged\n' % len ([1 for (s,l) in results if s == 0.0])) - - def create_text_result_page (self, dir1, dir2, dest_dir, threshold): - self.write_text_result_page (dest_dir + '/index.txt', threshold) - - def create_html_result_page (self, dir1, dir2, dest_dir, threshold): + def create_html_result_page (self, dir1, dir2): dir1 = dir1.replace ('//', '/') dir2 = dir2.replace ('//', '/') - results = [(link.distance(), link) - for link in self.file_links.values ()] + threshold = 1.0 + + results = [(score, oldfile, file) for (file, (score, oldfile)) in self.result_dict.items () + if score > threshold] + results.sort () results.reverse () html = '' old_prefix = os.path.split (dir1)[1] - for (score, link) in results: - if score <= threshold: - continue + os.mkdir (dir2 + '/' + old_prefix) + for (score, oldfile, newfile) in results: + old_base = re.sub ("-[0-9]+.signature", '', os.path.split (oldfile)[1]) + new_base = re.sub ("-[0-9]+.signature", '', newfile) + + for ext in 'png', 'ly': + shutil.copy2 (old_base + '.' + ext, dir2 + '/' + old_prefix) - link.write_html_system_details (dir1, dir2, dest_dir) - link.link_files_for_html (dir1, dir2, dest_dir) - html += link.html_record_string (dir1, dir2) + img_1 = os.path.join (old_prefix, old_base + '.png') + ly_1 = os.path.join (old_prefix, old_base + '.ly') + img_2 = new_base.replace (dir2, '') + '.png' + img_2 = re.sub ("^/*", '', img_2) + + ly_2 = img_2.replace ('.png','.ly') + + def img_cell (ly, img): + return ''' + + + +
+(source) + + +''' % locals () + + html_entry = ''' + + +%f + + +%s +%s + +''' % (score, img_cell (ly_1, img_1), img_cell (ly_2, img_2)) + + + html += html_entry html = ''' - +
- - + + %(html)s
distance%(dir1)s%(dir2)soldnew
''' % locals() - - html += ('

') - below_count =len ([1 for s,l in results - if threshold >= s > 0.0]) - - if below_count: - html += ('

%d below threshold

' % below_count) - - html += ('

%d unchanged

' - % len ([1 for (s,l) in results if s == 0.0])) - - - dest_file = dest_dir + '/index.html' - open_write_file (dest_file).write (html) + + open (os.path.join (dir2, old_prefix) + '.html', 'w').write (html) + - def print_results (self, threshold): - self.write_text_result_page ('', threshold) -def compare_trees (dir1, dir2, dest_dir, threshold): - data = ComparisonData () +def compare_trees (dir1, dir2): + data = ComparisonData () data.compare_trees (dir1, dir2) - data.print_results (threshold) - - if os.path.isdir (dest_dir): - system ('rm -rf %s '% dest_dir) - - data.create_html_result_page (dir1, dir2, dest_dir, threshold) - data.create_text_result_page (dir1, dir2, dest_dir, threshold) + data.print_results () + data.create_html_result_page (dir1, dir2) + data.create_text_result_page (dir1, dir2) ################################################################ # TESTING -def mkdir (x): - if not os.path.isdir (x): - print 'mkdir', x - os.makedirs (x) - -def link_file (x, y): - mkdir (os.path.split (y)[0]) - try: - os.link (x, y) - except OSError, z: - print 'OSError', x, y, z - raise OSError - -def open_write_file (x): - d = os.path.split (x)[0] - mkdir (d) - return open (x, 'w') - - +import os def system (x): print 'invoking', x @@ -650,30 +360,17 @@ def test_paired_files (): def test_compare_trees (): system ('rm -rf dir1 dir2') system ('mkdir dir1 dir2') - system ('cp 20{-*.signature,.ly,.png} dir1') - system ('cp 20{-*.signature,.ly,.png} dir2') - system ('cp 20expr{-*.signature,.ly,.png} dir1') - system ('cp 19{-*.signature,.ly,.png} dir2/') - system ('cp 19{-*.signature,.ly,.png} dir1/') - system ('cp 19-1.signature 19-sub-1.signature') - system ('cp 19.ly 19-sub.ly') - system ('cp 19.png 19-sub.png') - - system ('mkdir -p dir1/subdir/ dir2/subdir/') - system ('cp 19-sub{-*.signature,.ly,.png} dir1/subdir/') - system ('cp 19-sub{-*.signature,.ly,.png} dir2/subdir/') - system ('cp 20grob{-*.signature,.ly,.png} dir2/') - system ('cp 20grob{-*.signature,.ly,.png} dir1/') + system ('cp 20{-0.signature,.ly,.png} dir1') + system ('cp 20{-0.signature,.ly,.png} dir2') + system ('cp 20expr{-0.signature,.ly,.png} dir1') + system ('cp 19{-0.signature,.ly,.png} dir2/') + system ('cp 19{-0.signature,.ly,.png} dir1/') + system ('cp 20grob{-0.signature,.ly,.png} dir2/') - ## introduce differences - system ('cp 19-1.signature dir2/20-1.signature') - system ('cp 20-1.signature dir2/subdir/19-sub-1.signature') + ## introduce difference + system ('cp 19-0.signature dir2/20-0.signature') - ## radical diffs. - system ('cp 19-1.signature dir2/20grob-1.signature') - system ('cp 19-1.signature dir2/20grob-2.signature') - - compare_trees ('dir1', 'dir2', 'compare-dir1dir2', 0.5) + compare_trees ('dir1', 'dir2') def test_basic_compare (): @@ -684,17 +381,11 @@ def test_basic_compare (): (print-score-with-defaults p (scorify-music m p))))) -\sourcefilename "my-source.ly" - %(papermod)s -<< -\new Staff \relative c { - c4^"%(userstring)s" %(extragrob)s - } -\new Staff \relative c { - c4^"%(userstring)s" %(extragrob)s + +\relative c { + c^"%(userstring)s" %(extragrob)s } ->> """ dicts = [{ 'papermod' : '', @@ -711,10 +402,8 @@ def test_basic_compare (): 'userstring': 'blabla' }, { 'papermod' : '', 'name' : '20grob', - 'extragrob': 'r2. \\break c1', - 'userstring': 'test' } - - ] + 'extragrob': 'c4', + 'userstring': 'test' }] for d in dicts: open (d['name'] + '.ly','w').write (ly_template % d) @@ -722,63 +411,42 @@ def test_basic_compare (): names = [d['name'] for d in dicts] system ('lilypond -ddump-signatures --png -b eps ' + ' '.join (names)) - test_compare_signatures (names) -def test_compare_signatures (names, timing=False): - - import time - - times = 1 - if timing: - times = 100 - - t0 = time.clock () - - count = 0 - for t in range (0, times): - sigs = dict ((n, read_signature_file ('%s-1.signature' % n)) for n in names) - count += 1 - - if timing: - print 'elapsed', (time.clock() - t0)/count - - - t0 = time.clock () - count = 0 + sigs = dict ((n, read_signature_file ('%s-0.signature' % n)) for n in names) combinations = {} for (n1, s1) in sigs.items(): for (n2, s2) in sigs.items(): combinations['%s-%s' % (n1, n2)] = SystemLink (s1,s2).distance () - count += 1 - - if timing: - print 'elapsed', (time.clock() - t0)/count results = combinations.items () results.sort () for k,v in results: print '%-20s' % k, v - assert combinations['20-20'] == (0.0,0.0,0.0) - assert combinations['20-20expr'][0] > 0.0 - assert combinations['20-19'][2] < 10.0 - assert combinations['20-19'][2] > 0.0 + assert combinations['20-20'] == 0.0 + assert combinations['20-20expr'] > 50.0 + assert combinations['20-19'] < 10.0 + + +def test_sigs (a,b): + sa = read_signature_file (a) + sb = read_signature_file (b) + link = SystemLink (sa, sb) + print link.distance() def run_tests (): + do_clean = 1 dir = 'output-distance-test' - do_clean = not os.path.exists (dir) - print 'test results in ', dir if do_clean: system ('rm -rf ' + dir) system ('mkdir ' + dir) os.chdir (dir) - if do_clean: - test_basic_compare () - + + test_basic_compare () test_compare_trees () ################################################################ @@ -788,25 +456,10 @@ def main (): p = optparse.OptionParser ("output-distance - compare LilyPond formatting runs") p.usage = 'output-distance.py [options] tree1 tree2' - p.add_option ('', '--test-self', + p.add_option ('', '--test', dest="run_test", action="store_true", help='run test method') - - p.add_option ('--max-count', - dest="max_count", - metavar="COUNT", - type="int", - default=0, - action="store", - help='only analyze COUNT signature pairs') - - p.add_option ('', '--threshold', - dest="threshold", - default=0.3, - action="store", - type="float", - help='threshold for geometric distance') (o,a) = p.parse_args () @@ -818,11 +471,7 @@ def main (): p.print_usage() sys.exit (2) - global inspect_max_count - inspect_max_count = o.max_count - - compare_trees (a[0], a[1], os.path.join (a[1], 'compare-' + a[0]), - o.threshold) + compare_trees (a[0], a[1]) if __name__ == '__main__': main()