X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=scripts%2Fbuild%2Foutput-distance.py;h=625ce12a1e3f1ab70db53797457ddc400e2472a5;hb=84675bc536af363543bdae1fa422903a977e1ec8;hp=f3ef75385f318bc0b0634fb14749095b0a5ed3d6;hpb=74b336b3cab541e5f57c11c33ff52475eb655414;p=lilypond.git diff --git a/scripts/build/output-distance.py b/scripts/build/output-distance.py old mode 100644 new mode 100755 index f3ef75385f..625ce12a1e --- a/scripts/build/output-distance.py +++ b/scripts/build/output-distance.py @@ -3,6 +3,9 @@ import sys import optparse import os import math +import re + +import cgi ## so we can call directly as scripts/build/output-distance.py me_path = os.path.abspath (os.path.split (sys.argv[0])[0]) @@ -27,7 +30,7 @@ class TempDirectory: self.dir = tempfile.mkdtemp () print 'dir is', self.dir def __del__ (self): - print 'rm -rf %s' % self.dir + print 'rm -rf %s' % self.dir os.system ('rm -rf %s' % self.dir) def __call__ (self): return self.dir @@ -50,8 +53,7 @@ def system (c): raise Exception ("failed") return -def shorten_string (s): - threshold = 15 +def shorten_string (s, threshold = 15): if len (s) > 2*threshold: s = s[:threshold] + '..' + s[-threshold:] return s @@ -61,14 +63,14 @@ def max_distance (x1, x2): for (p,q) in zip (x1, x2): dist = max (abs (p-q), dist) - + return dist def compare_png_images (old, new, dest_dir): def png_dims (f): m = re.search ('([0-9]+) x ([0-9]+)', read_pipe ('file %s' % f)) - + return tuple (map (int, m.groups ())) dest = os.path.join (dest_dir, new.replace ('.png', '.compare.jpeg')) @@ -79,7 +81,7 @@ def compare_png_images (old, new, dest_dir): ## hmmm. what to do? system ('touch %(dest)s' % locals ()) return - + dims = (min (dims1[0], dims2[0]), min (dims1[1], dims2[1])) @@ -87,7 +89,7 @@ def compare_png_images (old, new, dest_dir): system ('convert -depth 8 -crop %dx%d+0+0 %s %s/crop1.png' % (dims + (old, dir))) system ('convert -depth 8 -crop %dx%d+0+0 %s %s/crop2.png' % (dims + (new, dir))) - system ('compare -depth 8 %(dir)s/crop1.png %(dir)s/crop2.png %(dir)s/diff.png' % locals ()) + system1 ('compare -depth 8 -dissimilarity-threshold 1 %(dir)s/crop1.png %(dir)s/crop2.png %(dir)s/diff.png' % locals ()) system ("convert -depth 8 %(dir)s/diff.png -blur 0x3 -negate -channel alpha,blue -type TrueColorMatte -fx 'intensity' %(dir)s/matte.png" % locals ()) @@ -104,8 +106,8 @@ def interval_is_empty (i): return i[0] > i[1] def interval_length (i): - return max (i[1]-i[0], 0) - + return max (i[1]-i[0], 0) + def interval_union (i1, i2): return (min (i1[0], i2[0]), max (i1[1], i2[1])) @@ -121,7 +123,7 @@ def bbox_is_empty (b): def bbox_union (b1, b2): return (interval_union (b1[X_AXIS], b2[X_AXIS]), interval_union (b1[Y_AXIS], b2[Y_AXIS])) - + def bbox_intersection (b1, b2): return (interval_intersect (b1[X_AXIS], b2[X_AXIS]), interval_intersect (b1[Y_AXIS], b2[Y_AXIS])) @@ -132,7 +134,7 @@ def bbox_area (b): def bbox_diameter (b): return max (interval_length (b[X_AXIS]), interval_length (b[Y_AXIS])) - + def difference_area (a, b): return bbox_area (a) - bbox_area (bbox_intersection (a,b)) @@ -141,7 +143,7 @@ class GrobSignature: def __init__ (self, exp_list): (self.name, self.origin, bbox_x, bbox_y, self.output_expression) = tuple (exp_list) - + self.bbox = (bbox_x, bbox_y) self.centroid = (bbox_x[0] + bbox_x[1], bbox_y[0] + bbox_y[1]) @@ -151,13 +153,13 @@ class GrobSignature: self.bbox[0][1], self.bbox[1][0], self.bbox[1][1]) - + def axis_centroid (self, axis): - return apply (sum, self.bbox[axis]) / 2 - + return apply (sum, self.bbox[axis]) / 2 + def centroid_distance (self, other, scale): - return max_distance (self.centroid, other.centroid) / scale - + return max_distance (self.centroid, other.centroid) / scale + def bbox_distance (self, other): divisor = bbox_area (self.bbox) + bbox_area (other.bbox) @@ -166,7 +168,7 @@ class GrobSignature: difference_area (other.bbox, self.bbox)) / divisor else: return 0.0 - + def expression_distance (self, other): if self.output_expression == other.output_expression: return 0 @@ -218,7 +220,7 @@ class SystemLink: def __init__ (self, system1, system2): self.system1 = system1 self.system2 = system2 - + self.link_list_dict = {} self.back_link_dict = {} @@ -235,15 +237,15 @@ class SystemLink: self._geometric_distance = None self._expression_change_count = None self._orphan_count = None - + for g in system1.grobs (): ## skip empty bboxes. if bbox_is_empty (g.bbox): continue - + closest = system2.closest (g.name, g.centroid) - + self.link_list_dict.setdefault (closest, []) self.link_list_dict[closest].append (g) self.back_link_dict[g] = closest @@ -260,17 +262,17 @@ class SystemLink: total += d self._geometric_distance = total - + def calc_orphan_count (self): count = 0 for (g1, g2) in self.back_link_dict.items (): if g2 == None: self.orphans.append ((g1, None)) - + count += 1 self._orphan_count = count - + def calc_output_exp_distance (self): d = 0 for (g1,g2) in self.back_link_dict.items (): @@ -281,12 +283,12 @@ class SystemLink: def output_expression_details_string (self): return ', '.join ([g1.name for g1 in self.expression_changed]) - + def geo_details_string (self): results = [(d, g1,g2) for ((g1, g2), d) in self.geo_distances.items()] results.sort () results.reverse () - + return ', '.join (['%s: %f' % (g1.name, d) for (d, g1, g2) in results]) def orphan_details_string (self): @@ -296,29 +298,34 @@ class SystemLink: if self._geometric_distance == None: self.calc_geometric_distance () return self._geometric_distance - + def orphan_count (self): if self._orphan_count == None: self.calc_orphan_count () - + return self._orphan_count - + def output_expression_change_count (self): if self._expression_change_count == None: self.calc_output_exp_distance () return self._expression_change_count - + def distance (self): return (self.output_expression_change_count (), self.orphan_count (), self.geometric_distance ()) - + +def scheme_float (s) : + if 'nan' not in s : + return float(s) + return float(s.split('.')[0]) + def read_signature_file (name): print 'reading', name - + entries = open (name).read ().split ('\n') def string_to_tup (s): - return tuple (map (float, s.split (' '))) + return tuple (map (scheme_float, s.split (' '))) def string_to_entry (s): fields = s.split('@') @@ -326,7 +333,7 @@ def read_signature_file (name): fields[3] = string_to_tup (fields[3]) return tuple (fields) - + entries = [string_to_entry (e) for e in entries if e and not e.startswith ('#')] @@ -344,13 +351,13 @@ class FileLink: def __init__ (self, f1, f2): self._distance = None self.file_names = (f1, f2) - + def text_record_string (self): return '%-30f %-20s\n' % (self.distance (), self.name () + os.path.splitext (self.file_names[1])[1] ) - + def calc_distance (self): return 0.0 @@ -359,23 +366,29 @@ class FileLink: self._distance = self.calc_distance () return self._distance - + def source_file (self): for ext in ('.ly', '.ly.txt'): base = os.path.splitext (self.file_names[1])[0] f = base + ext if os.path.exists (f): return f - + return '' - + + def directories (self): + return map (os.path.dirname, self.file_names) + def name (self): base = os.path.basename (self.file_names[1]) base = os.path.splitext (base)[0] base = hash_to_original_name.get (base, base) base = os.path.splitext (base)[0] - return base - + return os.path.join (self.prefix (), base) + + def prefix (self): + return os.path.dirname (os.path.commonprefix (self.file_names)) + def extension (self): return os.path.splitext (self.file_names[1])[1] @@ -388,13 +401,13 @@ class FileLink: def get_cell (self, oldnew): return '' - + def get_file (self, oldnew): return self.file_names[oldnew] - + def html_record_string (self, dest_dir): dist = self.distance() - + details = self.get_distance_details () if details: details_base = os.path.splitext (self.file_names[1])[0] @@ -410,7 +423,7 @@ class FileLink: name = self.name () + self.extension () file1 = self.get_file (0) file2 = self.get_file (1) - + return '''
%s' % str + + str = '
%s' % cgi.escape (str) return str - + def calc_distance (self): if self.contents[0] == self.contents[1]: d = 0.0 @@ -462,26 +475,37 @@ class GitFileCompareLink (FileCompareLink): return d - + +snippet_fn_re = re.compile (r"`\./([0-9a-f]{2}/lily-[0-9a-f]{8}).eps'"); class TextFileCompareLink (FileCompareLink): def calc_distance (self): import difflib - diff = difflib.unified_diff (self.contents[0].strip().split ('\n'), - self.contents[1].strip().split ('\n'), + # Extract the old and the new hashed snippet names from the log file + # and replace the old by the new, so file name changes don't show + # up as log differences... + cont0 = self.contents[0].strip(); + cont1 = self.contents[1].strip(); + m0 = re.search (snippet_fn_re, cont0); + m1 = re.search (snippet_fn_re, cont1); + if (m0 and m1 and (m0.group(1) != m1.group(1))): + cont0 = cont0.replace (m0.group(1), m1.group(1)); + + diff = difflib.unified_diff (cont0.split ('\n'), + cont1.split ('\n'), fromfiledate = self.file_names[0], tofiledate = self.file_names[1] ) - + self.diff_lines = [l for l in diff] self.diff_lines = self.diff_lines[2:] - + return math.sqrt (float (len ([l for l in self.diff_lines if l[0] in '-+']))) - + def get_cell (self, oldnew): str = '' if oldnew == 1: str = '\n'.join ([d.replace ('\n','') for d in self.diff_lines]) - str = '
%s' % str + str = '
%s' % cgi.escape (str) return str class LogFileCompareLink (TextFileCompareLink): @@ -489,12 +513,12 @@ class LogFileCompareLink (TextFileCompareLink): c = TextFileCompareLink.get_content (self, f) c = re.sub ("\nProcessing `[^\n]+'\n", '', c) return c - + class ProfileFileLink (FileCompareLink): def __init__ (self, f1, f2): FileCompareLink.__init__ (self, f1, f2) self.results = [{}, {}] - + def get_cell (self, oldnew): str = '' for k in ('time', 'cells'): @@ -504,8 +528,8 @@ class ProfileFileLink (FileCompareLink): str += '%-8s: %8d (%5.3f)\n' % (k, int (self.results[oldnew][k]), self.get_ratio (k)) - return '
%s' % str - + return '
%s' % cgi.escape (str) + def get_ratio (self, key): (v1,v2) = (self.results[0].get (key, -1), self.results[1].get (key, -1)) @@ -514,12 +538,12 @@ class ProfileFileLink (FileCompareLink): return 0.0 return (v1 - v2) / float (v1+v2) - + def calc_distance (self): for oldnew in (0,1): def note_info (m): self.results[oldnew][m.group(1)] = float (m.group (2)) - + re.sub ('([a-z]+): ([-0-9.]+)\n', note_info, self.contents[oldnew]) @@ -528,7 +552,7 @@ class ProfileFileLink (FileCompareLink): 'time': 0.1, 'cells': 5.0, } - + for k in ('time', 'cells'): real_val = math.tan (self.get_ratio (k) * 0.5 * math.pi) dist += math.exp (math.fabs (real_val) * factor[k]) - 1 @@ -536,11 +560,11 @@ class ProfileFileLink (FileCompareLink): dist = min (dist, 100) return dist - + class MidiFileLink (TextFileCompareLink): def get_content (self, oldnew): import midi - + data = FileCompareLink.get_content (self, oldnew) midi = midi.parse (data) tracks = midi[1] @@ -555,10 +579,10 @@ class MidiFileLink (TextFileCompareLink): ev_str = repr (e) if re.search ('LilyPond [0-9.]+', ev_str): continue - + str += ' ev %s\n' % `e` return str - + class SignatureFileLink (FileLink): @@ -576,16 +600,16 @@ class SignatureFileLink (FileLink): for l in self.system_links.values (): d = max (d, l.geometric_distance ()) orphan_distance += l.orphan_count () - + return d + orphan_distance def add_file_compare (self, f1, f2): - system_index = [] + system_index = [] def note_system_index (m): system_index.append (int (m.group (1))) return '' - + base1 = re.sub ("-([0-9]+).signature", note_system_index, f1) base2 = re.sub ("-([0-9]+).signature", note_system_index, f2) @@ -599,7 +623,7 @@ class SignatureFileLink (FileLink): self.add_system_link (link, system_index[0]) - + def create_images (self, dest_dir): files_created = [[], []] @@ -613,11 +637,12 @@ class SignatureFileLink (FileLink): if options.local_data_dir: data_option = ('-slilypond-datadir=%s/share/lilypond/current ' % os.path.dirname(infile)) - + mkdir (os.path.split (outfile)[0]) cmd = ('gs -sDEVICE=png16m -dGraphicsAlphaBits=4 -dTextAlphaBits=4 ' ' %(data_option)s ' ' -r101 ' + ' -dAutoRotatePages=/None ' ' -sOutputFile=%(outfile)s -dNOSAFER -dEPSCrop -q -dNOPAUSE ' ' %(infile)s -c quit ') % locals () @@ -625,7 +650,7 @@ class SignatureFileLink (FileLink): system (cmd) return files_created - + def link_files_for_html (self, dest_dir): FileLink.link_files_for_html (self, dest_dir) to_compare = [[], []] @@ -635,8 +660,8 @@ class SignatureFileLink (FileLink): to_compare = self.create_images (dest_dir) else: exts += ['.png', '-page*png'] - - for ext in exts: + + for ext in exts: for oldnew in (0,1): for f in glob.glob (self.base_names[oldnew] + ext): dst = dest_dir + '/' + f @@ -644,8 +669,8 @@ class SignatureFileLink (FileLink): if f.endswith ('.png'): to_compare[oldnew].append (f) - - if options.compare_images: + + if options.compare_images: for (old, new) in zip (to_compare[0], to_compare[1]): compare_png_images (old, new, dest_dir) @@ -656,7 +681,7 @@ class SignatureFileLink (FileLink): name = 'source' else: name = '%s' % name - + return ''' @@ -670,7 +695,7 @@ class SignatureFileLink (FileLink): imgs_str = '\n'.join ([''' -
distance | %(short_dir1)s | %(short_dir2)s |
---|
') + table_rows = '' + old_prefix = None + for link in changed: + this_prefix = link.prefix () + if (old_prefix != this_prefix): + old_prefix = this_prefix + short_dir1 = shorten_string (link.directories ()[0], 30) + short_dir2 = shorten_string (link.directories ()[1], 30) + table_rows += header_row % locals() + table_rows += link.html_record_string (dest_dir) + + summary = '' below_count = len (below) if below_count: - html += ('
%d below threshold
' % below_count) - - html += ('%d unchanged
' % len (unchanged)) + summary += '%d below threshold
' % below_count + + summary += '%d unchanged
' % len (unchanged) + + me = sys.argv[0] + + html = ''' + ++ click to filter rows by type: + ly / + profiling / + signature / + midi / + log / + gittxt / + reset to all +
+ +