From: Han-Wen Nienhuys Date: Tue, 2 Jan 2007 20:02:08 +0000 (+0100) Subject: Compare logfiles for tests. X-Git-Tag: release/2.11.8-1~33 X-Git-Url: https://git.donarmstrong.com/?a=commitdiff_plain;h=118d26a71a4836f74592b4f65afb66bff14a39de;p=lilypond.git Compare logfiles for tests. - Add -dseparate-log-files. This writes BASENAME.log for every file processed. The "failed files" notice is written to the default log. - Process .log files in output-distance too - Use difflib to measure differences in textfiles - Add ok-test target to toplevel make file. This moves the out-test directory to out-test-ok, which is now the base for comparisons. - Document this procedure in INSTALL.texi --- diff --git a/Documentation/topdocs/INSTALL.texi b/Documentation/topdocs/INSTALL.texi index 52d8d8cf76..6a7196fabd 100644 --- a/Documentation/topdocs/INSTALL.texi +++ b/Documentation/topdocs/INSTALL.texi @@ -157,6 +157,22 @@ make out=www web-install @end example @end quotation +@section Testing LilyPond + +LilyPond comes with an extensive suite that excercises the entire +program. This suite can be used to automatically check the impact of a +change. This is done as follows + +@example + make ok-test + @emph{apply your changes, compile} + make test-clean + make check +@end example + +This will leave an HTML page @file{out/test-results/index.html}. This +page shows all the important differences that your change introduced, +whether in the layout, the MIDI output, or error reporting. @section Building LilyPond diff --git a/GNUmakefile.in b/GNUmakefile.in index 37fb44abf4..f28f408fba 100644 --- a/GNUmakefile.in +++ b/GNUmakefile.in @@ -35,7 +35,6 @@ include $(depth)/make/stepmake.make # # suggested settings # -# CHECK_SOURCE= # LILYPOND_JOBS= -djob-count=X ## for SMP/Multicore machine # include local.make @@ -210,11 +209,15 @@ test-clean: $(MAKE) -C input/regression/ out=test clean test: - $(MAKE) -C input/regression/ out=test LILYPOND_BOOK_LILYPOND_FLAGS="--backend=eps --formats=ps $(LILYPOND_JOBS) -dinclude-eps-fonts -dgs-load-fonts --header=texidoc -I $(top-src-dir)/input/manual -ddump-profile -dcheck-internal-types -ddump-signatures -danti-alias-factor=1" LILYPOND_BOOK_VERBOSE= out-test/collated-files.html + $(MAKE) -C input/regression/ out=test LILYPOND_BOOK_LILYPOND_FLAGS="--backend=eps --formats=ps $(LILYPOND_JOBS) -dseparate-log-files -dinclude-eps-fonts -dgs-load-fonts --header=texidoc -I $(top-src-dir)/input/manual -ddump-profile -dcheck-internal-types -ddump-signatures -danti-alias-factor=1" LILYPOND_BOOK_VERBOSE= out-test/collated-files.html @find input ly -name '*.ly' -print |grep -v 'out.*/' | xargs grep '\\version' -L | grep -v "standard input" |sed 's/^/**** Missing version: /g' +ok-test: test + mv input/regression/out-test input/regression/out-testok + RESULT_DIR=$(top-build-dir)/out/test-results/ local-check: test rm -rf $(RESULT_DIR) mkdir -p $(RESULT_DIR) - $(PYTHON) $(buildscript-dir)/output-distance.py --create-images --output-dir $(RESULT_DIR) $(CHECK_SOURCE) input/regression/out-test/ + $(PYTHON) $(buildscript-dir)/output-distance.py --create-images --output-dir $(RESULT_DIR) input/regression/out-testok input/regression/out-test/ + diff --git a/buildscripts/output-distance.py b/buildscripts/output-distance.py index ec8b5d9e10..e6e62562fa 100644 --- a/buildscripts/output-distance.py +++ b/buildscripts/output-distance.py @@ -313,12 +313,21 @@ def compare_png_images (old, new, dir): system ("composite -quality 65 matte.png %(new)s %(dest)s" % locals ()) class FileLink: + def __init__ (self): + self._distance = None + def text_record_string (self): return '%-30f %-20s\n' % (self.distance (), self.name ()) - def distance (self): + def calc_distance (self): return 0.0 + def distance (self): + if self._distance == None: + self._distance = self.calc_distance () + + return self._distance + def name (self): return '' @@ -331,32 +340,27 @@ class FileLink: def html_record_string (self, old_dir, new_dir): return '' -class MidiFileLink (FileLink): - def get_midi (self, f): - s = open (f).read () - s = re.sub ('LilyPond [0-9.]+', '', s) - return s - +class FileCompareLink (FileLink): def __init__ (self, f1, f2): + FileLink.__init__ (self) self.files = (f1, f2) - - s1 = self.get_midi (self.files[0]) - s2 = self.get_midi (self.files[1]) - - self.same = (s1 == s2) + self.contents = (self.get_content (self.files[0]), + self.get_content (self.files[1])) def name (self): - name = os.path.split (self.files[0])[1] - name = re.sub ('.midi', '', name) + name = os.path.basename (self.files[0]) + name = os.path.splitext (name)[0] return name - def distance (self): + def calc_distance (self): ## todo: could use import MIDI to pinpoint ## what & where changed. - if self.same: - return 0 + + if self.contents[0] == self.contents[1]: + return 0.0 else: - return 100; + return 100.0; + def html_record_string (self, d1, d2): return ''' @@ -366,12 +370,52 @@ class MidiFileLink (FileLink): %s ''' % ((self.distance(),) + self.files) + def get_content (self, f): + s = open (f).read () + return s + +class TextFileCompareLink (FileCompareLink): + def calc_distance (self): + import difflib + diff = difflib.unified_diff (self.contents[0].split ('\n'), + self.contents[1].split ('\n'), + fromfiledate = self.files[0], + tofiledate = self.files[1] + ) + + self.diff_lines = [l for l in diff] + return float (len (self.diff_lines)) + + def link_files_for_html (self, old_dir, new_dir, dest_dir): + str = '\n'.join ([d.replace ('\n','') for d in self.diff_lines]) + f = os.path.join (new_dir, self.name ()) + '.diff.txt' + f = os.path.join (dest_dir, f) + open_write_file (f).write (str) + + def html_record_string (self, d1, d2): + return ''' + +%f + +%s +%s +''' % (self.distance(), + self.files[0], + os.path.join (d2, self.name ()), + self.files[1]) + +class MidiFileLink (FileCompareLink): + def get_content (self, f): + s = open (f).read () + s = re.sub ('LilyPond [0-9.]+', '', s) + return s + class SignatureFileLink (FileLink): def __init__ (self): + FileLink.__init__ (self) self.original_name = '' self.base_names = ('','') self.system_links = {} - self._distance = None def name (self): return self.original_name @@ -388,12 +432,6 @@ class SignatureFileLink (FileLink): return d + orphan_distance - def distance (self): - if type (self._distance) != type (0.0): - return self.calc_distance () - - return self._distance - def source_file (self): for ext in ('.ly', '.ly.txt'): if os.path.exists (self.base_names[1] + ext): @@ -609,8 +647,6 @@ class SignatureFileLink (FileLink): import glob import re - - def compare_signature_files (f1, f2): s1 = read_signature_file (f1) s2 = read_signature_file (f2) @@ -661,7 +697,7 @@ class ComparisonData: self.compare_trees (d1, d2) def compare_directories (self, dir1, dir2): - for ext in ['signature', 'midi']: + for ext in ['signature', 'midi', 'log']: (paired, m1, m2) = paired_files (dir1, dir2, '*.' + ext) self.missing += [(dir1, m) for m in m1] @@ -682,6 +718,21 @@ class ComparisonData: self.compare_signature_files (f1, f2) elif f1.endswith ('midi'): self.compare_midi_files (f1, f2) + elif f1.endswith ('log'): + self.compare_ascii_files (f1, f2) + + def compare_general_files (self, f1, f2): + name = os.path.split (f1)[1] + + file_link = FileCompareLink (f1, f2) + self.file_links[name] = file_link + + def compare_ascii_files (self, f1, f2): + name = os.path.split (f1)[1] + + file_link = TextFileCompareLink (f1, f2) + self.file_links[name] = file_link + def compare_midi_files (self, f1, f2): name = os.path.split (f1)[1] @@ -828,13 +879,14 @@ def test_paired_files (): def test_compare_trees (): system ('rm -rf dir1 dir2') system ('mkdir dir1 dir2') - system ('cp 20{-*.signature,.ly,.png,.eps} dir1') - system ('cp 20{-*.signature,.ly,.png,.eps} dir2') - system ('cp 20expr{-*.signature,.ly,.png,.eps} dir1') - system ('cp 19{-*.signature,.ly,.png,.eps} dir2/') - system ('cp 19{-*.signature,.ly,.png,.eps} dir1/') + system ('cp 20{-*.signature,.ly,.png,.eps,.log} dir1') + system ('cp 20{-*.signature,.ly,.png,.eps,.log} dir2') + system ('cp 20expr{-*.signature,.ly,.png,.eps,.log} dir1') + system ('cp 19{-*.signature,.ly,.png,.eps,.log} dir2/') + system ('cp 19{-*.signature,.ly,.png,.eps,.log} dir1/') system ('cp 19-1.signature 19-sub-1.signature') system ('cp 19.ly 19-sub.ly') + system ('cp 19.log 19-sub.log') system ('cp 19.png 19-sub.png') system ('cp 19.eps 19-sub.eps') @@ -844,10 +896,10 @@ def test_compare_trees (): system ('mkdir -p dir1/subdir/ dir2/subdir/') - system ('cp 19-sub{-*.signature,.ly,.png,.eps} dir1/subdir/') - system ('cp 19-sub{-*.signature,.ly,.png,.eps} dir2/subdir/') - system ('cp 20grob{-*.signature,.ly,.png,.eps} dir2/') - system ('cp 20grob{-*.signature,.ly,.png,.eps} dir1/') + system ('cp 19-sub{-*.signature,.ly,.png,.eps,.log} dir1/subdir/') + system ('cp 19-sub{-*.signature,.ly,.png,.eps,.log} dir2/subdir/') + system ('cp 20grob{-*.signature,.ly,.png,.eps,.log} dir2/') + system ('cp 20grob{-*.signature,.ly,.png,.eps,.log} dir1/') ## introduce differences system ('cp 19-1.signature dir2/20-1.signature') @@ -861,6 +913,8 @@ def test_compare_trees (): system ('cp 19-1.signature dir2/20grob-2.signature') system ('cp 19multipage.midi dir1/midi-differ.midi') system ('cp 20multipage.midi dir2/midi-differ.midi') + system ('cp 19multipage.log dir1/log-differ.log') + system ('cp 19multipage.log dir2/log-differ.log && echo different >> dir2/log-differ.log && echo different >> dir2/log-differ.log') compare_trees ('dir1', 'dir2', 'compare-dir1dir2', 0.5) @@ -869,12 +923,10 @@ def test_basic_compare (): ly_template = r""" \version "2.10.0" -#(set! toplevel-score-handler print-score-with-defaults) - #(set! toplevel-music-handler - (lambda (p m) - (if (not (eq? (ly:music-property m 'void) #t)) - (print-score-with-defaults - p (scorify-music m p))))) +#(define default-toplevel-book-handler + print-book-with-defaults-as-systems ) + +#(ly:set-option (quote no-point-and-click)) \sourcefilename "my-source.ly" @@ -917,7 +969,7 @@ def test_basic_compare (): names = [d['name'] for d in dicts] - system ('lilypond -ddump-signatures --png -b eps ' + ' '.join (names)) + system ('lilypond -dseparate-log-files -ddump-signatures --png -b eps ' + ' '.join (names)) multipage_str = r''' @@ -931,7 +983,7 @@ def test_basic_compare (): open ('20multipage', 'w').write (multipage_str.replace ('c1', 'd1')) open ('19multipage', 'w').write ('#(set-global-staff-size 19.5)\n' + multipage_str) - system ('lilypond -ddump-signatures --png 19multipage 20multipage ') + system ('lilypond -dseparate-log-files -ddump-signatures --png 19multipage 20multipage ') test_compare_signatures (names) diff --git a/scm/define-grobs.scm b/scm/define-grobs.scm index 91803b3afa..adfde1ad59 100644 --- a/scm/define-grobs.scm +++ b/scm/define-grobs.scm @@ -1373,7 +1373,6 @@ ;; padding set in script definitions. (staff-padding . 0.25) - ;; (script-priority . 0) priorities for scripts, see script.scm (X-offset . ,ly:self-alignment-interface::centered-on-x-parent) (Y-offset . ,ly:side-position-interface::y-aligned-side) (side-axis . ,Y) diff --git a/scm/lily-library.scm b/scm/lily-library.scm index 52331aa04c..53b5e58f24 100644 --- a/scm/lily-library.scm +++ b/scm/lily-library.scm @@ -49,58 +49,34 @@ (define-public (moment-min a b) (if (ly:moment output hooks. -;; parser stuff. -(define-public (print-music-as-book parser music) - (let* ((head (ly:parser-lookup parser '$defaultheader)) - (book (ly:make-book (ly:parser-lookup parser '$defaultpaper) - head (scorify-music music parser)))) - (print-book-with-defaults parser book))) - -(define-public (print-score-as-book parser score) - (let* ((head (ly:parser-lookup parser '$defaultheader)) - (book (ly:make-book (ly:parser-lookup parser '$defaultpaper) - head score))) - (print-book-with-defaults parser book))) - -(define-public (print-score parser score) - (let* ((head (ly:parser-lookup parser '$defaultheader)) - (book (ly:make-book (ly:parser-lookup parser '$defaultpaper) - head score))) - (ly:parser-print-score parser book))) (define-public (collect-scores-for-book parser score) (ly:parser-define! parser 'toplevel-scores (cons score (ly:parser-lookup parser 'toplevel-scores)))) -(define-public (scorify-music music parser) - (for-each (lambda (func) - (set! music (func music parser))) - toplevel-music-functions) - - (ly:make-score music)) - (define-public (collect-music-for-book parser music) ;; discard music if its 'void property is true. (let ((void-music (ly:music-property music 'void))) (if (or (null? void-music) (not void-music)) (collect-scores-for-book parser (scorify-music music parser))))) +(define-public (scorify-music music parser) + "Preprocess MUSIC." + + (for-each (lambda (func) + (set! music (func music parser))) + toplevel-music-functions) + + (ly:make-score music)) (define (print-book-with parser book process-procedure) (let* @@ -608,14 +584,16 @@ possibly turned off." (define-public (version-not-seen-message input-file-name) (ly:message - (string-append - input-file-name ": 0: " (_ "warning: ") - (format #f - (_ "no \\version statement found, please add~afor future compatibility") - (format #f "\n\n\\version ~s\n\n" (lilypond-version)))))) + "~a:0: ~a: ~a" + input-file-name + (_ "warning: ") + (format #f + (_ "no \\version statement found, please add~afor future compatibility") + (format #f "\n\n\\version ~s\n\n" (lilypond-version))))) (define-public (old-relative-not-used-message input-file-name) (ly:message - (string-append - input-file-name ": 0: " (_ "warning: ") - (_ "old relative compatibility not used")))) + "~a:0: ~a: ~a" + input-file-name + (_ "warning: ") + (_ "old relative compatibility not used"))) diff --git a/scm/lily.scm b/scm/lily.scm index 798df7ece4..4c61a2757a 100644 --- a/scm/lily.scm +++ b/scm/lily.scm @@ -69,8 +69,8 @@ on errors, and print a stack trace.") (read-file-list #f "Read files to be processed from command line arguments") (safe #f "Run safely") - (strict-infinity-checking #f "If yes, crash on encountering Inf/NaN") - + (strict-infinity-checking #f "If yes, crash on encountering Inf/NaN.") + (separate-log-files #f "Output to FILE.log per file.") (ttf-verbosity 0 "how much verbosity for TTF font embedding?") @@ -579,6 +579,7 @@ The syntax is the same as `define*-public'." )) (let* ((failed '()) + (separate-logs (ly:get-option 'separate-log-files)) (start-measurements (ly:get-option 'dump-profile)) (handler (lambda (key failed-file) (set! failed (append (list failed-file) failed))))) @@ -589,6 +590,10 @@ The syntax is the same as `define*-public'." (gc) (if start-measurements (set! start-measurements (profile-measurements))) + + (if separate-logs + (ly:stderr-redirect (format "~a.log" (basename x ".ly")) "w")) + (lilypond-file handler x) (if start-measurements (dump-profile x start-measurements (profile-measurements))) @@ -601,6 +606,15 @@ The syntax is the same as `define*-public'." (ly:reset-all-fonts)))) files) + + ;; we want the failed-files notice in the aggregrate logfile. + (if (ly:get-option 'separate-logs) + (ly:stderr-redirect + (if (string-or-symbol? (ly:get-option 'log-file)) + (format "~a.log" (ly:get-option 'log-file)) + "/dev/tty") "a")) + + failed)) (define (lilypond-file handler file-name)