From 5b2fa67db8a2f13ac9b83b832c2629efe19d377b Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 3 Jan 2007 13:12:21 +0100 Subject: [PATCH] Refine test machinery. - new targets test-baseline, test-real-clean. - test-clean now only removes changed results. - document new conventions --- Documentation/topdocs/INSTALL.texi | 16 +++++-- GNUmakefile.in | 16 ++++--- buildscripts/output-distance.py | 72 +++++++++++++++++++----------- lily/general-scheme.cc | 1 + scripts/lilypond-book.py | 3 +- 5 files changed, 72 insertions(+), 36 deletions(-) diff --git a/Documentation/topdocs/INSTALL.texi b/Documentation/topdocs/INSTALL.texi index 6a7196fabd..23b1d21c10 100644 --- a/Documentation/topdocs/INSTALL.texi +++ b/Documentation/topdocs/INSTALL.texi @@ -164,15 +164,25 @@ program. This suite can be used to automatically check the impact of a change. This is done as follows @example - make ok-test + make test-baseline @emph{apply your changes, compile} - make test-clean make check @end example This will leave an HTML page @file{out/test-results/index.html}. This page shows all the important differences that your change introduced, -whether in the layout, the MIDI output, or error reporting. +whether in the layout, the MIDI output, or error reporting. + +To rerun tests, use + +@example + make test-clean @emph{## remove files differing from baseline} + make test-real-clean @emph{## remove all test results} +@end example + +@noindent +and then run @code{make check} again. + @section Building LilyPond diff --git a/GNUmakefile.in b/GNUmakefile.in index f28f408fba..9718820f62 100644 --- a/GNUmakefile.in +++ b/GNUmakefile.in @@ -205,19 +205,25 @@ $(config_h): config.hh.in @false -test-clean: - $(MAKE) -C input/regression/ out=test clean test: + rm input/regression/out-test/collated-files.html $(MAKE) -C input/regression/ out=test LILYPOND_BOOK_LILYPOND_FLAGS="--backend=eps --formats=ps $(LILYPOND_JOBS) -dseparate-log-files -dinclude-eps-fonts -dgs-load-fonts --header=texidoc -I $(top-src-dir)/input/manual -ddump-profile -dcheck-internal-types -ddump-signatures -danti-alias-factor=1" LILYPOND_BOOK_VERBOSE= out-test/collated-files.html @find input ly -name '*.ly' -print |grep -v 'out.*/' | xargs grep '\\version' -L | grep -v "standard input" |sed 's/^/**** Missing version: /g' -ok-test: test - mv input/regression/out-test input/regression/out-testok +test-baseline: test + rm -rf input/regression/out-test-baseline + mv input/regression/out-test input/regression/out-test-baseline RESULT_DIR=$(top-build-dir)/out/test-results/ local-check: test rm -rf $(RESULT_DIR) mkdir -p $(RESULT_DIR) - $(PYTHON) $(buildscript-dir)/output-distance.py --create-images --output-dir $(RESULT_DIR) input/regression/out-testok input/regression/out-test/ + $(PYTHON) $(buildscript-dir)/output-distance.py --create-images --output-dir $(RESULT_DIR) input/regression/out-test-baseline input/regression/out-test/ + +test-clean: + $(PYTHON) $(buildscript-dir)/output-distance.py --remove-changed input/regression/out-test-baseline input/regression/out-test/ + +test-real-clean: + $(MAKE) -C input/regression/ out=test clean diff --git a/buildscripts/output-distance.py b/buildscripts/output-distance.py index e6e62562fa..5d11a388c5 100644 --- a/buildscripts/output-distance.py +++ b/buildscripts/output-distance.py @@ -416,8 +416,9 @@ class SignatureFileLink (FileLink): self.original_name = '' self.base_names = ('','') self.system_links = {} + def name (self): - return self.original_name + return os.path.splitext (self.original_name)[0] def add_system_link (self, link, number): self.system_links[number] = link @@ -753,6 +754,26 @@ class ComparisonData: file_link.add_file_compare (f1, f2) + def remove_changed (self, dir, threshold): + (changed, below, unchanged) = self.thresholded_results (threshold) + for link in changed: + try: + system ('rm -f %s*' % link.base_names[1]) + except AttributeError: ### UGH. + system ('rm -f %s/%s*' % (dir, link.name ())) + def thresholded_results (self, threshold): + ## todo: support more scores. + results = [(link.distance(), link) + for link in self.file_links.values ()] + results.sort () + results.reverse () + + unchanged = [r for (d,r) in results if d == 0.0] + below = [r for (d,r) in results if threshold >= d > 0.0] + changed = [r for (d,r) in results if d > threshold] + + return (changed, below, unchanged) + def write_text_result_page (self, filename, threshold): out = None if filename == '': @@ -761,21 +782,15 @@ class ComparisonData: print 'writing "%s"' % filename out = open_write_file (filename) - ## todo: support more scores. - results = [(link.distance(), link) - for link in self.file_links.values ()] - results.sort () - results.reverse () + (changed, below, unchanged) = self.thresholded_results (threshold) - for (score, link) in results: - if score > threshold: - out.write (link.text_record_string ()) + for link in changed: + out.write (link.text_record_string ()) out.write ('\n\n') - out.write ('%d below threshold\n' % len ([1 for s,l in results - if threshold >= s > 0.0])) - out.write ('%d unchanged\n' % len ([1 for (s,l) in results if s == 0.0])) + out.write ('%d below threshold\n' % len (below)) + out.write ('%d unchanged\n' % len (unchanged)) def create_text_result_page (self, dir1, dir2, dest_dir, threshold): self.write_text_result_page (dest_dir + '/index.txt', threshold) @@ -783,18 +798,13 @@ class ComparisonData: def create_html_result_page (self, dir1, dir2, dest_dir, threshold): dir1 = dir1.replace ('//', '/') dir2 = dir2.replace ('//', '/') - - results = [(link.distance(), link) - for link in self.file_links.values ()] - results.sort () - results.reverse () + + (changed, below, unchanged) = self.thresholded_results (threshold) + html = '' old_prefix = os.path.split (dir1)[1] - for (score, link) in results: - if score <= threshold: - continue - + for link in changed: link.link_files_for_html (dir1, dir2, dest_dir) link.write_html_system_details (dir1, dir2, dest_dir) @@ -815,15 +825,12 @@ class ComparisonData: ''' % locals() html += ('

') - below_count =len ([1 for s,l in results - if threshold >= s > 0.0]) + below_count = len (below) if below_count: html += ('

%d below threshold

' % below_count) - - html += ('

%d unchanged

' - % len ([1 for (s,l) in results if s == 0.0])) - + + html += ('

%d unchanged

' % len (unchanged)) dest_file = dest_dir + '/index.html' open_write_file (dest_file).write (html) @@ -836,6 +843,10 @@ def compare_trees (dir1, dir2, dest_dir, threshold): data.compare_trees (dir1, dir2) data.print_results (threshold) + if options.remove_changed: + data.remove_changed (dir2, threshold) + return + if os.path.isdir (dest_dir): system ('rm -rf %s '% dest_dir) @@ -1071,6 +1082,13 @@ def main (): type="float", help='threshold for geometric distance') + + p.add_option ('--remove-changed', + dest="remove_changed", + default=False, + action="store_true", + help="Remove all files from tree2 that are over the threshold.") + p.add_option ('--no-compare-images', dest="compare_images", default=True, diff --git a/lily/general-scheme.cc b/lily/general-scheme.cc index 5e161474dc..592c1494fc 100644 --- a/lily/general-scheme.cc +++ b/lily/general-scheme.cc @@ -303,6 +303,7 @@ LY_DEFINE (ly_stderr_redirect, "ly:stderr-redirect", m = ly_scm2newstr (mode, 0); /* dup2 and (fileno (current-error-port)) do not work with mingw'c gcc -mwindows. */ + fflush (stderr); freopen (ly_scm2newstr (file_name, 0), m, stderr); return SCM_UNSPECIFIED; } diff --git a/scripts/lilypond-book.py b/scripts/lilypond-book.py index 6a8fcc1963..d7f3069479 100644 --- a/scripts/lilypond-book.py +++ b/scripts/lilypond-book.py @@ -1565,6 +1565,7 @@ def write_file_map (lys, name): snippet_map = open ('snippet-map.ly', 'w') snippet_map.write (""" #(define version-seen #t) +#(define output-empty-score-list #f) #(ly:add-file-name-alist '( """) for ly in lys: @@ -1577,7 +1578,7 @@ def write_file_map (lys, name): def do_process_cmd (chunks, input_name): all_lys = filter (lambda x: is_derived_class (x.__class__, Lilypond_snippet), - chunks) + chunks) write_file_map (all_lys, input_name) ly_outdated = filter (lambda x: is_derived_class (x.__class__, -- 2.39.5