Refine test machinery.

- new targets test-baseline, test-real-clean.

- test-clean now only removes changed results.

- document new conventions

diff --git a/Documentation/topdocs/INSTALL.texi b/Documentation/topdocs/INSTALL.texi
index 6a7196fabd80e1c85de4f39aecbfa7fe426d92dc..23b1d21c1095fbf62f149eef5b8fa10250ce9d78 100644 (file)
--- a/Documentation/topdocs/INSTALL.texi
+++ b/Documentation/topdocs/INSTALL.texi
@@ -164,15 +164,25 @@ program. This suite can be used to automatically check the impact of a
 change. This is done as follows
 
 @example
-  make ok-test
+  make test-baseline
   @emph{apply your changes, compile}
-  make test-clean
   make check
 @end example
 
 This will leave an HTML page @file{out/test-results/index.html}.  This
 page shows all the important differences that your change introduced,
-whether in the layout, the MIDI output, or error reporting.
+whether in the layout, the MIDI output, or error reporting.  
+
+To rerun tests, use
+
+@example
+  make test-clean          @emph{## remove files differing from baseline}
+  make test-real-clean     @emph{## remove all test results}
+@end example
+
+@noindent
+and then run @code{make check} again.
+
 
 @section Building LilyPond
 

diff --git a/GNUmakefile.in b/GNUmakefile.in
index f28f408fba121738c0c85a5dba35a4ce930e74ef..9718820f6259dc0a3b9999e8f584fd018c7c9c09 100644 (file)
--- a/GNUmakefile.in
+++ b/GNUmakefile.in
@@ -205,19 +205,25 @@ $(config_h): config.hh.in
        @false
 
 
-test-clean:
-       $(MAKE) -C input/regression/ out=test clean
 
 test:
+       rm input/regression/out-test/collated-files.html
        $(MAKE) -C input/regression/ out=test LILYPOND_BOOK_LILYPOND_FLAGS="--backend=eps --formats=ps $(LILYPOND_JOBS) -dseparate-log-files -dinclude-eps-fonts -dgs-load-fonts --header=texidoc -I $(top-src-dir)/input/manual -ddump-profile -dcheck-internal-types -ddump-signatures -danti-alias-factor=1" LILYPOND_BOOK_VERBOSE= out-test/collated-files.html 
        @find input ly -name '*.ly' -print |grep -v 'out.*/' | xargs grep '\\version' -L | grep -v "standard input" |sed 's/^/**** Missing version: /g' 
 
-ok-test: test
-       mv input/regression/out-test input/regression/out-testok
+test-baseline: test
+       rm -rf input/regression/out-test-baseline
+       mv input/regression/out-test input/regression/out-test-baseline
 
 RESULT_DIR=$(top-build-dir)/out/test-results/
 local-check: test
        rm -rf $(RESULT_DIR)
        mkdir -p $(RESULT_DIR)
-       $(PYTHON) $(buildscript-dir)/output-distance.py --create-images --output-dir $(RESULT_DIR) input/regression/out-testok input/regression/out-test/
+       $(PYTHON) $(buildscript-dir)/output-distance.py --create-images --output-dir $(RESULT_DIR) input/regression/out-test-baseline input/regression/out-test/
+
 
+test-clean:
+       $(PYTHON) $(buildscript-dir)/output-distance.py --remove-changed input/regression/out-test-baseline input/regression/out-test/
+
+test-real-clean:
+       $(MAKE) -C input/regression/ out=test clean

diff --git a/buildscripts/output-distance.py b/buildscripts/output-distance.py
index e6e62562fa3177576e587e047a7b8edc50484401..5d11a388c5a98fe2133c1ea29a7883eaf171250d 100644 (file)
--- a/buildscripts/output-distance.py
+++ b/buildscripts/output-distance.py
@@ -416,8 +416,9 @@ class SignatureFileLink (FileLink):
         self.original_name = ''
         self.base_names = ('','')
         self.system_links = {}
+        
     def name (self):
-        return self.original_name
+        return os.path.splitext (self.original_name)[0]
     
     def add_system_link (self, link, number):
         self.system_links[number] = link
@@ -753,6 +754,26 @@ class ComparisonData:
 
         file_link.add_file_compare (f1, f2)
 
+    def remove_changed (self, dir, threshold):
+        (changed, below, unchanged) = self.thresholded_results (threshold)
+        for link in changed:
+            try:
+                system ('rm -f %s*' % link.base_names[1])
+            except AttributeError: ### UGH.
+                system ('rm -f %s/%s*' % (dir, link.name ()))
+    def thresholded_results (self, threshold):
+        ## todo: support more scores.
+        results = [(link.distance(), link)
+                   for link in self.file_links.values ()]
+        results.sort ()
+        results.reverse ()
+
+        unchanged = [r for (d,r) in results if d == 0.0]
+        below = [r for (d,r) in results if threshold >= d > 0.0]
+        changed = [r for (d,r) in results if d > threshold]
+
+        return (changed, below, unchanged)
+                
     def write_text_result_page (self, filename, threshold):
         out = None
         if filename == '':
@@ -761,21 +782,15 @@ class ComparisonData:
             print 'writing "%s"' % filename
             out = open_write_file (filename)
 
-        ## todo: support more scores.
-        results = [(link.distance(), link)
-                   for link in self.file_links.values ()]
-        results.sort ()
-        results.reverse ()
+        (changed, below, unchanged) = self.thresholded_results (threshold)
 
         
-        for (score, link) in results:
-            if score > threshold:
-                out.write (link.text_record_string ())
+        for link in changed:
+            out.write (link.text_record_string ())
 
         out.write ('\n\n')
-        out.write ('%d below threshold\n' % len ([1 for s,l  in results
-                                                    if threshold >=  s > 0.0]))
-        out.write ('%d unchanged\n' % len ([1 for (s,l) in results if s == 0.0]))
+        out.write ('%d below threshold\n' % len (below))
+        out.write ('%d unchanged\n' % len (unchanged))
         
     def create_text_result_page (self, dir1, dir2, dest_dir, threshold):
         self.write_text_result_page (dest_dir + '/index.txt', threshold)
@@ -783,18 +798,13 @@ class ComparisonData:
     def create_html_result_page (self, dir1, dir2, dest_dir, threshold):
         dir1 = dir1.replace ('//', '/')
         dir2 = dir2.replace ('//', '/')
-        
-        results = [(link.distance(), link)
-                   for link in self.file_links.values ()]
-        results.sort ()
-        results.reverse ()
+
+        (changed, below, unchanged) = self.thresholded_results (threshold)
+
 
         html = ''
         old_prefix = os.path.split (dir1)[1]
-        for (score, link) in results:
-            if score <= threshold:
-                continue
-
+        for link in changed:
             link.link_files_for_html (dir1, dir2, dest_dir) 
             link.write_html_system_details (dir1, dir2, dest_dir)
             
@@ -815,15 +825,12 @@ class ComparisonData:
 </html>''' % locals()
 
         html += ('<p>')
-        below_count  =len ([1 for s,l  in results
-                            if threshold >=  s > 0.0])
+        below_count = len (below)
 
         if below_count:
             html += ('<p>%d below threshold</p>' % below_count)
-
-        html += ('<p>%d unchanged</p>'
-                 % len ([1 for (s,l) in results if s == 0.0]))
-
+            
+        html += ('<p>%d unchanged</p>' % len (unchanged))
 
         dest_file = dest_dir + '/index.html'
         open_write_file (dest_file).write (html)
@@ -836,6 +843,10 @@ def compare_trees (dir1, dir2, dest_dir, threshold):
     data.compare_trees (dir1, dir2)
     data.print_results (threshold)
 
+    if options.remove_changed:
+        data.remove_changed (dir2, threshold)
+        return
+    
     if os.path.isdir (dest_dir):
         system ('rm -rf %s '% dest_dir)
 
@@ -1071,6 +1082,13 @@ def main ():
                   type="float",
                   help='threshold for geometric distance')
 
+
+    p.add_option ('--remove-changed',
+                  dest="remove_changed",
+                  default=False,
+                  action="store_true",
+                  help="Remove all files from tree2 that are over the threshold.")
+
     p.add_option ('--no-compare-images',
                   dest="compare_images",
                   default=True,

diff --git a/lily/general-scheme.cc b/lily/general-scheme.cc
index 5e161474dc312fbebb1b8c7a5a1e02cbbc999841..592c1494fccff4baef6acef6671361ba9ec46cab 100644 (file)
--- a/lily/general-scheme.cc
+++ b/lily/general-scheme.cc
@@ -303,6 +303,7 @@ LY_DEFINE (ly_stderr_redirect, "ly:stderr-redirect",
     m = ly_scm2newstr (mode, 0);
   /* dup2 and (fileno (current-error-port)) do not work with mingw'c
      gcc -mwindows.  */
+  fflush (stderr); 
   freopen (ly_scm2newstr (file_name, 0), m, stderr);
   return SCM_UNSPECIFIED;
 }

diff --git a/scripts/lilypond-book.py b/scripts/lilypond-book.py
index 6a8fcc1963eff910910707d0226c6addcd3dbb12..d7f306947983243a672a588d6e33849717579083 100644 (file)
--- a/scripts/lilypond-book.py
+++ b/scripts/lilypond-book.py
@@ -1565,6 +1565,7 @@ def write_file_map (lys, name):
     snippet_map = open ('snippet-map.ly', 'w')
     snippet_map.write ("""
 #(define version-seen #t)
+#(define output-empty-score-list #f)
 #(ly:add-file-name-alist '(
 """)
     for ly in lys:
@@ -1577,7 +1578,7 @@ def write_file_map (lys, name):
 def do_process_cmd (chunks, input_name):
     all_lys = filter (lambda x: is_derived_class (x.__class__,
                            Lilypond_snippet),
-             chunks)
+                      chunks)
 
     write_file_map (all_lys, input_name)
     ly_outdated = filter (lambda x: is_derived_class (x.__class__,