Issue 3297: Make lexer more robust against unexpected EOF in main input

This approach works by letting Lily_lexer::pop_state not cross the
maininput threshold.  Since Lily_lexer::pop_state is the "cooked" form
of yy_pop_state (also hiding the extratoken machinery), this keeps the
"maininput" state firmly under the lexer's control and avoids exiting
it prematurely.

diff --git a/lily/lexer.ll b/lily/lexer.ll
index 18b999da82f20f2a3045f796b37322ff5fd3d65d..cebbd19627190c2504fbe5b547f8be83435cd16f 100644 (file)
--- a/lily/lexer.ll
+++ b/lily/lexer.ll
@@ -736,10 +736,11 @@ BOM_UTF8  \357\273\277
                        {
                                LexerError (_ ("Unfinished main input").c_str ());
                                do {
-                                       pop_state ();
+                                       yy_pop_state ();
                                } while (YYSTATE != maininput);
                        }
-                       pop_state ();
+                       extra_tokens_ = SCM_EOL;
+                       yy_pop_state ();
                }
                if (!close_input () || !is_main_input_)
                /* Returns YY_NULL */
@@ -890,7 +891,9 @@ Lily_lexer::pop_state ()
        if (YYSTATE == notes || YYSTATE == chords)
                pitchname_tab_stack_ = scm_cdr (pitchname_tab_stack_);
 
-       yy_pop_state ();
+       // don't cross the maininput threshold
+       if (YYSTATE != maininput)
+               yy_pop_state ();
 
        if (extra) {
                hidden_state_ = YYSTATE;