Parser: let MYREPARSE and MYBACKUP back up even in presence of lookahead

This implies that the lookahead token must not yet have made an impact
on the state stack other than causing the reduction of the current
rule, or switching the lookahead token while Bison is mulling it over
will cause trouble.

Since an LALR(1) works on the top of the stack, this can usually be
guaranteed.  The "Too much lookahead" message is not produced.  If
these constructs are misused, the parser should typically complain
about the replacement token being unexpected.

The advantage is the ability to vastly simplify syntax rules that up
to now have to avoid lookahead tokens with a lot of drudgework.

diff --git a/lily/parser.yy b/lily/parser.yy
index 385c906724bd9810bdd90dac87d5c8a82b290c45..216ca3b225f7bf359189507b88f11ecd0d7adede 100644 (file)
--- a/lily/parser.yy
+++ b/lily/parser.yy
@@ -146,32 +146,31 @@ Lily_parser::parser_error (Input const *i, Lily_parser *parser, SCM *, const str
        parser->parser_error (*i, s);
 }
 
+// The following are somewhat precarious constructs as they may change
+// the value of the lookahead token.  That implies that the lookahead
+// token must not yet have made an impact on the state stack other
+// than causing the reduction of the current rule, or switching the
+// lookahead token while Bison is mulling it over will cause trouble.
+
 #define MYBACKUP(Token, Value, Location)                               \
-do                                                                     \
-       if (yychar == YYEMPTY)                                          \
-       {                                                               \
+       do {                                                            \
+               if (yychar != YYEMPTY)                                  \
+                       parser->lexer_->push_extra_token (yychar, yylval); \
                if (Token)                                              \
                        parser->lexer_->push_extra_token (Token, Value); \
                parser->lexer_->push_extra_token (BACKUP);              \
-       } else {                                                        \
-               parser->parser_error                                    \
-                       (Location, _("Too much lookahead"));            \
-       }                                                               \
-while (0)
+               yychar = YYEMPTY;                                       \
+       } while (0)
 
 
 #define MYREPARSE(Location, Pred, Token, Value)                                \
-do                                                                     \
-       if (yychar == YYEMPTY)                                          \
-       {                                                               \
+       do {                                                            \
+               if (yychar != YYEMPTY)                                  \
+                       parser->lexer_->push_extra_token (yychar, yylval); \
                parser->lexer_->push_extra_token (Token, Value);        \
-               parser->lexer_->push_extra_token (REPARSE,              \
-                                                 Pred);                \
-       } else {                                                        \
-               parser->parser_error                                    \
-                       (Location, _("Too much lookahead"));            \
-       }                                                               \
-while (0)
+               parser->lexer_->push_extra_token (REPARSE, Pred);       \
+               yychar = YYEMPTY;                                       \
+       } while (0)
 
 %}