From 5bf2b8b2d50667582a61abea0e6257be0d033131 Mon Sep 17 00:00:00 2001
From: David Kastrup <dak@gnu.org>
Date: Wed, 11 Dec 2013 21:35:53 +0100
Subject: [PATCH] Move the extra_token mechanism out of the lexer proper

Calling lex->pop_extra_token explicitly in the parser's yylex function
makes things considerably more robust.  It also allows pushing and
popping the EOF condition, necessary for backing up right at the end
of a file, like with ##{ \relative { c d e f } #} where the music
expression is checked against being a pitch, and then is backed up.
---
 lily/include/lily-lexer.hh |  2 +-
 lily/lexer.ll              | 74 ++++++++------------------------------
 lily/parser.yy             |  3 ++
 3 files changed, 18 insertions(+), 61 deletions(-)

diff --git a/lily/include/lily-lexer.hh b/lily/include/lily-lexer.hh
index 4bc4cbe887..ce428603cb 100644
--- a/lily/include/lily-lexer.hh
+++ b/lily/include/lily-lexer.hh
@@ -49,7 +49,6 @@ private:
   Keyword_table *keytable_;
   SCM scopes_;
   SCM start_module_;
-  int hidden_state_;
   Input override_input_;
   SCM eval_scm (SCM, char extra_token = 0);
 public:
@@ -95,6 +94,7 @@ public:
   SCM lookup_identifier (const string &s);
   SCM lookup_identifier_symbol (SCM s);
   void push_extra_token (int token_type, SCM scm = SCM_UNSPECIFIED);
+  int pop_extra_token ();
   void push_chord_state (SCM alist);
   void push_figuredbass_state ();
   void push_lyric_state ();
diff --git a/lily/lexer.ll b/lily/lexer.ll
index 71473faa30..a360de48c1 100644
--- a/lily/lexer.ll
+++ b/lily/lexer.ll
@@ -123,7 +123,6 @@ SCM (* scm_parse_error_handler) (void *);
 %option never-interactive 
 %option warn
 
-%x extratoken
 %x chords
 %x figures
 %x incl
@@ -186,36 +185,6 @@ BOM_UTF8	\357\273\277
 	// swallow and ignore carriage returns
 }
 
-<extratoken>{ANY_CHAR}	{
-  /* Generate a token without swallowing anything */
-
-  /* First unswallow the eaten character */
-  add_lexed_char (-YYLeng ());
-  yyless (0);
-
-  /* produce requested token */
-  int type = scm_to_int (scm_caar (extra_tokens_));
-  yylval = scm_cdar (extra_tokens_);
-  extra_tokens_ = scm_cdr (extra_tokens_);
-  if (scm_is_null (extra_tokens_))
-    yy_pop_state ();
-
-  return type;
-}
-
-<extratoken><<EOF>>	{
-  /* Generate a token without swallowing anything */
-
-  /* produce requested token */
-  int type = scm_to_int (scm_caar (extra_tokens_));
-  yylval = scm_cdar (extra_tokens_);
-  extra_tokens_ = scm_cdr (extra_tokens_);
-  if (scm_is_null (extra_tokens_))
-    yy_pop_state ();
-
-  return type;
-}
-
    /* Use the trailing context feature. Otherwise, the BOM will not be
       found if the file starts with an identifier definition. */
 <INITIAL,chords,lyrics,figures,notes>{BOM_UTF8}/.* {
@@ -848,15 +817,22 @@ BOM_UTF8	\357\273\277
 void
 Lily_lexer::push_extra_token (int token_type, SCM scm)
 {
-	if (scm_is_null (extra_tokens_))
-	{
-		if (YY_START != extratoken)
-			hidden_state_ = YY_START;
-		yy_push_state (extratoken);
-	}
 	extra_tokens_ = scm_acons (scm_from_int (token_type), scm, extra_tokens_);
 }
 
+int
+Lily_lexer::pop_extra_token ()
+{
+	if (scm_is_null (extra_tokens_))
+		return -1;
+
+  /* produce requested token */
+	int type = scm_to_int (scm_caar (extra_tokens_));
+	yylval = scm_cdar (extra_tokens_);
+	extra_tokens_ = scm_cdr (extra_tokens_);
+	return type;
+}
+
 void
 Lily_lexer::push_chord_state (SCM alist)
 {
@@ -895,32 +871,17 @@ Lily_lexer::push_markup_state ()
 void
 Lily_lexer::push_note_state (SCM alist)
 {
-	bool extra = (YYSTATE == extratoken);
-
 	SCM p = scm_assq (alist, pitchname_tab_stack_);
 
-	if (extra)
-		yy_pop_state ();
-
 	if (scm_is_false (p))
 		p = scm_cons (alist, alist_to_hashq (alist));
 	pitchname_tab_stack_ = scm_cons (p, pitchname_tab_stack_);
 	yy_push_state (notes);
-
-	if (extra) {
-		hidden_state_ = YYSTATE;
-		yy_push_state (extratoken);
-	}
 }
 
 void
 Lily_lexer::pop_state ()
 {
-	bool extra = (YYSTATE == extratoken);
-
-	if (extra)
-		yy_pop_state ();
-
 	if (YYSTATE == notes || YYSTATE == chords)
 		pitchname_tab_stack_ = scm_cdr (pitchname_tab_stack_);
 
@@ -928,10 +889,6 @@ Lily_lexer::pop_state ()
 	if (YYSTATE != maininput)
 		yy_pop_state ();
 
-	if (extra) {
-		hidden_state_ = YYSTATE;
-		yy_push_state (extratoken);
-	}
 }
 
 int
@@ -1075,10 +1032,7 @@ Lily_lexer::scan_bare_word (const string &str)
 int
 Lily_lexer::get_state () const
 {
-	if (YY_START == extratoken)
-		return hidden_state_;
-	else
-		return YY_START;
+	return YY_START;
 }
 
 bool
diff --git a/lily/parser.yy b/lily/parser.yy
index 7891a3b22d..385c906724 100644
--- a/lily/parser.yy
+++ b/lily/parser.yy
@@ -3970,6 +3970,9 @@ yylex (YYSTYPE *s, YYLTYPE *loc, Lily_parser *parser)
 
 	lex->lexval_ = s;
 	lex->lexloc_ = loc;
+	int tok = lex->pop_extra_token ();
+	if (tok >= 0)
+		return tok;
 	lex->prepare_for_next_token ();
 	return lex->yylex ();
 }
-- 
2.39.5