Merge branch 'translation'

[lilypond.git] / lily / lexer.ll

diff --git a/lily/lexer.ll b/lily/lexer.ll
index 71473faa30fc9d4375de779494b183cff29416fb..51a33fbb4b2a57b446ef954c4af9c1b962f27003 100644 (file)
--- a/lily/lexer.ll
+++ b/lily/lexer.ll
@@ -123,7 +123,6 @@ SCM (* scm_parse_error_handler) (void *);
 %option never-interactive 
 %option warn
 
-%x extratoken
 %x chords
 %x figures
 %x incl
@@ -186,36 +185,6 @@ BOM_UTF8   \357\273\277
        // swallow and ignore carriage returns
 }
 
-<extratoken>{ANY_CHAR} {
-  /* Generate a token without swallowing anything */
-
-  /* First unswallow the eaten character */
-  add_lexed_char (-YYLeng ());
-  yyless (0);
-
-  /* produce requested token */
-  int type = scm_to_int (scm_caar (extra_tokens_));
-  yylval = scm_cdar (extra_tokens_);
-  extra_tokens_ = scm_cdr (extra_tokens_);
-  if (scm_is_null (extra_tokens_))
-    yy_pop_state ();
-
-  return type;
-}
-
-<extratoken><<EOF>>    {
-  /* Generate a token without swallowing anything */
-
-  /* produce requested token */
-  int type = scm_to_int (scm_caar (extra_tokens_));
-  yylval = scm_cdar (extra_tokens_);
-  extra_tokens_ = scm_cdr (extra_tokens_);
-  if (scm_is_null (extra_tokens_))
-    yy_pop_state ();
-
-  return type;
-}
-
    /* Use the trailing context feature. Otherwise, the BOM will not be
       found if the file starts with an identifier definition. */
 <INITIAL,chords,lyrics,figures,notes>{BOM_UTF8}/.* {
@@ -717,17 +686,17 @@ BOM_UTF8  \357\273\277
                // value (for token type MARKUP_FUNCTION or
                // MARKUP_LIST_FUNCTION).
 
-               push_extra_token(EXPECT_NO_MORE_ARGS);
+               push_extra_token (here_input (), EXPECT_NO_MORE_ARGS);
                s = scm_cdr(s);
                for (; scm_is_pair(s); s = scm_cdr(s)) {
                  SCM predicate = scm_car(s);
 
                  if (predicate == ly_lily_module_constant ("markup-list?"))
-                   push_extra_token(EXPECT_MARKUP_LIST);
+                   push_extra_token (here_input (), EXPECT_MARKUP_LIST);
                  else if (predicate == ly_lily_module_constant ("markup?"))
-                   push_extra_token(EXPECT_MARKUP);
+                   push_extra_token (here_input (), EXPECT_MARKUP);
                  else
-                   push_extra_token(EXPECT_SCM, predicate);
+                   push_extra_token (here_input (), EXPECT_SCM, predicate);
                }
                return token_type;
        }
@@ -846,15 +815,25 @@ BOM_UTF8  \357\273\277
 /* Make the lexer generate a token of the given type as the next token. 
  TODO: make it possible to define a value for the token as well */
 void
-Lily_lexer::push_extra_token (int token_type, SCM scm)
+Lily_lexer::push_extra_token (Input const &where, int token_type, SCM scm)
+{
+       extra_tokens_ = scm_cons (scm_cons2 (make_input (where),
+                                            scm_from_int (token_type),
+                                            scm), extra_tokens_);
+}
+
+int
+Lily_lexer::pop_extra_token ()
 {
        if (scm_is_null (extra_tokens_))
-       {
-               if (YY_START != extratoken)
-                       hidden_state_ = YY_START;
-               yy_push_state (extratoken);
-       }
-       extra_tokens_ = scm_acons (scm_from_int (token_type), scm, extra_tokens_);
+               return -1;
+
+  /* produce requested token */
+       yylloc = *unsmob_input (scm_caar (extra_tokens_));
+       int type = scm_to_int (scm_cadar (extra_tokens_));
+       yylval = scm_cddar (extra_tokens_);
+       extra_tokens_ = scm_cdr (extra_tokens_);
+       return type;
 }
 
 void
@@ -895,32 +874,17 @@ Lily_lexer::push_markup_state ()
 void
 Lily_lexer::push_note_state (SCM alist)
 {
-       bool extra = (YYSTATE == extratoken);
-
        SCM p = scm_assq (alist, pitchname_tab_stack_);
 
-       if (extra)
-               yy_pop_state ();
-
        if (scm_is_false (p))
                p = scm_cons (alist, alist_to_hashq (alist));
        pitchname_tab_stack_ = scm_cons (p, pitchname_tab_stack_);
        yy_push_state (notes);
-
-       if (extra) {
-               hidden_state_ = YYSTATE;
-               yy_push_state (extratoken);
-       }
 }
 
 void
 Lily_lexer::pop_state ()
 {
-       bool extra = (YYSTATE == extratoken);
-
-       if (extra)
-               yy_pop_state ();
-
        if (YYSTATE == notes || YYSTATE == chords)
                pitchname_tab_stack_ = scm_cdr (pitchname_tab_stack_);
 
@@ -928,10 +892,6 @@ Lily_lexer::pop_state ()
        if (YYSTATE != maininput)
                yy_pop_state ();
 
-       if (extra) {
-               hidden_state_ = YYSTATE;
-               yy_push_state (extratoken);
-       }
 }
 
 int
@@ -1017,7 +977,7 @@ Lily_lexer::scan_scm_id (SCM sid)
                        funtype = SCM_FUNCTION;
                else programming_error ("Bad syntax function predicate");
 
-               push_extra_token (EXPECT_NO_MORE_ARGS);
+               push_extra_token (here_input (), EXPECT_NO_MORE_ARGS);
                for (s = scm_cdr (s); scm_is_pair (s); s = scm_cdr (s))
                {
                        SCM optional = SCM_UNDEFINED;
@@ -1030,14 +990,14 @@ Lily_lexer::scan_scm_id (SCM sid)
                        }
                        
                        if (ly_is_procedure (cs))
-                               push_extra_token (EXPECT_SCM, cs);
+                               push_extra_token (here_input (), EXPECT_SCM, cs);
                        else
                        {
                                programming_error ("Function parameter without type-checking predicate");
                                continue;
                        }
                        if (!scm_is_eq (optional, SCM_UNDEFINED))
-                               push_extra_token (EXPECT_OPTIONAL, optional);
+                               push_extra_token (here_input (), EXPECT_OPTIONAL, optional);
                }
                return funtype;
        }
@@ -1075,10 +1035,7 @@ Lily_lexer::scan_bare_word (const string &str)
 int
 Lily_lexer::get_state () const
 {
-       if (YY_START == extratoken)
-               return hidden_state_;
-       else
-               return YY_START;
+       return YY_START;
 }
 
 bool
@@ -1152,10 +1109,12 @@ Lily_lexer::eval_scm (SCM readerdata, char extra_token)
                                case '$':
                                        token = scan_scm_id (v);
                                        if (!scm_is_eq (yylval, SCM_UNSPECIFIED))
-                                               push_extra_token (token, yylval);
+                                               push_extra_token (here_input (),
+                                                                 token, yylval);
                                        break;
                                case '#':
-                                       push_extra_token (SCM_IDENTIFIER, v);
+                                       push_extra_token (here_input (),
+                                                         SCM_IDENTIFIER, v);
                                        break;
                                }
                        }