Add regtests for issue 1650

[lilypond.git] / lily / lexer.ll

diff --git a/lily/lexer.ll b/lily/lexer.ll
index c2d8b6baeff1273520aa7a68eb32492ad286ec3e..22959fc99cfeeb9c72236d45563902f048263236 100644 (file)
--- a/lily/lexer.ll
+++ b/lily/lexer.ll
@@ -152,11 +152,10 @@ AA                {A}|_
 N              [0-9]
 AN             {AA}|{N}
 ANY_CHAR       (.|\n)
-PUNCT          [?!:'`]
-ACCENT         \\[`'"^]
+PUNCT          [][()?!:'`]
 SPECIAL_CHAR           [&@]
 NATIONAL       [\001-\006\021-\027\031\036]
-TEX            {AA}|-|{PUNCT}|{ACCENT}|{NATIONAL}|{SPECIAL_CHAR}
+TEX            {AA}|-|{PUNCT}|{NATIONAL}|{SPECIAL_CHAR}
 DASHED_WORD            {A}({AN}|-)*
 DASHED_KEY_WORD                \\{DASHED_WORD}
 
@@ -277,7 +276,7 @@ BOM_UTF8    \357\273\277
 
 
 }
-<sourcefilename>\"[^"]*\"     {
+<sourcefilename>\"[^""]*\"     {
        string s (YYText_utf8 () + 1);
        s = s.substr (0, s.rfind ('\"'));
 
@@ -439,11 +438,11 @@ BOM_UTF8  \357\273\277
        }
        char_count_stack_.back () += n;
 
-       sval = eval_scm (sval);
-               
+       sval = eval_scm (sval, '$');
+
        int token = scan_scm_id (sval);
        if (!scm_is_eq (yylval.scm, SCM_UNSPECIFIED))
-         return token;
+               return token;
 }
 
 <INITIAL,notes,lyrics>{ 
@@ -529,10 +528,6 @@ BOM_UTF8   \357\273\277
                yylval.scm =  scan_fraction (YYText ());
                return FRACTION;
        }
-       {UNSIGNED}/\/[^0-9] { // backup rule
-               yylval.scm = scm_c_read_string (YYText ());
-               return UNSIGNED;
-       }
        {UNSIGNED}/\/   | // backup rule
        {UNSIGNED}              {
                yylval.scm = scm_c_read_string (YYText ());
@@ -574,10 +569,6 @@ BOM_UTF8   \357\273\277
                yylval.scm =  scan_fraction (YYText ());
                return FRACTION;
        }
-       {UNSIGNED}/\/[^0-9] { // backup rule
-               yylval.scm = scm_c_read_string (YYText ());
-               return UNSIGNED;
-       }
        {UNSIGNED}/\/   | // backup rule
        {UNSIGNED}              {
                yylval.scm = scm_c_read_string (YYText ());
@@ -711,16 +702,18 @@ BOM_UTF8  \357\273\277
        }
 }
 
+{FRACTION}     {
+       yylval.scm =  scan_fraction (YYText ());
+       return FRACTION;
+}
+
 -{UNSIGNED}    | // backup rule
 {REAL}         {
        yylval.scm = scm_c_read_string (YYText ());
        return REAL;
 }
--\.    { // backup rule
-       yylval.scm = scm_from_double (0.0);
-       return REAL;
-}
 
+{UNSIGNED}/\/  | // backup rule
 {UNSIGNED}     {
        yylval.scm = scm_c_read_string (YYText ());
        return UNSIGNED;
@@ -731,6 +724,8 @@ BOM_UTF8    \357\273\277
 
        return YYText ()[0];
 }
+
+-/\.   | // backup rule
 [*:=]          {
        char c = YYText ()[0];
 
@@ -794,9 +789,13 @@ Lily_lexer::push_extra_token (int token_type, SCM scm)
 }
 
 void
-Lily_lexer::push_chord_state (SCM tab)
+Lily_lexer::push_chord_state (SCM alist)
 {
-       pitchname_tab_stack_ = scm_cons (tab, pitchname_tab_stack_);
+       SCM p = scm_assq (alist, pitchname_tab_stack_);
+
+       if (scm_is_false (p))
+               p = scm_cons (alist, alist_to_hashq (alist));
+       pitchname_tab_stack_ = scm_cons (p, pitchname_tab_stack_);
        yy_push_state (chords);
 }
 
@@ -825,9 +824,13 @@ Lily_lexer::push_markup_state ()
 }
 
 void
-Lily_lexer::push_note_state (SCM tab)
+Lily_lexer::push_note_state (SCM alist)
 {
-       pitchname_tab_stack_ = scm_cons (tab, pitchname_tab_stack_);
+       SCM p = scm_assq (alist, pitchname_tab_stack_);
+
+       if (scm_is_false (p))
+               p = scm_cons (alist, alist_to_hashq (alist));
+       pitchname_tab_stack_ = scm_cons (p, pitchname_tab_stack_);
        yy_push_state (notes);
 }
 
@@ -890,9 +893,9 @@ Lily_lexer::scan_scm_id (SCM sid)
        {
                int funtype = SCM_FUNCTION;
 
-               yylval.scm = get_music_function_transform (sid);
+               yylval.scm = sid;
 
-               SCM s = scm_object_property (yylval.scm, ly_symbol2scm ("music-function-signature"));
+               SCM s = get_music_function_signature (sid);
                SCM cs = scm_car (s);
 
                if (scm_is_pair (cs))
@@ -947,7 +950,7 @@ Lily_lexer::scan_bare_word (string str)
        if ((YYSTATE == notes) || (YYSTATE == chords)) {
                SCM handle = SCM_BOOL_F;
                if (scm_is_pair (pitchname_tab_stack_))
-                       handle = scm_hashq_get_handle (scm_car (pitchname_tab_stack_), sym);
+                       handle = scm_hashq_get_handle (scm_cdar (pitchname_tab_stack_), sym);
                
                if (scm_is_pair (handle)) {
                        yylval.scm = scm_cdr (handle);
@@ -1000,8 +1003,15 @@ Lily_lexer::is_figure_state () const
        return get_state () == figures;
 }
 
+// The extra_token parameter specifies how to convert multiple values
+// into additional tokens.  For '#', additional values get pushed as
+// SCM_IDENTIFIER.  For '$', they get checked for their type and get
+// pushed as a corresponding *_IDENTIFIER token.  Since the latter
+// tampers with yylval, it can only be done from the lexer itself, so
+// this function is private.
+
 SCM
-Lily_lexer::eval_scm (SCM readerdata)
+Lily_lexer::eval_scm (SCM readerdata, char extra_token)
 {
        SCM sval = SCM_UNDEFINED;
 
@@ -1018,6 +1028,33 @@ Lily_lexer::eval_scm (SCM readerdata)
                error_level_ = 1;
                return SCM_UNSPECIFIED;
        }
+
+       if (extra_token && SCM_VALUESP (sval))
+       {
+               sval = scm_struct_ref (sval, SCM_INUM0);
+
+               if (scm_is_pair (sval)) {
+                       for (SCM v = scm_reverse (scm_cdr (sval));
+                            scm_is_pair (v);
+                            v = scm_cdr (v))
+                       {
+                               int token;
+                               switch (extra_token) {
+                               case '$':
+                                       token = scan_scm_id (scm_car (v));
+                                       if (!scm_is_eq (yylval.scm, SCM_UNSPECIFIED))
+                                               push_extra_token (token, yylval.scm);
+                                       break;
+                               case '#':
+                                       push_extra_token (SCM_IDENTIFIER, scm_car (v));
+                                       break;
+                               }
+                       }
+                       sval = scm_car (sval);
+               } else
+                       sval = SCM_UNSPECIFIED;
+       }
+
        return sval;
 }