Issue 5145/1: Lexer: factor out push_markup_predicates

[lilypond.git] / lily / lexer.ll

diff --git a/lily/lexer.ll b/lily/lexer.ll
index 760ab9718dff0f7349caeb74cbc43d627b9b9703..19f84616f36ca3d38179f397daddfb1291a1d9ab 100644 (file)
--- a/lily/lexer.ll
+++ b/lily/lexer.ll
@@ -2,7 +2,7 @@
 /*
   This file is part of LilyPond, the GNU music typesetter.
 
 /*
   This file is part of LilyPond, the GNU music typesetter.
 

-  Copyright (C) 1996--2014 Han-Wen Nienhuys <hanwen@xs4all.nl>
+  Copyright (C) 1996--2015 Han-Wen Nienhuys <hanwen@xs4all.nl>

                  Jan Nieuwenhuizen <janneke@gnu.org>
 
   LilyPond is free software: you can redistribute it and/or modify
                  Jan Nieuwenhuizen <janneke@gnu.org>
 
   LilyPond is free software: you can redistribute it and/or modify


@@ -30,9 +30,9 @@
 /*
   backup rules
 
 /*
   backup rules
 

-  after making a change to the lexer rules, run 
+  after making a change to the lexer rules, run

       flex -b <this lexer file>
       flex -b <this lexer file>

-  and make sure that 
+  and make sure that

       lex.backup
   contains no backup states, but only the reminder
       Compressed tables always back up.
       lex.backup
   contains no backup states, but only the reminder
       Compressed tables always back up.


@@ -71,6 +71,7 @@ using namespace std;
 #include "std-string.hh"
 #include "version.hh"
 #include "warn.hh"
 #include "std-string.hh"
 #include "version.hh"
 #include "warn.hh"

+#include "lily-imports.hh"

 
 /*
 RH 7 fix (?)
 
 /*
 RH 7 fix (?)


@@ -119,7 +120,7 @@ SCM (* scm_parse_error_handler) (void *);
 %option debug
 %option yyclass="Lily_lexer"
 %option stack
 %option debug
 %option yyclass="Lily_lexer"
 %option stack

-%option never-interactive 
+%option never-interactive

 %option warn
 
 %x chords
 %option warn
 
 %x chords


@@ -156,8 +157,8 @@ A           [a-zA-Z\200-\377]
 AA             {A}|_
 N              [0-9]
 ANY_CHAR       (.|\n)
 AA             {A}|_
 N              [0-9]
 ANY_CHAR       (.|\n)

-WORD           {A}([-_]{A}|{A})*
-COMMAND                \\{WORD}
+SYMBOL         {A}([-_]{A}|{A})*
+COMMAND                \\{SYMBOL}

 /* SPECIAL category is for every letter that needs to get passed to
  * the parser rather than being redefinable by the user */
 SPECIAL                [-+*/=<>{}!?_^'',.:]
 /* SPECIAL category is for every letter that needs to get passed to
  * the parser rather than being redefinable by the user */
 SPECIAL                [-+*/=<>{}!?_^'',.:]


@@ -187,7 +188,7 @@ BOM_UTF8    \357\273\277
    /* Use the trailing context feature. Otherwise, the BOM will not be
       found if the file starts with an identifier definition. */
 <INITIAL,chords,lyrics,figures,notes>{BOM_UTF8}/.* {
    /* Use the trailing context feature. Otherwise, the BOM will not be
       found if the file starts with an identifier definition. */
 <INITIAL,chords,lyrics,figures,notes>{BOM_UTF8}/.* {

-  if (this->lexloc_->line_number () != 1 || this->lexloc_->column_number () != 0)
+  if (lexloc_->line_number () != 1 || lexloc_->column_number () != 0)

     {
       LexerWarning (_ ("stray UTF-8 BOM encountered").c_str ());
       // exit (1);
     {
       LexerWarning (_ ("stray UTF-8 BOM encountered").c_str ());
       // exit (1);


@@ -243,7 +244,7 @@ BOM_UTF8    \357\273\277
        s = s.substr (0, s.rfind ('\"'));
 
        yy_pop_state ();
        s = s.substr (0, s.rfind ('\"'));
 
        yy_pop_state ();

-       this->here_input().get_source_file ()->name_ = s;
+       here_input().get_source_file ()->name_ = s;

        message (_f ("Renaming input to: `%s'", s.c_str ()));
        progress_indication ("\n");
        scm_module_define (scm_car (scopes_),
        message (_f ("Renaming input to: `%s'", s.c_str ()));
        progress_indication ("\n");
        scm_module_define (scm_car (scopes_),


@@ -257,7 +258,7 @@ BOM_UTF8    \357\273\277
        sscanf (YYText (), "%d", &i);
 
        yy_pop_state ();
        sscanf (YYText (), "%d", &i);
 
        yy_pop_state ();

-       this->here_input ().get_source_file ()->set_line (here_input ().start (), i);
+       here_input ().get_source_file ()->set_line (here_input ().start (), i);

 }
 
 <version>{ANY_CHAR}    {
 }
 
 <version>{ANY_CHAR}    {


@@ -346,7 +347,7 @@ BOM_UTF8    \357\273\277
                yy_pop_state ();
        } else {
                LexerError (_ ("string expected after \\include").c_str ());
                yy_pop_state ();
        } else {
                LexerError (_ ("string expected after \\include").c_str ());

-               if (sval != SCM_UNDEFINED) {
+               if (!SCM_UNBNDP (sval)) {

                        SCM err = scm_current_error_port ();
                        scm_puts ("This value was found instead: ", err);
                        scm_display (sval, err);
                        SCM err = scm_current_error_port ();
                        scm_puts ("This value was found instead: ", err);
                        scm_display (sval, err);


@@ -361,14 +362,14 @@ BOM_UTF8  \357\273\277
 
     /* Flex picks the longest matching pattern including trailing
      * contexts.  Without the backup pattern, r-. does not trigger the
 
     /* Flex picks the longest matching pattern including trailing
      * contexts.  Without the backup pattern, r-. does not trigger the

-     * {RESTNAME} rule but rather the {WORD}/[-_] rule coming later,
+     * {RESTNAME} rule but rather the {SYMBOL}/[-_] rule coming later,

      * needed for avoiding backup states.
      */
 
 <chords,notes,figures>{RESTNAME}/[-_]  |  // pseudo backup rule
 <chords,notes,figures>{RESTNAME}       {
        char const *s = YYText ();
      * needed for avoiding backup states.
      */
 
 <chords,notes,figures>{RESTNAME}/[-_]  |  // pseudo backup rule
 <chords,notes,figures>{RESTNAME}       {
        char const *s = YYText ();

-       yylval = scm_from_locale_string (s);
+       yylval = scm_from_ascii_string (s);

        return RESTNAME;
 }
 <chords,notes,figures>q/[-_]   | // pseudo backup rule
        return RESTNAME;
 }
 <chords,notes,figures>q/[-_]   | // pseudo backup rule


@@ -387,7 +388,7 @@ BOM_UTF8    \357\273\277
        hi.step_forward ();
        SCM sval = ly_parse_scm (hi, be_safe_global && is_main_input_, parser_);
 
        hi.step_forward ();
        SCM sval = ly_parse_scm (hi, be_safe_global && is_main_input_, parser_);
 

-       if (sval == SCM_UNDEFINED)
+       if (SCM_UNBNDP (sval))

                error_level_ = 1;
 
        int n = hi.end () - hi.start ();
                error_level_ = 1;
 
        int n = hi.end () - hi.start ();


@@ -475,8 +476,8 @@ BOM_UTF8    \357\273\277
 }
 
 <notes,figures>{
 }
 
 <notes,figures>{

-       {WORD}/[-_]     | // backup rule
-       {WORD}  {
+       {SYMBOL}/[-_]   | // backup rule
+       {SYMBOL}        {

                return scan_bare_word (YYText_utf8 ());
        }
        \\\"    {
                return scan_bare_word (YYText_utf8 ());
        }
        \\\"    {


@@ -508,11 +509,11 @@ BOM_UTF8  \357\273\277
 <quote,commandquote>{
        \\{ESCAPED}     {
                 char c = escaped_char (YYText ()[1]);
 <quote,commandquote>{
        \\{ESCAPED}     {
                 char c = escaped_char (YYText ()[1]);

-               yylval = scm_cons (scm_from_locale_stringn (&c, 1),
+               yylval = scm_cons (scm_from_ascii_stringn (&c, 1),

                                    yylval);
        }
        [^\\""]+        {
                                    yylval);
        }
        [^\\""]+        {

-                yylval = scm_cons (scm_from_locale_string (YYText_utf8 ()),
+                yylval = scm_cons (scm_from_utf8_string (YYText_utf8 ()),

                                    yylval);
        }
        \"      {
                                    yylval);
        }
        \"      {


@@ -533,7 +534,7 @@ BOM_UTF8    \357\273\277
                return STRING;
        }
        \\      {
                return STRING;
        }
        \\      {

-                yylval = scm_cons (scm_from_locale_string (YYText ()),
+                yylval = scm_cons (scm_from_ascii_string (YYText ()),

                                    yylval);
        }
 }
                                    yylval);
        }
 }


@@ -582,7 +583,7 @@ BOM_UTF8    \357\273\277
                s = lyric_fudge (s);
                yylval = ly_string2scm (s);
 
                s = lyric_fudge (s);
                yylval = ly_string2scm (s);
 

-               return STRING;
+               return SYMBOL;

        }
        /* This should really just cover {} */
        [{}] {
        }
        /* This should really just cover {} */
        [{}] {


@@ -591,8 +592,8 @@ BOM_UTF8    \357\273\277
        }
 }
 <chords>{
        }
 }
 <chords>{

-       {WORD}/[-_]     | // backup rule
-       {WORD}  {
+       {SYMBOL}/[-_]   | // backup rule
+       {SYMBOL}        {

                return scan_bare_word (YYText_utf8 ());
        }
        \\\"    {
                return scan_bare_word (YYText_utf8 ());
        }
        \\\"    {


@@ -683,25 +684,15 @@ BOM_UTF8  \357\273\277
                // value (for token type MARKUP_FUNCTION or
                // MARKUP_LIST_FUNCTION).
 
                // value (for token type MARKUP_FUNCTION or
                // MARKUP_LIST_FUNCTION).
 

-               push_extra_token (here_input (), EXPECT_NO_MORE_ARGS);
-               s = scm_cdr(s);
-               for (; scm_is_pair(s); s = scm_cdr(s)) {
-                 SCM predicate = scm_car(s);
-
-                 if (predicate == ly_lily_module_constant ("markup-list?"))
-                   push_extra_token (here_input (), EXPECT_MARKUP_LIST);
-                 else if (predicate == ly_lily_module_constant ("markup?"))
-                   push_extra_token (here_input (), EXPECT_MARKUP);
-                 else
-                   push_extra_token (here_input (), EXPECT_SCM, predicate);
-               }
+               push_markup_predicates (scm_cdr (s));
+

                return token_type;
        }
        [^$#{}\"\\ \t\n\r\f]+ {
                string s (YYText_utf8 ()); 
 
                yylval = ly_string2scm (s);
                return token_type;
        }
        [^$#{}\"\\ \t\n\r\f]+ {
                string s (YYText_utf8 ()); 
 
                yylval = ly_string2scm (s);

-               return STRING;
+               return SYMBOL;

        }
        [{}]  {
                 yylval = SCM_UNSPECIFIED;
        }
        [{}]  {
                 yylval = SCM_UNSPECIFIED;


@@ -755,8 +746,8 @@ BOM_UTF8    \357\273\277
 }
 
 <INITIAL>{
 }
 
 <INITIAL>{

-       {WORD}/[-_]     | // backup rule
-       {WORD}  {
+       {SYMBOL}/[-_]   | // backup rule
+       {SYMBOL}        {

                return scan_bare_word (YYText_utf8 ());
        }
        \\\"    {
                return scan_bare_word (YYText_utf8 ());
        }
        \\\"    {


@@ -809,7 +800,7 @@ BOM_UTF8    \357\273\277
 
 %%
 
 
 %%
 

-/* Make the lexer generate a token of the given type as the next token. 
+/* Make the lexer generate a token of the given type as the next token.

  TODO: make it possible to define a value for the token as well */
 void
 Lily_lexer::push_extra_token (Input const &where, int token_type, SCM scm)
  TODO: make it possible to define a value for the token as well */
 void
 Lily_lexer::push_extra_token (Input const &where, int token_type, SCM scm)


@@ -826,7 +817,7 @@ Lily_lexer::pop_extra_token ()
                return -1;
 
   /* produce requested token */
                return -1;
 
   /* produce requested token */

-       yylloc = *Input::unsmob (scm_caar (extra_tokens_));
+       yylloc = *unsmob<Input> (scm_caar (extra_tokens_));

        int type = scm_to_int (scm_cadar (extra_tokens_));
        yylval = scm_cddar (extra_tokens_);
        extra_tokens_ = scm_cdr (extra_tokens_);
        int type = scm_to_int (scm_cadar (extra_tokens_));
        yylval = scm_cddar (extra_tokens_);
        extra_tokens_ = scm_cdr (extra_tokens_);


@@ -891,6 +882,23 @@ Lily_lexer::pop_state ()
 
 }
 
 
 }
 

+void
+Lily_lexer::push_markup_predicates (SCM sig)
+{
+       push_extra_token (here_input (), EXPECT_NO_MORE_ARGS);
+       for (SCM s = sig; scm_is_pair(s); s = scm_cdr(s)) {
+               SCM predicate = scm_car(s);
+
+               if (scm_is_eq (predicate, SCM (Lily::markup_list_p)))
+                       push_extra_token (here_input (), EXPECT_MARKUP_LIST);
+               else if (scm_is_eq (predicate, SCM (Lily::markup_p)))
+                       push_extra_token (here_input (), EXPECT_MARKUP);
+               else
+                       push_extra_token (here_input (), EXPECT_SCM, predicate);
+       }
+}
+
+

 int
 Lily_lexer::identifier_type (SCM sid)
 {
 int
 Lily_lexer::identifier_type (SCM sid)
 {


@@ -913,35 +921,35 @@ Lily_lexer::scan_escaped_word (const string &str)
                return i;
 
        SCM sid = lookup_identifier (str);
                return i;
 
        SCM sid = lookup_identifier (str);

-       if (Music *m = Music::unsmob (sid))
+       if (Music *m = unsmob<Music> (sid))

        {
                m->set_spot (override_input (here_input ()));
        }
 
        {
                m->set_spot (override_input (here_input ()));
        }
 

-       if (sid != SCM_UNDEFINED)
+       if (!SCM_UNBNDP (sid))

                return scan_scm_id (sid);
 
                return scan_scm_id (sid);
 

-       string msg (_f ("unknown escaped string: `\\%s'", str));        
+       string msg (_f ("unknown escaped string: `\\%s'", str));

        LexerError (msg.c_str ());
 
        yylval = ly_string2scm (str);
 
        LexerError (msg.c_str ());
 
        yylval = ly_string2scm (str);
 

-       return STRING;
+       return STRING; // SYMBOL would cause additional processing

 }
 
 int
 Lily_lexer::scan_shorthand (const string &str)
 {
        SCM sid = lookup_identifier (str);
 }
 
 int
 Lily_lexer::scan_shorthand (const string &str)
 {
        SCM sid = lookup_identifier (str);

-       if (Music *m = Music::unsmob (sid))
+       if (Music *m = unsmob<Music> (sid))

        {
                m->set_spot (override_input (here_input ()));
        }
 
        {
                m->set_spot (override_input (here_input ()));
        }
 

-       if (sid != SCM_UNDEFINED)
+       if (!SCM_UNBNDP (sid))

                return scan_scm_id (sid);
 
                return scan_scm_id (sid);
 

-       string msg (_f ("undefined character or shorthand: %s", str));  
+       string msg (_f ("undefined character or shorthand: %s", str));

        LexerError (msg.c_str ());
 
        yylval = ly_string2scm (str);
        LexerError (msg.c_str ());
 
        yylval = ly_string2scm (str);


@@ -952,13 +960,13 @@ Lily_lexer::scan_shorthand (const string &str)
 int
 Lily_lexer::scan_scm_id (SCM sid)
 {
 int
 Lily_lexer::scan_scm_id (SCM sid)
 {

-       if (is_music_function (sid))
+       if (Music_function *fun = unsmob<Music_function> (sid))

        {
                int funtype = SCM_FUNCTION;
 
                yylval = sid;
 
        {
                int funtype = SCM_FUNCTION;
 
                yylval = sid;
 

-               SCM s = get_music_function_signature (sid);
+               SCM s = fun->get_signature ();

                SCM cs = scm_car (s);
 
                if (scm_is_pair (cs))
                SCM cs = scm_car (s);
 
                if (scm_is_pair (cs))


@@ -966,9 +974,9 @@ Lily_lexer::scan_scm_id (SCM sid)
                        cs = SCM_CAR (cs);
                }
 
                        cs = SCM_CAR (cs);
                }
 

-               if (scm_is_eq (cs, ly_lily_module_constant ("ly:music?")))
+               if (scm_is_eq (cs, SCM (Lily::ly_music_p)))

                        funtype = MUSIC_FUNCTION;
                        funtype = MUSIC_FUNCTION;

-               else if (scm_is_eq (cs, ly_lily_module_constant ("ly:event?")))
+               else if (scm_is_eq (cs, SCM (Lily::ly_event_p)))

                        funtype = EVENT_FUNCTION;
                else if (ly_is_procedure (cs))
                        funtype = SCM_FUNCTION;
                        funtype = EVENT_FUNCTION;
                else if (ly_is_procedure (cs))
                        funtype = SCM_FUNCTION;


@@ -985,7 +993,7 @@ Lily_lexer::scan_scm_id (SCM sid)
                                optional = SCM_CDR (cs);
                                cs = SCM_CAR (cs);
                        }
                                optional = SCM_CDR (cs);
                                cs = SCM_CAR (cs);
                        }

-                       
+

                        if (ly_is_procedure (cs))
                                push_extra_token (here_input (), EXPECT_SCM, cs);
                        else
                        if (ly_is_procedure (cs))
                                push_extra_token (here_input (), EXPECT_SCM, cs);
                        else


@@ -1003,30 +1011,41 @@ Lily_lexer::scan_scm_id (SCM sid)
 }
 
 int
 }
 
 int

-Lily_lexer::scan_bare_word (const string &str)
+Lily_lexer::scan_word (SCM & output, SCM sym)

 {
 {

-       SCM sym = ly_symbol2scm (str.c_str ());

        if ((YYSTATE == notes) || (YYSTATE == chords)) {
                SCM handle = SCM_BOOL_F;
                if (scm_is_pair (pitchname_tab_stack_))
                        handle = scm_hashq_get_handle (scm_cdar (pitchname_tab_stack_), sym);
        if ((YYSTATE == notes) || (YYSTATE == chords)) {
                SCM handle = SCM_BOOL_F;
                if (scm_is_pair (pitchname_tab_stack_))
                        handle = scm_hashq_get_handle (scm_cdar (pitchname_tab_stack_), sym);

-               
+

                if (scm_is_pair (handle)) {
                if (scm_is_pair (handle)) {

-                       yylval = scm_cdr (handle);
-                       if (Pitch::unsmob (yylval))
+                       output = scm_cdr (handle);
+                       if (unsmob<Pitch> (yylval))

                            return (YYSTATE == notes) ? NOTENAME_PITCH : TONICNAME_PITCH;
                        else if (scm_is_symbol (yylval))
                            return DRUM_PITCH;
                }
                else if ((YYSTATE == chords)
                            return (YYSTATE == notes) ? NOTENAME_PITCH : TONICNAME_PITCH;
                        else if (scm_is_symbol (yylval))
                            return DRUM_PITCH;
                }
                else if ((YYSTATE == chords)

-                       && (handle = scm_hashq_get_handle (chordmodifier_tab_, sym))!= SCM_BOOL_F)
+                       && scm_is_true (handle = scm_hashq_get_handle (chordmodifier_tab_, sym)))

                {
                {

-                   yylval = scm_cdr (handle);
+                   output = scm_cdr (handle);

                    return CHORD_MODIFIER;
                }
        }
                    return CHORD_MODIFIER;
                }
        }

+       output = SCM_UNDEFINED;
+       return -1;
+}
+
+int
+Lily_lexer::scan_bare_word (const string &str)
+{
+       int state = scan_word (yylval, ly_symbol2scm (str.c_str ()));
+       if (state >= 0)
+       {
+               return state;
+       }

        yylval = ly_string2scm (str);
        yylval = ly_string2scm (str);

-       return STRING;
+       return SYMBOL;

 }
 
 int
 }
 
 int


@@ -1095,12 +1114,12 @@ Lily_lexer::eval_scm (SCM readerdata, Input hi, char extra_token)
                             p = scm_cdr (p))
                        {
                                SCM v = scm_car (p);
                             p = scm_cdr (p))
                        {
                                SCM v = scm_car (p);

-                               if (Music *m = Music::unsmob (v))
+                               if (Music *m = unsmob<Music> (v))

                                {
                                {

-                                       if (!Input::unsmob (m->get_property ("origin")))
+                                       if (!unsmob<Input> (m->get_property ("origin")))

                                                m->set_spot (override_input (here_input ()));
                                }
                                                m->set_spot (override_input (here_input ()));
                                }

-                                       
+

                                int token;
                                switch (extra_token) {
                                case '$':
                                int token;
                                switch (extra_token) {
                                case '$':


@@ -1120,9 +1139,9 @@ Lily_lexer::eval_scm (SCM readerdata, Input hi, char extra_token)
                        sval = SCM_UNSPECIFIED;
        }
 
                        sval = SCM_UNSPECIFIED;
        }
 

-       if (Music *m = Music::unsmob (sval))
+       if (Music *m = unsmob<Music> (sval))

        {
        {

-               if (!Input::unsmob (m->get_property ("origin")))
+               if (!unsmob<Input> (m->get_property ("origin")))

                        m->set_spot (override_input (here_input ()));
        }
 
                        m->set_spot (override_input (here_input ()));
        }
 


@@ -1240,7 +1259,7 @@ Lily_lexer::YYText_utf8 ()
 
 /*
  urg, belong to string (_convert)
 
 /*
  urg, belong to string (_convert)

- and should be generalised 
+ and should be generalised

  */
 void
 strip_leading_white (string&s)
  */
 void
 strip_leading_white (string&s)


@@ -1256,8 +1275,8 @@ strip_leading_white (string&s)
 void
 strip_trailing_white (string&s)
 {
 void
 strip_trailing_white (string&s)
 {

-       ssize i = s.length ();  
-       while (i--) 
+       ssize i = s.length ();
+       while (i--)

                if (!isspace (s[i]))
                        break;
 
                if (!isspace (s[i]))
                        break;
 


@@ -1274,8 +1293,13 @@ is_valid_version (string s)
 {
   Lilypond_version current ( MAJOR_VERSION "." MINOR_VERSION "." PATCH_LEVEL );
   Lilypond_version ver (s);
 {
   Lilypond_version current ( MAJOR_VERSION "." MINOR_VERSION "." PATCH_LEVEL );
   Lilypond_version ver (s);

-  if (int (ver) < oldest_version)
-       {       
+  if (!ver)
+  {
+         non_fatal_error (_f ("Invalid version string \"%s\"", s));
+         return false;
+  }
+  if (ver < oldest_version)
+       {

                non_fatal_error (_f ("file too old: %s (oldest supported: %s)", ver.to_string (), oldest_version.to_string ()));
                non_fatal_error (_ ("consider updating the input with the convert-ly script"));
                return false;
                non_fatal_error (_f ("file too old: %s (oldest supported: %s)", ver.to_string (), oldest_version.to_string ()));
                non_fatal_error (_ ("consider updating the input with the convert-ly script"));
                return false;


@@ -1288,7 +1312,7 @@ is_valid_version (string s)
        }
   return true;
 }
        }
   return true;
 }

-       
+

 
 /*
   substitute _
 
 /*
   substitute _


@@ -1322,15 +1346,13 @@ scan_fraction (string frac)
 SCM
 lookup_markup_command (string s)
 {
 SCM
 lookup_markup_command (string s)
 {

-       SCM proc = ly_lily_module_constant ("lookup-markup-command");
-       return scm_call_1 (proc, ly_string2scm (s));
+       return Lily::lookup_markup_command (ly_string2scm (s));

 }
 
 SCM
 lookup_markup_list_command (string s)
 {
 }
 
 SCM
 lookup_markup_list_command (string s)
 {

-       SCM proc = ly_lily_module_constant ("lookup-markup-list-command");
-       return scm_call_1 (proc, ly_string2scm (s));
+       return Lily::lookup_markup_list_command (ly_string2scm (s));

 }
 
 /* Shut up lexer warnings.  */
 }
 
 /* Shut up lexer warnings.  */