Replace WORD token in parser and lexer with SYMBOL

[lilypond.git] / lily / lexer.ll

diff --git a/lily/lexer.ll b/lily/lexer.ll
index df05b45c38a7db4a72dcbe1e69d87313f2316574..93184ebe085dc2d432274a06d4eeb887e565fe01 100644 (file)
--- a/lily/lexer.ll
+++ b/lily/lexer.ll
@@ -2,7 +2,7 @@
 /*
   This file is part of LilyPond, the GNU music typesetter.
 
 /*
   This file is part of LilyPond, the GNU music typesetter.
 

-  Copyright (C) 1996--2012 Han-Wen Nienhuys <hanwen@xs4all.nl>
+  Copyright (C) 1996--2015 Han-Wen Nienhuys <hanwen@xs4all.nl>

                  Jan Nieuwenhuizen <janneke@gnu.org>
 
   LilyPond is free software: you can redistribute it and/or modify
                  Jan Nieuwenhuizen <janneke@gnu.org>
 
   LilyPond is free software: you can redistribute it and/or modify


@@ -30,9 +30,9 @@
 /*
   backup rules
 
 /*
   backup rules
 

-  after making a change to the lexer rules, run 
+  after making a change to the lexer rules, run

       flex -b <this lexer file>
       flex -b <this lexer file>

-  and make sure that 
+  and make sure that

       lex.backup
   contains no backup states, but only the reminder
       Compressed tables always back up.
       lex.backup
   contains no backup states, but only the reminder
       Compressed tables always back up.


@@ -69,9 +69,9 @@ using namespace std;
 #include "pitch.hh"
 #include "source-file.hh"
 #include "std-string.hh"
 #include "pitch.hh"
 #include "source-file.hh"
 #include "std-string.hh"

-#include "string-convert.hh"

 #include "version.hh"
 #include "warn.hh"
 #include "version.hh"
 #include "warn.hh"

+#include "lily-imports.hh"

 
 /*
 RH 7 fix (?)
 
 /*
 RH 7 fix (?)


@@ -91,6 +91,15 @@ bool is_valid_version (string s);
                 yylval = SCM_EOL;               \
         } while (0)
 
                 yylval = SCM_EOL;               \
         } while (0)
 

+/*
+  The inside of \"violin1" is marked by commandquote mode
+*/
+
+#define start_command_quote() do {             \
+                yy_push_state (commandquote);  \
+                yylval = SCM_EOL;               \
+        } while (0)
+

 #define yylval (*lexval_)
 
 #define yylloc (*lexloc_)
 #define yylval (*lexval_)
 
 #define yylloc (*lexloc_)


@@ -111,10 +120,9 @@ SCM (* scm_parse_error_handler) (void *);
 %option debug
 %option yyclass="Lily_lexer"
 %option stack
 %option debug
 %option yyclass="Lily_lexer"
 %option stack

-%option never-interactive 
+%option never-interactive

 %option warn
 
 %option warn
 

-%x extratoken

 %x chords
 %x figures
 %x incl
 %x chords
 %x figures
 %x incl


@@ -124,6 +132,7 @@ SCM (* scm_parse_error_handler) (void *);
 %x markup
 %x notes
 %x quote
 %x markup
 %x notes
 %x quote

+%x commandquote

 %x sourcefileline
 %x sourcefilename
 %x version
 %x sourcefileline
 %x sourcefilename
 %x version


@@ -148,14 +157,18 @@ A         [a-zA-Z\200-\377]
 AA             {A}|_
 N              [0-9]
 ANY_CHAR       (.|\n)
 AA             {A}|_
 N              [0-9]
 ANY_CHAR       (.|\n)

-WORD           {A}([-_]{A}|{A})*
-COMMAND                \\{WORD}
-
+SYMBOL         {A}([-_]{A}|{A})*
+COMMAND                \\{SYMBOL}
+/* SPECIAL category is for every letter that needs to get passed to
+ * the parser rather than being redefinable by the user */
+SPECIAL                [-+*/=<>{}!?_^'',.:]
+SHORTHAND      (.|\\.)

 UNSIGNED       {N}+
 E_UNSIGNED     \\{N}+
 FRACTION       {N}+\/{N}+
 INT            -?{UNSIGNED}
 REAL           ({INT}\.{N}*)|(-?\.{N}+)
 UNSIGNED       {N}+
 E_UNSIGNED     \\{N}+
 FRACTION       {N}+\/{N}+
 INT            -?{UNSIGNED}
 REAL           ({INT}\.{N}*)|(-?\.{N}+)

+STRICTREAL      {UNSIGNED}\.{UNSIGNED}

 WHITE          [ \n\t\f\r]
 HORIZONTALWHITE                [ \t]
 BLACK          [^ \n\t\f\r]
 WHITE          [ \n\t\f\r]
 HORIZONTALWHITE                [ \t]
 BLACK          [^ \n\t\f\r]


@@ -172,40 +185,10 @@ BOM_UTF8  \357\273\277
        // swallow and ignore carriage returns
 }
 
        // swallow and ignore carriage returns
 }
 

-<extratoken>{ANY_CHAR} {
-  /* Generate a token without swallowing anything */
-
-  /* First unswallow the eaten character */
-  add_lexed_char (-YYLeng ());
-  yyless (0);
-
-  /* produce requested token */
-  int type = scm_to_int (scm_caar (extra_tokens_));
-  yylval = scm_cdar (extra_tokens_);
-  extra_tokens_ = scm_cdr (extra_tokens_);
-  if (scm_is_null (extra_tokens_))
-    yy_pop_state ();
-
-  return type;
-}
-
-<extratoken><<EOF>>    {
-  /* Generate a token without swallowing anything */
-
-  /* produce requested token */
-  int type = scm_to_int (scm_caar (extra_tokens_));
-  yylval = scm_cdar (extra_tokens_);
-  extra_tokens_ = scm_cdr (extra_tokens_);
-  if (scm_is_null (extra_tokens_))
-    yy_pop_state ();
-
-  return type;
-}
-

    /* Use the trailing context feature. Otherwise, the BOM will not be
       found if the file starts with an identifier definition. */
 <INITIAL,chords,lyrics,figures,notes>{BOM_UTF8}/.* {
    /* Use the trailing context feature. Otherwise, the BOM will not be
       found if the file starts with an identifier definition. */
 <INITIAL,chords,lyrics,figures,notes>{BOM_UTF8}/.* {

-  if (this->lexloc_->line_number () != 1 || this->lexloc_->column_number () != 0)
+  if (lexloc_->line_number () != 1 || lexloc_->column_number () != 0)

     {
       LexerWarning (_ ("stray UTF-8 BOM encountered").c_str ());
       // exit (1);
     {
       LexerWarning (_ ("stray UTF-8 BOM encountered").c_str ());
       // exit (1);


@@ -217,16 +200,10 @@ BOM_UTF8  \357\273\277
   "%{" {
        yy_push_state (longcomment);
   }
   "%{" {
        yy_push_state (longcomment);
   }

-  %[^{\n\r][^\n\r]*[\n\r]      {
+  %[^{\n\r][^\n\r]*[\n\r]?     {

          (void) YYText_utf8 ();
   }
          (void) YYText_utf8 ();
   }

-  %[^{\n\r]    { // backup rule
-         (void) YYText_utf8 ();
-  }
-  %[\n\r]      {
-  }
-  %[^{\n\r][^\n\r]*    {
-         (void) YYText_utf8 ();
+  %[\n\r]?     {

   }
   {WHITE}+     {
 
   }
   {WHITE}+     {
 


@@ -267,7 +244,7 @@ BOM_UTF8    \357\273\277
        s = s.substr (0, s.rfind ('\"'));
 
        yy_pop_state ();
        s = s.substr (0, s.rfind ('\"'));
 
        yy_pop_state ();

-       this->here_input().get_source_file ()->name_ = s;
+       here_input().get_source_file ()->name_ = s;

        message (_f ("Renaming input to: `%s'", s.c_str ()));
        progress_indication ("\n");
        scm_module_define (scm_car (scopes_),
        message (_f ("Renaming input to: `%s'", s.c_str ()));
        progress_indication ("\n");
        scm_module_define (scm_car (scopes_),


@@ -281,7 +258,7 @@ BOM_UTF8    \357\273\277
        sscanf (YYText (), "%d", &i);
 
        yy_pop_state ();
        sscanf (YYText (), "%d", &i);
 
        yy_pop_state ();

-       this->here_input ().get_source_file ()->set_line (here_input ().start (), i);
+       here_input ().get_source_file ()->set_line (here_input ().start (), i);

 }
 
 <version>{ANY_CHAR}    {
 }
 
 <version>{ANY_CHAR}    {


@@ -320,7 +297,7 @@ BOM_UTF8    \357\273\277
                yy_push_state (state);
        }
        else
                yy_push_state (state);
        }
        else

-               error (_ ("\\maininput not allowed outside init files"));
+               LexerError (_ ("\\maininput not allowed outside init files").c_str ());

 }
 
 <INITIAL,chords,lyrics,figures,notes>\\include           {
 }
 
 <INITIAL,chords,lyrics,figures,notes>\\include           {


@@ -353,12 +330,11 @@ BOM_UTF8  \357\273\277
          }
 }
 <incl>(\$|#) { // scm for the filename
          }
 }
 <incl>(\$|#) { // scm for the filename

-       int n = 0;

        Input hi = here_input();
        hi.step_forward ();
        Input hi = here_input();
        hi.step_forward ();

-       SCM sval = ly_parse_scm (hi.start (), &n, hi,
-               be_safe_global && is_main_input_, parser_);
-       sval = eval_scm (sval);
+       SCM sval = ly_parse_scm (hi, be_safe_global && is_main_input_, parser_);
+       sval = eval_scm (sval, hi);
+       int n = hi.end () - hi.start ();

 
        for (int i = 0; i < n; i++)
        {
 
        for (int i = 0; i < n; i++)
        {


@@ -371,7 +347,7 @@ BOM_UTF8    \357\273\277
                yy_pop_state ();
        } else {
                LexerError (_ ("string expected after \\include").c_str ());
                yy_pop_state ();
        } else {
                LexerError (_ ("string expected after \\include").c_str ());

-               if (sval != SCM_UNDEFINED) {
+               if (!SCM_UNBNDP (sval)) {

                        SCM err = scm_current_error_port ();
                        scm_puts ("This value was found instead: ", err);
                        scm_display (sval, err);
                        SCM err = scm_current_error_port ();
                        scm_puts ("This value was found instead: ", err);
                        scm_display (sval, err);


@@ -380,20 +356,20 @@ BOM_UTF8  \357\273\277
 }
 
 <incl,version,sourcefilename>\"[^""]*   { // backup rule
 }
 
 <incl,version,sourcefilename>\"[^""]*   { // backup rule

-       error (_ ("end quote missing"));
-       exit (1);
+       LexerError (_ ("end quote missing").c_str ());
+       yy_pop_state ();

 }
 
     /* Flex picks the longest matching pattern including trailing
      * contexts.  Without the backup pattern, r-. does not trigger the
 }
 
     /* Flex picks the longest matching pattern including trailing
      * contexts.  Without the backup pattern, r-. does not trigger the

-     * {RESTNAME} rule but rather the {WORD}/[-_] rule coming later,
+     * {RESTNAME} rule but rather the {SYMBOL}/[-_] rule coming later,

      * needed for avoiding backup states.
      */
 
 <chords,notes,figures>{RESTNAME}/[-_]  |  // pseudo backup rule
 <chords,notes,figures>{RESTNAME}       {
        char const *s = YYText ();
      * needed for avoiding backup states.
      */
 
 <chords,notes,figures>{RESTNAME}/[-_]  |  // pseudo backup rule
 <chords,notes,figures>{RESTNAME}       {
        char const *s = YYText ();

-       yylval = scm_from_locale_string (s);
+       yylval = scm_from_ascii_string (s);

        return RESTNAME;
 }
 <chords,notes,figures>q/[-_]   | // pseudo backup rule
        return RESTNAME;
 }
 <chords,notes,figures>q/[-_]   | // pseudo backup rule


@@ -408,15 +384,14 @@ BOM_UTF8  \357\273\277
        return MULTI_MEASURE_REST;
 }
 <INITIAL,chords,figures,lyrics,markup,notes>#  { //embedded scm
        return MULTI_MEASURE_REST;
 }
 <INITIAL,chords,figures,lyrics,markup,notes>#  { //embedded scm

-       int n = 0;

        Input hi = here_input();
        hi.step_forward ();
        Input hi = here_input();
        hi.step_forward ();

-       SCM sval = ly_parse_scm (hi.start (), &n, hi,
-               be_safe_global && is_main_input_, parser_);
+       SCM sval = ly_parse_scm (hi, be_safe_global && is_main_input_, parser_);

 
 

-       if (sval == SCM_UNDEFINED)
+       if (SCM_UNBNDP (sval))

                error_level_ = 1;
 
                error_level_ = 1;
 

+       int n = hi.end () - hi.start ();

        for (int i = 0; i < n; i++)
        {
                yyinput ();
        for (int i = 0; i < n; i++)
        {
                yyinput ();


@@ -428,11 +403,11 @@ BOM_UTF8  \357\273\277
 }
 
 <INITIAL,chords,figures,lyrics,markup,notes>\$ { //immediate scm
 }
 
 <INITIAL,chords,figures,lyrics,markup,notes>\$ { //immediate scm

-       int n = 0;

        Input hi = here_input();
        hi.step_forward ();
        Input hi = here_input();
        hi.step_forward ();

-       SCM sval = ly_parse_scm (hi.start (), &n, hi,
-               be_safe_global && is_main_input_, parser_);
+       SCM sval = ly_parse_scm (hi, be_safe_global && is_main_input_, parser_);
+
+       int n = hi.end () - hi.start ();

 
        for (int i = 0; i < n; i++)
        {
 
        for (int i = 0; i < n; i++)
        {


@@ -440,14 +415,14 @@ BOM_UTF8  \357\273\277
        }
        char_count_stack_.back () += n;
 
        }
        char_count_stack_.back () += n;
 

-       sval = eval_scm (sval, '$');
+       sval = eval_scm (sval, hi, '$');

 
        int token = scan_scm_id (sval);
        if (!scm_is_eq (yylval, SCM_UNSPECIFIED))
                return token;
 }
 
 
        int token = scan_scm_id (sval);
        if (!scm_is_eq (yylval, SCM_UNSPECIFIED))
                return token;
 }
 

-<INITIAL,notes,lyrics>{ 
+<INITIAL,notes,lyrics,chords>{

        \<\<    {
                 yylval = SCM_UNSPECIFIED;
                return DOUBLE_ANGLE_OPEN;
        \<\<    {
                 yylval = SCM_UNSPECIFIED;
                return DOUBLE_ANGLE_OPEN;


@@ -458,7 +433,7 @@ BOM_UTF8    \357\273\277
        }
 }
 
        }
 }
 

-<INITIAL,notes>{
+<INITIAL,notes,chords>{

        \<      {
                 yylval = SCM_UNSPECIFIED;
                return ANGLE_OPEN;
        \<      {
                 yylval = SCM_UNSPECIFIED;
                return ANGLE_OPEN;


@@ -482,14 +457,32 @@ BOM_UTF8  \357\273\277
                 yylval = SCM_UNSPECIFIED;
                return FIGURE_OPEN;
        }
                 yylval = SCM_UNSPECIFIED;
                return FIGURE_OPEN;
        }

+       \\\+    {
+               yylval = SCM_UNSPECIFIED;
+               return E_PLUS;
+       }
+       \\!     {
+               yylval = SCM_UNSPECIFIED;
+               return E_EXCLAMATION;
+       }
+       \\\\    {
+               yylval = SCM_UNSPECIFIED;
+               return E_BACKSLASH;
+       }
+       [][]    {
+               yylval = SCM_UNSPECIFIED;
+               return  YYText ()[0];
+       }

 }
 
 <notes,figures>{
 }
 
 <notes,figures>{

-       {WORD}/[-_]     | // backup rule
-       {WORD}  {
+       {SYMBOL}/[-_]   | // backup rule
+       {SYMBOL}        {

                return scan_bare_word (YYText_utf8 ());
        }
                return scan_bare_word (YYText_utf8 ());
        }

-
+       \\\"    {
+               start_command_quote ();
+       }

        {COMMAND}/[-_]  | // backup rule
        {COMMAND}       {
                return scan_escaped_word (YYText_utf8 () + 1); 
        {COMMAND}/[-_]  | // backup rule
        {COMMAND}       {
                return scan_escaped_word (YYText_utf8 () + 1); 


@@ -498,8 +491,12 @@ BOM_UTF8   \357\273\277
                yylval =  scan_fraction (YYText ());
                return FRACTION;
        }
                yylval =  scan_fraction (YYText ());
                return FRACTION;
        }

-       {UNSIGNED}/\/   | // backup rule
-       {UNSIGNED}              {
+       {STRICTREAL}    {
+               yylval = scm_c_read_string (YYText ());
+               return REAL;
+       }
+       {UNSIGNED}/[/.] | // backup rule
+       {UNSIGNED}      {

                yylval = scm_c_read_string (YYText ());
                return UNSIGNED;
        }
                yylval = scm_c_read_string (YYText ());
                return UNSIGNED;
        }


@@ -509,30 +506,35 @@ BOM_UTF8  \357\273\277
        }
 }
 
        }
 }
 

-<quote>{
+<quote,commandquote>{

        \\{ESCAPED}     {
                 char c = escaped_char (YYText ()[1]);
        \\{ESCAPED}     {
                 char c = escaped_char (YYText ()[1]);

-               yylval = scm_cons (scm_from_locale_stringn (&c, 1),
+               yylval = scm_cons (scm_from_ascii_stringn (&c, 1),

                                    yylval);
        }
        [^\\""]+        {
                                    yylval);
        }
        [^\\""]+        {

-                yylval = scm_cons (scm_from_locale_string (YYText_utf8 ()),
+                yylval = scm_cons (scm_from_utf8_string (YYText_utf8 ()),

                                    yylval);
        }
        \"      {
 
                                    yylval);
        }
        \"      {
 

-               yy_pop_state ();
-

                /* yylval is union. Must remember STRING before setting SCM*/
 
                 yylval = scm_string_concatenate_reverse (yylval,
                                                          SCM_UNDEFINED,
                                                          SCM_UNDEFINED);
 
                /* yylval is union. Must remember STRING before setting SCM*/
 
                 yylval = scm_string_concatenate_reverse (yylval,
                                                          SCM_UNDEFINED,
                                                          SCM_UNDEFINED);
 

+               if (get_state () == commandquote) {
+                       yy_pop_state ();
+                       return scan_escaped_word (ly_scm2string (yylval));
+               }
+
+               yy_pop_state ();
+

                return STRING;
        }
        \\      {
                return STRING;
        }
        \\      {

-                yylval = scm_cons (scm_from_locale_string (YYText ()),
+                yylval = scm_cons (scm_from_ascii_string (YYText ()),

                                    yylval);
        }
 }
                                    yylval);
        }
 }


@@ -545,21 +547,32 @@ BOM_UTF8  \357\273\277
                yylval =  scan_fraction (YYText ());
                return FRACTION;
        }
                yylval =  scan_fraction (YYText ());
                return FRACTION;
        }

-       {UNSIGNED}/\/   | // backup rule
+       {STRICTREAL}    {
+               yylval = scm_c_read_string (YYText ());
+               return REAL;
+       }
+       {UNSIGNED}/[/.] | // backup rule

        {UNSIGNED}              {
                yylval = scm_c_read_string (YYText ());
                return UNSIGNED;
        }
        {UNSIGNED}              {
                yylval = scm_c_read_string (YYText ());
                return UNSIGNED;
        }

+       \\\"    {
+               start_command_quote ();
+       }

        {COMMAND}/[-_]  | // backup rule
        {COMMAND}       {
                return scan_escaped_word (YYText_utf8 () + 1);
        }
        {COMMAND}/[-_]  | // backup rule
        {COMMAND}       {
                return scan_escaped_word (YYText_utf8 () + 1);
        }

-       /* Characters needed to express durations, assignments, barchecks */
-       [*.=|]  {
+       \\.|\|  {
+               // UTF-8 already covered by COMMAND
+               return scan_shorthand (YYText ());
+       }
+       /* Characters needed to express durations, assignments */
+       [*.=]   {

                 yylval = SCM_UNSPECIFIED;
                return YYText ()[0];
        }
                 yylval = SCM_UNSPECIFIED;
                return YYText ()[0];
        }

-       [^$#{}\"\\ \t\n\r\f0-9]+ {
+       [^|*.=$#{}\"\\ \t\n\r\f0-9][^$#{}\"\\ \t\n\r\f0-9]* {

                /* ugr. This sux. */
                string s (YYText_utf8 ());
                 yylval = SCM_UNSPECIFIED;
                /* ugr. This sux. */
                string s (YYText_utf8 ());
                 yylval = SCM_UNSPECIFIED;


@@ -570,19 +583,22 @@ BOM_UTF8  \357\273\277
                s = lyric_fudge (s);
                yylval = ly_string2scm (s);
 
                s = lyric_fudge (s);
                yylval = ly_string2scm (s);
 

-               return STRING;
+               return SYMBOL;

        }
        /* This should really just cover {} */
        }
        /* This should really just cover {} */

-       . {
+       [{}] {

                 yylval = SCM_UNSPECIFIED;
                 yylval = SCM_UNSPECIFIED;

-               return YYText ()[0]; // above catches all multibytes.
+               return YYText ()[0];

        }
 }
 <chords>{
        }
 }
 <chords>{

-       {WORD}/[-_]     | // backup rule
-       {WORD}  {
+       {SYMBOL}/[-_]   | // backup rule
+       {SYMBOL}        {

                return scan_bare_word (YYText_utf8 ());
        }
                return scan_bare_word (YYText_utf8 ());
        }

+       \\\"    {
+               start_command_quote ();
+       }

        {COMMAND}/[-_]  | // backup rule
        {COMMAND}       {
                return scan_escaped_word (YYText_utf8 () + 1);
        {COMMAND}/[-_]  | // backup rule
        {COMMAND}       {
                return scan_escaped_word (YYText_utf8 () + 1);


@@ -616,10 +632,6 @@ BOM_UTF8   \357\273\277
                 yylval = SCM_UNSPECIFIED;
                return CHORD_CARET;
        }
                 yylval = SCM_UNSPECIFIED;
                return CHORD_CARET;
        }

-       . {
-                yylval = SCM_UNSPECIFIED;
-               return YYText ()[0]; // WORD catches all multibyte.
-       }

 }
 
 
 }
 
 


@@ -628,6 +640,13 @@ BOM_UTF8   \357\273\277
                 yylval = SCM_UNSPECIFIED;
                return SCORE;
        }
                 yylval = SCM_UNSPECIFIED;
                return SCORE;
        }

+       \\score-lines {
+               yylval = SCM_UNSPECIFIED;
+               return SCORELINES;
+       }
+       \\\"    {
+               start_command_quote ();
+       }

        {COMMAND}/[-_]  | // backup rule
        {COMMAND} {
                string str (YYText_utf8 () + 1);
        {COMMAND}/[-_]  | // backup rule
        {COMMAND} {
                string str (YYText_utf8 () + 1);


@@ -665,17 +684,17 @@ BOM_UTF8  \357\273\277
                // value (for token type MARKUP_FUNCTION or
                // MARKUP_LIST_FUNCTION).
 
                // value (for token type MARKUP_FUNCTION or
                // MARKUP_LIST_FUNCTION).
 

-               push_extra_token(EXPECT_NO_MORE_ARGS);
+               push_extra_token (here_input (), EXPECT_NO_MORE_ARGS);

                s = scm_cdr(s);
                for (; scm_is_pair(s); s = scm_cdr(s)) {
                  SCM predicate = scm_car(s);
 
                s = scm_cdr(s);
                for (; scm_is_pair(s); s = scm_cdr(s)) {
                  SCM predicate = scm_car(s);
 

-                 if (predicate == ly_lily_module_constant ("markup-list?"))
-                   push_extra_token(EXPECT_MARKUP_LIST);
-                 else if (predicate == ly_lily_module_constant ("markup?"))
-                   push_extra_token(EXPECT_MARKUP);
+                 if (scm_is_eq (predicate, SCM (Lily::markup_list_p)))
+                   push_extra_token (here_input (), EXPECT_MARKUP_LIST);
+                 else if (scm_is_eq (predicate, SCM (Lily::markup_p)))
+                   push_extra_token (here_input (), EXPECT_MARKUP);

                  else
                  else

-                   push_extra_token(EXPECT_SCM, predicate);
+                   push_extra_token (here_input (), EXPECT_SCM, predicate);

                }
                return token_type;
        }
                }
                return token_type;
        }


@@ -683,11 +702,11 @@ BOM_UTF8  \357\273\277
                string s (YYText_utf8 ()); 
 
                yylval = ly_string2scm (s);
                string s (YYText_utf8 ()); 
 
                yylval = ly_string2scm (s);

-               return STRING;
+               return SYMBOL;

        }
        }

-       .  {
+       [{}]  {

                 yylval = SCM_UNSPECIFIED;
                 yylval = SCM_UNSPECIFIED;

-               return YYText ()[0];  // Above is catchall for multibyte
+               return YYText ()[0];

        }
 }
 
        }
 }
 


@@ -696,7 +715,7 @@ BOM_UTF8    \357\273\277
                yy_pop_state ();
        }
 
                yy_pop_state ();
        }
 

-<quote><<EOF>> {
+<quote,commandquote><<EOF>> {

        LexerError (_ ("EOF found inside string").c_str ());
        yy_pop_state ();
 }
        LexerError (_ ("EOF found inside string").c_str ());
        yy_pop_state ();
 }


@@ -714,10 +733,11 @@ BOM_UTF8  \357\273\277
                        {
                                LexerError (_ ("Unfinished main input").c_str ());
                                do {
                        {
                                LexerError (_ ("Unfinished main input").c_str ());
                                do {

-                                       pop_state ();
+                                       yy_pop_state ();

                                } while (YYSTATE != maininput);
                        }
                                } while (YYSTATE != maininput);
                        }

-                       pop_state ();
+                       extra_tokens_ = SCM_EOL;
+                       yy_pop_state ();

                }
                if (!close_input () || !is_main_input_)
                /* Returns YY_NULL */
                }
                if (!close_input () || !is_main_input_)
                /* Returns YY_NULL */


@@ -728,7 +748,7 @@ BOM_UTF8    \357\273\277
                yyterminate ();
 }
 
                yyterminate ();
 }
 

-<maininput>. {
+<maininput>{ANY_CHAR} {

        while (include_stack_.size () > main_input_level_
               && close_input ())
                ;
        while (include_stack_.size () > main_input_level_
               && close_input ())
                ;


@@ -736,10 +756,13 @@ BOM_UTF8  \357\273\277
 }
 
 <INITIAL>{
 }
 
 <INITIAL>{

-       {WORD}/[-_]     | // backup rule
-       {WORD}  {
+       {SYMBOL}/[-_]   | // backup rule
+       {SYMBOL}        {

                return scan_bare_word (YYText_utf8 ());
        }
                return scan_bare_word (YYText_utf8 ());
        }

+       \\\"    {
+               start_command_quote ();
+       }

        {COMMAND}/[-_]  | // backup rule
        {COMMAND}       {
                return scan_escaped_word (YYText_utf8 () + 1);
        {COMMAND}/[-_]  | // backup rule
        {COMMAND}       {
                return scan_escaped_word (YYText_utf8 () + 1);


@@ -764,51 +787,18 @@ BOM_UTF8  \357\273\277
 }
 
 
 }
 
 

-[{}]   {
-        yylval = SCM_UNSPECIFIED;
-       return YYText ()[0];
-}
-
--/\.   | // backup rule
-[*:=]          {
+-/\.   { // backup rule

         yylval = SCM_UNSPECIFIED;
        return YYText ()[0];
 }
 
         yylval = SCM_UNSPECIFIED;
        return YYText ()[0];
 }
 

-<INITIAL,notes,figures>.       {
+<INITIAL,chords,lyrics,figures,notes>{SPECIAL} {

         yylval = SCM_UNSPECIFIED;
        return YYText ()[0];
 }
 
         yylval = SCM_UNSPECIFIED;
        return YYText ()[0];
 }
 

-<INITIAL,lyrics,notes,figures>\\. {
-    yylval = SCM_UNSPECIFIED;
-    char c = YYText ()[1];
-
-    switch (c) {
-    case '>':
-       return E_ANGLE_CLOSE;
-    case '<':
-       return E_ANGLE_OPEN;
-    case '!':
-       return E_EXCLAMATION;
-    case '(':
-       return E_OPEN;
-    case ')':
-       return E_CLOSE;
-    case '[':
-       return E_BRACKET_OPEN;
-    case '+':
-       return E_PLUS;
-    case ']':
-       return E_BRACKET_CLOSE;
-    case '~':
-       return E_TILDE;
-    case '\\':
-       return E_BACKSLASH;
-
-    default:
-       return E_CHAR;
-    }
+<INITIAL,chords,lyrics,figures,notes>{SHORTHAND}       {
+       return scan_shorthand (YYText_utf8 ()); // should not be utf-8

 }
 
 <*>.[\200-\277]*       {
 }
 
 <*>.[\200-\277]*       {


@@ -820,18 +810,28 @@ BOM_UTF8  \357\273\277
 
 %%
 
 
 %%
 

-/* Make the lexer generate a token of the given type as the next token. 
+/* Make the lexer generate a token of the given type as the next token.

  TODO: make it possible to define a value for the token as well */
 void
  TODO: make it possible to define a value for the token as well */
 void

-Lily_lexer::push_extra_token (int token_type, SCM scm)
+Lily_lexer::push_extra_token (Input const &where, int token_type, SCM scm)
+{
+       extra_tokens_ = scm_cons (scm_cons2 (where.smobbed_copy (),
+                                            scm_from_int (token_type),
+                                            scm), extra_tokens_);
+}
+
+int
+Lily_lexer::pop_extra_token ()

 {
        if (scm_is_null (extra_tokens_))
 {
        if (scm_is_null (extra_tokens_))

-       {
-               if (YY_START != extratoken)
-                       hidden_state_ = YY_START;
-               yy_push_state (extratoken);
-       }
-       extra_tokens_ = scm_acons (scm_from_int (token_type), scm, extra_tokens_);
+               return -1;
+
+  /* produce requested token */
+       yylloc = *unsmob<Input> (scm_caar (extra_tokens_));
+       int type = scm_to_int (scm_cadar (extra_tokens_));
+       yylval = scm_cddar (extra_tokens_);
+       extra_tokens_ = scm_cdr (extra_tokens_);
+       return type;

 }
 
 void
 }
 
 void


@@ -872,41 +872,24 @@ Lily_lexer::push_markup_state ()
 void
 Lily_lexer::push_note_state (SCM alist)
 {
 void
 Lily_lexer::push_note_state (SCM alist)
 {

-       bool extra = (YYSTATE == extratoken);
-

        SCM p = scm_assq (alist, pitchname_tab_stack_);
 
        SCM p = scm_assq (alist, pitchname_tab_stack_);
 

-       if (extra)
-               yy_pop_state ();
-

        if (scm_is_false (p))
                p = scm_cons (alist, alist_to_hashq (alist));
        pitchname_tab_stack_ = scm_cons (p, pitchname_tab_stack_);
        yy_push_state (notes);
        if (scm_is_false (p))
                p = scm_cons (alist, alist_to_hashq (alist));
        pitchname_tab_stack_ = scm_cons (p, pitchname_tab_stack_);
        yy_push_state (notes);

-
-       if (extra) {
-               hidden_state_ = YYSTATE;
-               yy_push_state (extratoken);
-       }

 }
 
 void
 Lily_lexer::pop_state ()
 {
 }
 
 void
 Lily_lexer::pop_state ()
 {

-       bool extra = (YYSTATE == extratoken);
-
-       if (extra)
-               yy_pop_state ();
-

        if (YYSTATE == notes || YYSTATE == chords)
                pitchname_tab_stack_ = scm_cdr (pitchname_tab_stack_);
 
        if (YYSTATE == notes || YYSTATE == chords)
                pitchname_tab_stack_ = scm_cdr (pitchname_tab_stack_);
 

-       yy_pop_state ();
+       // don't cross the maininput threshold
+       if (YYSTATE != maininput)
+               yy_pop_state ();

 
 

-       if (extra) {
-               hidden_state_ = YYSTATE;
-               yy_push_state (extratoken);
-       }

 }
 
 int
 }
 
 int


@@ -918,7 +901,7 @@ Lily_lexer::identifier_type (SCM sid)
 
 
 int
 
 
 int

-Lily_lexer::scan_escaped_word (string str)
+Lily_lexer::scan_escaped_word (const string &str)

 {
        // use more SCM for this.
 
 {
        // use more SCM for this.
 


@@ -931,10 +914,35 @@ Lily_lexer::scan_escaped_word (string str)
                return i;
 
        SCM sid = lookup_identifier (str);
                return i;
 
        SCM sid = lookup_identifier (str);

-       if (sid != SCM_UNDEFINED)
+       if (Music *m = unsmob<Music> (sid))
+       {
+               m->set_spot (override_input (here_input ()));
+       }
+
+       if (!SCM_UNBNDP (sid))
+               return scan_scm_id (sid);
+
+       string msg (_f ("unknown escaped string: `\\%s'", str));
+       LexerError (msg.c_str ());
+
+       yylval = ly_string2scm (str);
+
+       return STRING; // SYMBOL would cause additional processing
+}
+
+int
+Lily_lexer::scan_shorthand (const string &str)
+{
+       SCM sid = lookup_identifier (str);
+       if (Music *m = unsmob<Music> (sid))
+       {
+               m->set_spot (override_input (here_input ()));
+       }
+
+       if (!SCM_UNBNDP (sid))

                return scan_scm_id (sid);
 
                return scan_scm_id (sid);
 

-       string msg (_f ("unknown escaped string: `\\%s'", str));        
+       string msg (_f ("undefined character or shorthand: %s", str));

        LexerError (msg.c_str ());
 
        yylval = ly_string2scm (str);
        LexerError (msg.c_str ());
 
        yylval = ly_string2scm (str);


@@ -945,13 +953,13 @@ Lily_lexer::scan_escaped_word (string str)
 int
 Lily_lexer::scan_scm_id (SCM sid)
 {
 int
 Lily_lexer::scan_scm_id (SCM sid)
 {

-       if (is_music_function (sid))
+       if (Music_function *fun = unsmob<Music_function> (sid))

        {
                int funtype = SCM_FUNCTION;
 
                yylval = sid;
 
        {
                int funtype = SCM_FUNCTION;
 
                yylval = sid;
 

-               SCM s = get_music_function_signature (sid);
+               SCM s = fun->get_signature ();

                SCM cs = scm_car (s);
 
                if (scm_is_pair (cs))
                SCM cs = scm_car (s);
 
                if (scm_is_pair (cs))


@@ -959,15 +967,15 @@ Lily_lexer::scan_scm_id (SCM sid)
                        cs = SCM_CAR (cs);
                }
 
                        cs = SCM_CAR (cs);
                }
 

-               if (scm_is_eq (cs, ly_lily_module_constant ("ly:music?")))
+               if (scm_is_eq (cs, SCM (Lily::ly_music_p)))

                        funtype = MUSIC_FUNCTION;
                        funtype = MUSIC_FUNCTION;

-               else if (scm_is_eq (cs, ly_lily_module_constant ("ly:event?")))
+               else if (scm_is_eq (cs, SCM (Lily::ly_event_p)))

                        funtype = EVENT_FUNCTION;
                else if (ly_is_procedure (cs))
                        funtype = SCM_FUNCTION;
                else programming_error ("Bad syntax function predicate");
 
                        funtype = EVENT_FUNCTION;
                else if (ly_is_procedure (cs))
                        funtype = SCM_FUNCTION;
                else programming_error ("Bad syntax function predicate");
 

-               push_extra_token (EXPECT_NO_MORE_ARGS);
+               push_extra_token (here_input (), EXPECT_NO_MORE_ARGS);

                for (s = scm_cdr (s); scm_is_pair (s); s = scm_cdr (s))
                {
                        SCM optional = SCM_UNDEFINED;
                for (s = scm_cdr (s); scm_is_pair (s); s = scm_cdr (s))
                {
                        SCM optional = SCM_UNDEFINED;


@@ -978,20 +986,16 @@ Lily_lexer::scan_scm_id (SCM sid)
                                optional = SCM_CDR (cs);
                                cs = SCM_CAR (cs);
                        }
                                optional = SCM_CDR (cs);
                                cs = SCM_CAR (cs);
                        }

-                       
-                       if (cs == Pitch_type_p_proc)
-                               push_extra_token (EXPECT_PITCH);
-                       else if (cs == Duration_type_p_proc)
-                               push_extra_token (EXPECT_DURATION);
-                       else if (ly_is_procedure (cs))
-                               push_extra_token (EXPECT_SCM, cs);
+
+                       if (ly_is_procedure (cs))
+                               push_extra_token (here_input (), EXPECT_SCM, cs);

                        else
                        {
                                programming_error ("Function parameter without type-checking predicate");
                                continue;
                        }
                        if (!scm_is_eq (optional, SCM_UNDEFINED))
                        else
                        {
                                programming_error ("Function parameter without type-checking predicate");
                                continue;
                        }
                        if (!scm_is_eq (optional, SCM_UNDEFINED))

-                               push_extra_token (EXPECT_OPTIONAL, optional);
+                               push_extra_token (here_input (), EXPECT_OPTIONAL, optional);

                }
                return funtype;
        }
                }
                return funtype;
        }


@@ -1000,39 +1004,47 @@ Lily_lexer::scan_scm_id (SCM sid)
 }
 
 int
 }
 
 int

-Lily_lexer::scan_bare_word (string str)
+Lily_lexer::scan_word (SCM & output, SCM sym)

 {
 {

-       SCM sym = ly_symbol2scm (str.c_str ());

        if ((YYSTATE == notes) || (YYSTATE == chords)) {
                SCM handle = SCM_BOOL_F;
                if (scm_is_pair (pitchname_tab_stack_))
                        handle = scm_hashq_get_handle (scm_cdar (pitchname_tab_stack_), sym);
        if ((YYSTATE == notes) || (YYSTATE == chords)) {
                SCM handle = SCM_BOOL_F;
                if (scm_is_pair (pitchname_tab_stack_))
                        handle = scm_hashq_get_handle (scm_cdar (pitchname_tab_stack_), sym);

-               
+

                if (scm_is_pair (handle)) {
                if (scm_is_pair (handle)) {

-                       yylval = scm_cdr (handle);
-                       if (unsmob_pitch (yylval))
+                       output = scm_cdr (handle);
+                       if (unsmob<Pitch> (yylval))

                            return (YYSTATE == notes) ? NOTENAME_PITCH : TONICNAME_PITCH;
                        else if (scm_is_symbol (yylval))
                            return DRUM_PITCH;
                }
                else if ((YYSTATE == chords)
                            return (YYSTATE == notes) ? NOTENAME_PITCH : TONICNAME_PITCH;
                        else if (scm_is_symbol (yylval))
                            return DRUM_PITCH;
                }
                else if ((YYSTATE == chords)

-                       && (handle = scm_hashq_get_handle (chordmodifier_tab_, sym))!= SCM_BOOL_F)
+                       && scm_is_true (handle = scm_hashq_get_handle (chordmodifier_tab_, sym)))

                {
                {

-                   yylval = scm_cdr (handle);
+                   output = scm_cdr (handle);

                    return CHORD_MODIFIER;
                }
        }
                    return CHORD_MODIFIER;
                }
        }

+       output = SCM_UNDEFINED;
+       return -1;
+}
+
+int
+Lily_lexer::scan_bare_word (const string &str)
+{
+       int state = scan_word (yylval, ly_symbol2scm (str.c_str ()));
+       if (state >= 0)
+       {
+               return state;
+       }

        yylval = ly_string2scm (str);
        yylval = ly_string2scm (str);

-       return STRING;
+       return SYMBOL;

 }
 
 int
 Lily_lexer::get_state () const
 {
 }
 
 int
 Lily_lexer::get_state () const
 {

-       if (YY_START == extratoken)
-               return hidden_state_;
-       else
-               return YY_START;
+       return YY_START;

 }
 
 bool
 }
 
 bool


@@ -1067,14 +1079,14 @@ Lily_lexer::is_figure_state () const
 // this function is private.
 
 SCM
 // this function is private.
 
 SCM

-Lily_lexer::eval_scm (SCM readerdata, char extra_token)
+Lily_lexer::eval_scm (SCM readerdata, Input hi, char extra_token)

 {
        SCM sval = SCM_UNDEFINED;
 
        if (!SCM_UNBNDP (readerdata))
        {
 {
        SCM sval = SCM_UNDEFINED;
 
        if (!SCM_UNBNDP (readerdata))
        {

-               sval = ly_eval_scm (scm_car (readerdata),
-                                   *unsmob_input (scm_cdr (readerdata)),
+               sval = ly_eval_scm (readerdata,
+                                   hi,

                                    be_safe_global && is_main_input_,
                                    parser_);
        }
                                    be_safe_global && is_main_input_,
                                    parser_);
        }


@@ -1090,19 +1102,28 @@ Lily_lexer::eval_scm (SCM readerdata, char extra_token)
                sval = scm_struct_ref (sval, SCM_INUM0);
 
                if (scm_is_pair (sval)) {
                sval = scm_struct_ref (sval, SCM_INUM0);
 
                if (scm_is_pair (sval)) {

-                       for (SCM v = scm_reverse (scm_cdr (sval));
-                            scm_is_pair (v);
-                            v = scm_cdr (v))
+                       for (SCM p = scm_reverse (scm_cdr (sval));
+                            scm_is_pair (p);
+                            p = scm_cdr (p))

                        {
                        {

+                               SCM v = scm_car (p);
+                               if (Music *m = unsmob<Music> (v))
+                               {
+                                       if (!unsmob<Input> (m->get_property ("origin")))
+                                               m->set_spot (override_input (here_input ()));
+                               }
+

                                int token;
                                switch (extra_token) {
                                case '$':
                                int token;
                                switch (extra_token) {
                                case '$':

-                                       token = scan_scm_id (scm_car (v));
+                                       token = scan_scm_id (v);

                                        if (!scm_is_eq (yylval, SCM_UNSPECIFIED))
                                        if (!scm_is_eq (yylval, SCM_UNSPECIFIED))

-                                               push_extra_token (token, yylval);
+                                               push_extra_token (here_input (),
+                                                                 token, yylval);

                                        break;
                                case '#':
                                        break;
                                case '#':

-                                       push_extra_token (SCM_IDENTIFIER, scm_car (v));
+                                       push_extra_token (here_input (),
+                                                         SCM_IDENTIFIER, v);

                                        break;
                                }
                        }
                                        break;
                                }
                        }


@@ -1111,6 +1132,12 @@ Lily_lexer::eval_scm (SCM readerdata, char extra_token)
                        sval = SCM_UNSPECIFIED;
        }
 
                        sval = SCM_UNSPECIFIED;
        }
 

+       if (Music *m = unsmob<Music> (sval))
+       {
+               if (!unsmob<Input> (m->get_property ("origin")))
+                       m->set_spot (override_input (here_input ()));
+       }
+

        return sval;
 }
 
        return sval;
 }
 


@@ -1225,7 +1252,7 @@ Lily_lexer::YYText_utf8 ()
 
 /*
  urg, belong to string (_convert)
 
 /*
  urg, belong to string (_convert)

- and should be generalised 
+ and should be generalised

  */
 void
 strip_leading_white (string&s)
  */
 void
 strip_leading_white (string&s)


@@ -1241,8 +1268,8 @@ strip_leading_white (string&s)
 void
 strip_trailing_white (string&s)
 {
 void
 strip_trailing_white (string&s)
 {

-       ssize i = s.length ();  
-       while (i--) 
+       ssize i = s.length ();
+       while (i--)

                if (!isspace (s[i]))
                        break;
 
                if (!isspace (s[i]))
                        break;
 


@@ -1259,8 +1286,13 @@ is_valid_version (string s)
 {
   Lilypond_version current ( MAJOR_VERSION "." MINOR_VERSION "." PATCH_LEVEL );
   Lilypond_version ver (s);
 {
   Lilypond_version current ( MAJOR_VERSION "." MINOR_VERSION "." PATCH_LEVEL );
   Lilypond_version ver (s);

-  if (int (ver) < oldest_version)
-       {       
+  if (!ver)
+  {
+         non_fatal_error (_f ("Invalid version string \"%s\"", s));
+         return false;
+  }
+  if (ver < oldest_version)
+       {

                non_fatal_error (_f ("file too old: %s (oldest supported: %s)", ver.to_string (), oldest_version.to_string ()));
                non_fatal_error (_ ("consider updating the input with the convert-ly script"));
                return false;
                non_fatal_error (_f ("file too old: %s (oldest supported: %s)", ver.to_string (), oldest_version.to_string ()));
                non_fatal_error (_ ("consider updating the input with the convert-ly script"));
                return false;


@@ -1273,7 +1305,7 @@ is_valid_version (string s)
        }
   return true;
 }
        }
   return true;
 }

-       
+

 
 /*
   substitute _
 
 /*
   substitute _


@@ -1300,23 +1332,20 @@ scan_fraction (string frac)
        string left = frac.substr (0, i);
        string right = frac.substr (i + 1, (frac.length () - i + 1));
 
        string left = frac.substr (0, i);
        string right = frac.substr (i + 1, (frac.length () - i + 1));
 

-       int n = String_convert::dec2int (left);
-       int d = String_convert::dec2int (right);
-       return scm_cons (scm_from_int (n), scm_from_int (d));
+       return scm_cons (scm_c_read_string (left.c_str ()),
+                        scm_c_read_string (right.c_str ()));

 }
 
 SCM
 lookup_markup_command (string s)
 {
 }
 
 SCM
 lookup_markup_command (string s)
 {

-       SCM proc = ly_lily_module_constant ("lookup-markup-command");
-       return scm_call_1 (proc, ly_string2scm (s));
+       return Lily::lookup_markup_command (ly_string2scm (s));

 }
 
 SCM
 lookup_markup_list_command (string s)
 {
 }
 
 SCM
 lookup_markup_list_command (string s)
 {

-       SCM proc = ly_lily_module_constant ("lookup-markup-list-command");
-       return scm_call_1 (proc, ly_string2scm (s));
+       return Lily::lookup_markup_list_command (ly_string2scm (s));

 }
 
 /* Shut up lexer warnings.  */
 }
 
 /* Shut up lexer warnings.  */