X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=lily%2Flexer.ll;h=f9601985f40b6bed668c7853e61bb359f2ab1f13;hb=e4597b9d661ccb3d1031b9e17ec0bcca2bfaa66d;hp=b4049350e15da715d5b6031ff4b4b258c78e33d8;hpb=df56feefa28914b45532b0a31b24c304c162b497;p=lilypond.git diff --git a/lily/lexer.ll b/lily/lexer.ll index b4049350e1..f9601985f4 100644 --- a/lily/lexer.ll +++ b/lily/lexer.ll @@ -4,7 +4,7 @@ source file of the LilyPond music typesetter - (c) 1996--2004 Han-Wen Nienhuys + (c) 1996--2006 Han-Wen Nienhuys Jan Nieuwenhuizen */ @@ -22,9 +22,10 @@ */ -#include -#include -#include + +#include +#include +#include /* Flex >= 2.5.29 fix; FlexLexer.h's multiple include bracing breaks when building the actual lexer. */ @@ -32,50 +33,52 @@ #define LEXER_CC #include - using namespace std; -#include "music-function.hh" -#include "source-file.hh" -#include "parse-scm.hh" -#include "lily-guile.hh" -#include "string.hh" -#include "string-convert.hh" -#include "lily-lexer.hh" +#include "context-def.hh" +#include "identifier-smob.hh" +#include "international.hh" #include "interval.hh" #include "lily-guile.hh" -#include "parser.hh" -#include "warn.hh" +#include "lily-lexer.hh" +#include "lilypond-input-version.hh" #include "main.hh" +#include "music-function.hh" +#include "parse-scm.hh" +#include "parser.hh" +#include "source-file.hh" +#include "std-string.hh" +#include "string-convert.hh" #include "version.hh" -#include "lilypond-input-version.hh" -#include "context-def.hh" -#include "identifier-smob.hh" +#include "warn.hh" /* RH 7 fix (?) */ #define isatty HORRIBLEKLUDGE -void strip_trailing_white (String&); -void strip_leading_white (String&); -String lyric_fudge (String s); +void strip_trailing_white (string&); +void strip_leading_white (string&); +string lyric_fudge (string s); int music_function_type (SCM); -SCM lookup_markup_command (String s); -bool is_valid_version (String s); +SCM lookup_markup_command (string s); +bool is_valid_version (string s); #define start_quote() \ yy_push_state (quote);\ - yylval.string = new String + yylval.string = new string #define start_lyric_quote() \ yy_push_state (lyric_quote);\ - yylval.string = new String + yylval.string = new string #define yylval \ (*(YYSTYPE*)lexval) +#define yylloc \ + (*(YYLTYPE*)lexloc) + #define YY_USER_ACTION add_lexed_char (YYLeng ()); /* @@ -84,7 +87,7 @@ LYRICS ({AA}|{TEX})[^0-9 \t\n\f]* */ -SCM scan_fraction (String); +SCM scan_fraction (string); SCM (* scm_parse_error_handler) (void *); @@ -106,10 +109,10 @@ SCM (* scm_parse_error_handler) (void *); %x lyrics %x lyric_quote %x longcomment -%x markup +%x markup %x notes %x quote -%x renameinput +%x sourcefilename %x version A [a-zA-Z] @@ -121,6 +124,8 @@ ACCENT \\[`'"^] NATIONAL [\001-\006\021-\027\031\036\200-\377] TEX {AA}|-|{PUNCT}|{ACCENT}|{NATIONAL} WORD {A}{AN}* +DASHED_WORD {A}({AN}|-)* +DASHED_KEY_WORD \\{DASHED_WORD} ALPHAWORD {A}+ DIGIT {N} UNSIGNED {N}+ @@ -139,6 +144,7 @@ LYRICS ({AA}|{TEX})[^0-9 \t\n\r\f]* ESCAPED [nt\\'"] EXTENDER __ HYPHEN -- +BOM_UTF8 \357\273\277 %% @@ -146,7 +152,17 @@ HYPHEN -- // windows-suck-suck-suck } -{ +{BOM_UTF8} { + if (this->lexloc->line_number () != 1 || this->lexloc->column_number () != 0) + { + LexerError (_ ("stray UTF-8 BOM encountered").c_str ()); + exit (1); + } + if (be_verbose_global) + message (_ ("Skipping UTF-8 BOM")); +} + +{ "%{" { yy_push_state (longcomment); } @@ -166,12 +182,12 @@ HYPHEN -- \\version{WHITE}* { yy_push_state (version); } -\\renameinput{WHITE}* { - yy_push_state (renameinput); +\\sourcefilename{WHITE}* { + yy_push_state (sourcefilename); } \"[^"]*\" { /* got the version number */ - String s (YYText () + 1); - s = s.left_string (s.index_last ('\"')); + string s (YYText () + 1); + s = s.substr (0, s.rfind ('\"')); yy_pop_state (); if (!is_valid_version (s)) @@ -181,25 +197,25 @@ HYPHEN -- scm_module_define (top_scope, ly_symbol2scm ("version-seen?"), SCM_BOOL_T); } -\"[^"]*\" { - String s (YYText () + 1); - s = s.left_string (s.index_last ('\"')); +\"[^"]*\" { + string s (YYText () + 1); + s = s.substr (0, s.rfind ('\"')); - yy_pop_state(); - this->here_input().source_file_->name_ = s; - progress_indication (_f ("Renaming input to: `%s'", s.to_str0 ())); + yy_pop_state (); + this->here_input().get_source_file ()->name_ = s; + message (_f ("Renaming input to: `%s'", s.c_str ())); progress_indication ("\n"); scm_module_define (scm_car (scopes_), ly_symbol2scm ("input-file-name"), - scm_makfrom0str (s.to_str0 ())); + scm_makfrom0str (s.c_str ())); } . { - LexerError (_ ("No quoted string found after \\version").to_str0 ()); + LexerError (_ ("quoted string expected after \\version").c_str ()); yy_pop_state (); } -. { - LexerError (_ ("No quoted string found after \\renameinput").to_str0 ()); +. { + LexerError (_ ("quoted string expected after \\sourcefilename").c_str ()); yy_pop_state (); } { @@ -212,8 +228,8 @@ HYPHEN -- yy_pop_state (); } <> { - LexerError (_ ("EOF found inside a comment").to_str0 ()); - main_input_b_ = false; + LexerError (_ ("EOF found inside a comment").c_str ()); + is_main_input_ = false; if (! close_input ()) yyterminate (); // can't move this, since it actually rets a YY_NULL } @@ -221,10 +237,10 @@ HYPHEN -- \\maininput { - if (!main_input_b_) + if (!is_main_input_) { start_main_input (); - main_input_b_ = true; + is_main_input_ = true; } else error (_ ("\\maininput not allowed outside init files")); @@ -234,49 +250,49 @@ HYPHEN -- yy_push_state (incl); } \"[^"]*\" { /* got the include file name */ - String s (YYText ()+1); - s = s.left_string (s.index_last ('"')); + string s (YYText ()+1); + s = s.substr (0, s.rfind ('"')); new_input (s, sources_); yy_pop_state (); } \\{BLACK}*{WHITE} { /* got the include identifier */ - String s = YYText () + 1; + string s = YYText () + 1; strip_trailing_white (s); if (s.length () && (s[s.length () - 1] == ';')) - s = s.left_string (s.length () - 1); + s = s.substr (0, s.length () - 1); SCM sid = lookup_identifier (s); if (scm_is_string (sid)) { new_input (ly_scm2string (sid), sources_); yy_pop_state (); } else { - String msg (_f ("wrong or undefined identifier: `%s'", s )); + string msg (_f ("wrong or undefined identifier: `%s'", s )); - LexerError (msg.to_str0 ()); + LexerError (msg.c_str ()); SCM err = scm_current_error_port (); scm_puts ("This value was found in the table: ", err); scm_display (sid, err); } } \"[^"]* { // backup rule - error (_ ("Missing end quote")); + error (_ ("end quote missing")); exit (1); } {RESTNAME} { - const char *s = YYText (); + char const *s = YYText (); yylval.scm = scm_makfrom0str (s); return RESTNAME; } R { return MULTI_MEASURE_REST; } -# { //embedded scm - //char const* s = YYText () + 1; - char const* s = here_str0 (); +# { //embedded scm int n = 0; - SCM sval = ly_parse_scm (s, &n, here_input (), - safe_global_b && main_input_b_); + Input hi = here_input(); + hi.step_forward (); + SCM sval = ly_parse_scm (hi.start (), &n, hi, + be_safe_global && is_main_input_); if (sval == SCM_UNDEFINED) { @@ -288,7 +304,7 @@ HYPHEN -- { yyinput (); } - char_count_stack_.top () += n; + char_count_stack_.back () += n; if (unpack_identifier (sval) != SCM_UNDEFINED) { @@ -333,15 +349,15 @@ HYPHEN -- } {DIGIT} { - yylval.i = String_convert::dec2int (String (YYText ())); + yylval.i = String_convert::dec2int (string (YYText ())); return DIGIT; } {UNSIGNED} { - yylval.i = String_convert::dec2int (String (YYText ())); + yylval.i = String_convert::dec2int (string (YYText ())); return UNSIGNED; } {E_UNSIGNED} { - yylval.i = String_convert::dec2int (String (YYText () +1)); + yylval.i = String_convert::dec2int (string (YYText () +1)); return E_UNSIGNED; } \" { @@ -364,8 +380,8 @@ HYPHEN -- yy_pop_state (); /* yylval is union. Must remember STRING before setting SCM*/ - String *sp = yylval.string; - yylval.scm = scm_makfrom0str (sp->to_str0 ()); + string *sp = yylval.string; + yylval.scm = scm_makfrom0str (sp->c_str ()); delete sp; return STRING; } @@ -385,8 +401,8 @@ HYPHEN -- yy_pop_state (); /* yylval is union. Must remember STRING before setting SCM*/ - String *sp = yylval.string; - yylval.scm = scm_makfrom0str (sp->to_str0 ()); + string *sp = yylval.string; + yylval.scm = scm_makfrom0str (sp->c_str ()); delete sp; return LYRICS_STRING; } @@ -404,7 +420,7 @@ HYPHEN -- return FRACTION; } {UNSIGNED} { - yylval.i = String_convert::dec2int (String (YYText ())); + yylval.i = String_convert::dec2int (string (YYText ())); return UNSIGNED; } {NOTECOMMAND} { @@ -412,7 +428,7 @@ HYPHEN -- } {LYRICS} { /* ugr. This sux. */ - String s (YYText ()); + string s (YYText ()); if (s == "__") return yylval.i = EXTENDER; if (s == "--") @@ -423,7 +439,7 @@ HYPHEN -- if (c == '{' || c == '}') // brace open is for not confusing dumb tools. here_input ().warning ( _ ("Brace found at end of lyric. Did you forget a space?")); - yylval.scm = scm_makfrom0str (s.to_str0 ()); + yylval.scm = scm_makfrom0str (s.c_str ()); return LYRICS_STRING; @@ -444,7 +460,7 @@ HYPHEN -- return FRACTION; } {UNSIGNED} { - yylval.i = String_convert::dec2int (String (YYText ())); + yylval.i = String_convert::dec2int (string (YYText ())); return UNSIGNED; } \" { @@ -479,7 +495,7 @@ HYPHEN -- return SCORE; } {MARKUPCOMMAND} { - String str (YYText () + 1); + string str (YYText () + 1); SCM s = lookup_markup_command (str); if (scm_is_pair (s) && scm_is_symbol (scm_cdr (s)) ) { @@ -504,7 +520,7 @@ HYPHEN -- else if (tag == ly_symbol2scm ("scheme0-scheme1-scheme2")) return MARKUP_HEAD_SCM0_SCM1_SCM2; else { - programming_error ("No parser tag defined for this signature. Abort"); + programming_error ("no parser tag defined for this markup signature"); ly_display_scm (s); assert(false); } @@ -515,14 +531,14 @@ HYPHEN -- return YYText ()[0]; } [^#{}"\\ \t\n\r\f]+ { - String s (YYText ()); + string s (YYText ()); char c = s[s.length () - 1]; /* brace open is for not confusing dumb tools. */ if (c == '{' || c == '}') here_input ().warning ( _ ("Brace found at end of markup. Did you forget a space?")); - yylval.scm = scm_makfrom0str (s.to_str0 ()); + yylval.scm = scm_makfrom0str (s.c_str ()); return STRING; @@ -533,9 +549,9 @@ HYPHEN -- } <> { - if (main_input_b_) + if (is_main_input_) { - main_input_b_ = false; + is_main_input_ = false; if (!close_input ()) /* Returns YY_NULL */ yyterminate (); @@ -545,6 +561,14 @@ HYPHEN -- yyterminate (); } +{ + {DASHED_WORD} { + return scan_bare_word (YYText ()); + } + {DASHED_KEY_WORD} { + return scan_escaped_word (YYText () + 1); + } +} {WORD} { return scan_bare_word (YYText ()); @@ -556,13 +580,14 @@ HYPHEN -- Real r; int cnv = sscanf (YYText (), "%lf", &r); assert (cnv == 1); + (void) cnv; - yylval.scm = scm_make_real (r); + yylval.scm = scm_from_double (r); return REAL; } {UNSIGNED} { - yylval.i = String_convert::dec2int (String (YYText ())); + yylval.i = String_convert::dec2int (string (YYText ())); return UNSIGNED; } @@ -586,9 +611,9 @@ HYPHEN -- switch (c) { case '>': - return E_BIGGER; + return E_ANGLE_CLOSE; case '<': - return E_SMALLER; + return E_ANGLE_OPEN; case '!': return E_EXCLAMATION; case '(': @@ -596,9 +621,11 @@ HYPHEN -- case ')': return E_CLOSE; case '[': - return E_LEFTSQUARE; + return E_BRACKET_OPEN; + case '+': + return E_PLUS; case ']': - return E_RIGHTSQUARE; + return E_BRACKET_CLOSE; case '~': return E_TILDE; case '\\': @@ -610,8 +637,8 @@ HYPHEN -- } <*>. { - String msg = _f ("invalid character: `%c'", YYText ()[0]); - LexerError (msg.to_str0 ()); + string msg = _f ("invalid character: `%c'", YYText ()[0]); + LexerError (msg.c_str ()); return YYText ()[0]; } @@ -672,21 +699,22 @@ Lily_lexer::identifier_type (SCM sid) int -Lily_lexer::scan_escaped_word (String str) +Lily_lexer::scan_escaped_word (string str) { // use more SCM for this. -// SCM sym = ly_symbol2scm (str.to_str0 ()); +// SCM sym = ly_symbol2scm (str.c_str ()); + + int i = lookup_keyword (str); + if (i == MARKUP && is_lyric_state ()) + return LYRIC_MARKUP; + if (i != -1) + return i; - int l = lookup_keyword (str); - if (l != -1) { - return l; - } SCM sid = lookup_identifier (str); if (is_music_function (sid)) { yylval.scm = get_music_function_transform (sid); - return music_function_type (yylval.scm); } @@ -696,18 +724,18 @@ Lily_lexer::scan_escaped_word (String str) return identifier_type (sid); } - String msg (_f ("unknown escaped string: `\\%s'", str)); - LexerError (msg.to_str0 ()); + string msg (_f ("unknown escaped string: `\\%s'", str)); + LexerError (msg.c_str ()); - yylval.scm = scm_makfrom0str (str.to_str0 ()); + yylval.scm = scm_makfrom0str (str.c_str ()); return STRING; } int -Lily_lexer::scan_bare_word (String str) +Lily_lexer::scan_bare_word (string str) { - SCM sym = ly_symbol2scm (str.to_str0 ()); + SCM sym = ly_symbol2scm (str.c_str ()); if ((YYSTATE == notes) || (YYSTATE == chords)) { SCM handle = SCM_BOOL_F; if (scm_is_pair (pitchname_tab_stack_)) @@ -727,7 +755,7 @@ Lily_lexer::scan_bare_word (String str) } } - yylval.scm = scm_makfrom0str (str.to_str0 ()); + yylval.scm = scm_makfrom0str (str.c_str ()); return STRING; } @@ -756,39 +784,38 @@ Lily_lexer::is_figure_state () const } /* - urg, belong to String (_convert) + urg, belong to string (_convert) and should be generalised */ void -strip_leading_white (String&s) +strip_leading_white (string&s) { - int i = 0; - for (; i < s.length (); i++) + ssize i = 0; + for (; i < s.length (); i++) if (!isspace (s[i])) break; - s = s.nomid_string (0, i); + s = s.substr (i); } void -strip_trailing_white (String&s) +strip_trailing_white (string&s) { - int i = s.length (); + ssize i = s.length (); while (i--) if (!isspace (s[i])) break; - s = s.left_string (i+1); + s = s.substr (0, i + 1); } -/* 2.1.2x something -> \property -> \set. */ -Lilypond_version oldest_version ("2.3.22"); +Lilypond_version oldest_version ("2.7.38"); bool -is_valid_version (String s) +is_valid_version (string s) { Lilypond_version current ( MAJOR_VERSION "." MINOR_VERSION "." PATCH_LEVEL ); Lilypond_version ver (s); @@ -805,25 +832,25 @@ is_valid_version (String s) /* substitute _ and \, */ -String -lyric_fudge (String s) +string +lyric_fudge (string s) { - char * chars = s.get_copy_str0 (); + char *chars = string_copy (s); - for (char * p = chars; *p ; p++) + for (char *p = chars; *p ; p++) { if (*p == '_' && (p == chars || *(p-1) != '\\')) *p = ' '; } - s = String (chars); + s = string (chars); delete[] chars; - int i = 0; - if ((i = s.index ("\\,")) != -1) // change "\," to TeX's "\c " + ssize i = 0; + if ((i = s.find ("\\,")) != NPOS) // change "\," to TeX's "\c " { - * (s.get_str0 () + i + 1) = 'c'; - s = s.left_string (i+2) + " " + s.right_string (s.length ()-i-2); + * (((char*)s.c_str ()) + i + 1) = 'c'; + s = s.substr (0, i + 2) + " " + s.substr (i - 2); } return s; @@ -833,107 +860,104 @@ lyric_fudge (String s) Convert "NUM/DEN" into a '(NUM . DEN) cons. */ SCM -scan_fraction (String frac) +scan_fraction (string frac) { - int i = frac.index ('/'); - int l = frac.length (); - String left = frac.left_string (i); - String right = frac.right_string (l - i - 1); + ssize i = frac.find ('/'); + string left = frac.substr (0, i); + string right = frac.substr (i + 1, (frac.length () - i + 1)); int n = String_convert::dec2int (left); int d = String_convert::dec2int (right); - return scm_cons (scm_int2num (n), scm_int2num (d)); -} - -// Breaks for flex 2.5.31 -#if 0 -/* avoid silly flex induced gcc warnings */ -static void yy_push_state (int) {;} -static void yy_pop_state () {;} -static int yy_top_state () { return 0; } - -static void -avoid_silly_flex_induced_gcc_warnings () -{ - (void)yy_start_stack_ptr; - (void)yy_start_stack_depth; - (void)yy_start_stack; - yy_push_state (0); - yy_pop_state (); - yy_top_state (); - avoid_silly_flex_induced_gcc_warnings (); + return scm_cons (scm_from_int (n), scm_from_int (d)); } -#endif SCM -lookup_markup_command (String s) +lookup_markup_command (string s) { SCM proc = ly_lily_module_constant ("lookup-markup-command"); - return scm_call_1 (proc, scm_makfrom0str (s.to_str0 ())); + return scm_call_1 (proc, scm_makfrom0str (s.c_str ())); } +struct Parser_signature +{ + char *symbol; + int token_type; +}; +static SCM signature_hash_table; + +static void init_signature_hash_table () +{ + signature_hash_table = scm_gc_protect_object (scm_c_make_hash_table (31)); + Parser_signature sigs[] = { + {"scm", MUSIC_FUNCTION_SCM}, + {"music", MUSIC_FUNCTION_MUSIC}, + {"scm-music", MUSIC_FUNCTION_SCM_MUSIC}, + {"scm-scm", MUSIC_FUNCTION_SCM_SCM}, + {"music-music", MUSIC_FUNCTION_MUSIC_MUSIC}, + {"scm-music-music", MUSIC_FUNCTION_SCM_MUSIC_MUSIC}, + {"scm-scm-music", MUSIC_FUNCTION_SCM_SCM_MUSIC}, + {"scm-scm-scm-music", MUSIC_FUNCTION_SCM_SCM_SCM_SCM_MUSIC}, + {"scm-scm-scm-scm-music", MUSIC_FUNCTION_SCM_SCM_SCM_MUSIC}, + {"scm-scm-scm", MUSIC_FUNCTION_SCM_SCM_SCM}, + {"markup", MUSIC_FUNCTION_MARKUP}, + {"markup-music", MUSIC_FUNCTION_MARKUP_MUSIC}, + {"markup-markup", MUSIC_FUNCTION_MARKUP_MARKUP}, + {"markup-music-music", MUSIC_FUNCTION_MARKUP_MUSIC_MUSIC}, + {"markup-markup-music", MUSIC_FUNCTION_MARKUP_MARKUP_MUSIC}, + {"noarg", MUSIC_FUNCTION}, + {0,0} + }; + + for (int i = 0; sigs[i].symbol; i++) + scm_hashq_set_x (signature_hash_table, scm_gc_protect_object (ly_symbol2scm (sigs[i].symbol)), + scm_from_int (sigs[i].token_type)); +} int music_function_type (SCM func) { + if (!signature_hash_table) + init_signature_hash_table (); + SCM type = scm_object_property (func, ly_symbol2scm ("music-function-signature-keyword")); - if (type == ly_symbol2scm ("scm")) - { - return MUSIC_FUNCTION_SCM; - } - else if (type == ly_symbol2scm ("music")) - { - return MUSIC_FUNCTION_MUSIC; - } - else if (type == ly_symbol2scm ("scm-music")) - { - return MUSIC_FUNCTION_SCM_MUSIC; - } - else if (type == ly_symbol2scm ("scm-scm")) - { - return MUSIC_FUNCTION_SCM_SCM; - } - else if (type == ly_symbol2scm ("music-music")) - { - return MUSIC_FUNCTION_MUSIC_MUSIC; - } - else if (type == ly_symbol2scm ("scm-music-music")) - { - return MUSIC_FUNCTION_SCM_MUSIC_MUSIC; - } - else if (type == ly_symbol2scm ("scm-scm-music")) - { - return MUSIC_FUNCTION_SCM_SCM_MUSIC; - } - else if (type == ly_symbol2scm ("markup")) - { - return MUSIC_FUNCTION_MARKUP; - } - else if (type == ly_symbol2scm ("markup-music")) - { - return MUSIC_FUNCTION_MARKUP_MUSIC; - } - else if (type == ly_symbol2scm ("markup-markup")) - { - return MUSIC_FUNCTION_MARKUP_MARKUP; - } - else if (type == ly_symbol2scm ("markup-music-music")) - { - return MUSIC_FUNCTION_MARKUP_MUSIC_MUSIC; - } - else if (type == ly_symbol2scm ("markup-markup-music")) - { - return MUSIC_FUNCTION_MARKUP_MARKUP_MUSIC; - } - else if (type == ly_symbol2scm ("noarg")) - { - return MUSIC_FUNCTION; - } - else + SCM token_type = scm_hashq_ref (signature_hash_table, type, SCM_BOOL_F); + if (!scm_is_number (token_type)) { - /* TODO: print location */ - error ("Can not find signature for music function."); + programming_error (_ ("can't find signature for music function")); + return MUSIC_FUNCTION_SCM; } + + return scm_to_int (token_type); +} + +/* Shut up lexer warnings. */ +#if YY_STACK_USED - return MUSIC_FUNCTION_SCM; +static void +yy_push_state (int) +{ } + +static void +yy_pop_state () +{ +} + +static int +yy_top_state () +{ + return 0; +} + +static void +silence_lexer_warnings () +{ + (void) yy_start_stack_ptr; + (void) yy_start_stack_depth; + (void) yy_start_stack; + (void) yy_push_state; + (void) yy_pop_state; + (void) yy_top_state; + (void) silence_lexer_warnings; +} +#endif