1 %{ // -*- mode: c++; c-file-style: "linux" -*-
3 This file is part of LilyPond, the GNU music typesetter.
5 Copyright (C) 1996--2011 Han-Wen Nienhuys <hanwen@xs4all.nl>
6 Jan Nieuwenhuizen <janneke@gnu.org>
8 LilyPond is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 3 of the License, or
11 (at your option) any later version.
13 LilyPond is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with LilyPond. If not, see <http://www.gnu.org/licenses/>.
22 /* Mode and indentation are at best a rough approximation based on TAB
23 * formatting (reasonable for compatibility with unspecific editor
24 * modes as Flex modes are hard to find) and need manual correction
25 * frequently. Without a reasonably dependable way of formatting a
26 * Flex file sensibly, there is little point in trying to fix the
27 * inconsistent state of indentation.
33 after making a change to the lexer rules, run
34 flex -b <this lexer file>
37 contains no backup states, but only the reminder
38 Compressed tables always back up.
39 (don-t forget to rm lex.yy.cc :-)
48 /* Flex >= 2.5.29 fix; FlexLexer.h's multiple include bracing breaks
49 when building the actual lexer. */
56 #include "context-def.hh"
57 #include "duration.hh"
58 #include "international.hh"
59 #include "interval.hh"
60 #include "lily-guile.hh"
61 #include "lily-lexer.hh"
62 #include "lily-parser.hh"
63 #include "lilypond-version.hh"
66 #include "music-function.hh"
67 #include "parse-scm.hh"
70 #include "source-file.hh"
71 #include "std-string.hh"
72 #include "string-convert.hh"
79 #define isatty HORRIBLEKLUDGE
81 void strip_trailing_white (string&);
82 void strip_leading_white (string&);
83 string lyric_fudge (string s);
84 SCM lookup_markup_command (string s);
85 SCM lookup_markup_list_command (string s);
86 bool is_valid_version (string s);
89 #define start_quote() \
90 yy_push_state (quote);\
91 yylval.string = new string
93 #define start_lyric_quote() \
94 yy_push_state (lyric_quote);\
95 yylval.string = new string
97 #define yylval (*lexval_)
99 #define yylloc (*lexloc_)
101 #define YY_USER_ACTION add_lexed_char (YYLeng ());
104 SCM scan_fraction (string);
105 SCM (* scm_parse_error_handler) (void *);
115 %option yyclass="Lily_lexer"
117 %option never-interactive
142 NATIONAL [\001-\006\021-\027\031\036]
143 TEX {AA}|-|{PUNCT}|{ACCENT}|{NATIONAL}|{SPECIAL_CHAR}
144 DASHED_WORD {A}({AN}|-)*
145 DASHED_KEY_WORD \\{DASHED_WORD}
154 REAL ({INT}\.{N}*)|(-?\.{N}+)
156 HORIZONTALWHITE [ \t]
160 MARKUPCOMMAND \\({A}|[-_])+
161 LYRICS ({AA}|{TEX})[^0-9 \t\n\r\f]*
165 BOM_UTF8 \357\273\277
171 // swallow and ignore carriage returns
174 <extratoken>{ANY_CHAR} {
175 /* Generate a token without swallowing anything */
177 /* First unswallow the eaten character */
178 add_lexed_char (-YYLeng ());
181 /* produce requested token */
182 int type = scm_to_int (scm_caar (extra_tokens_));
183 yylval.scm = scm_cdar (extra_tokens_);
184 extra_tokens_ = scm_cdr (extra_tokens_);
185 if (scm_is_null (extra_tokens_))
191 <extratoken><<EOF>> {
192 /* Generate a token without swallowing anything */
194 /* produce requested token */
195 int type = scm_to_int (scm_caar (extra_tokens_));
196 yylval.scm = scm_cdar (extra_tokens_);
197 extra_tokens_ = scm_cdr (extra_tokens_);
198 if (scm_is_null (extra_tokens_))
204 /* Use the trailing context feature. Otherwise, the BOM will not be
205 found if the file starts with an identifier definition. */
206 <INITIAL,chords,lyrics,figures,notes>{BOM_UTF8}/.* {
207 if (this->lexloc_->line_number () != 1 || this->lexloc_->column_number () != 0)
209 LexerWarning (_ ("stray UTF-8 BOM encountered").c_str ());
212 debug_output (_ ("Skipping UTF-8 BOM"));
215 <INITIAL,chords,figures,incl,lyrics,markup,notes>{
217 yy_push_state (longcomment);
219 %[^{\n\r][^\n\r]*[\n\r] {
221 %[^{\n\r] { // backup rule
232 <INITIAL,notes,figures,chords,markup>{
238 <INITIAL,chords,lyrics,notes,figures>\\version{WHITE}* {
239 yy_push_state (version);
241 <INITIAL,chords,lyrics,notes,figures>\\sourcefilename{WHITE}* {
242 yy_push_state (sourcefilename);
244 <INITIAL,chords,lyrics,notes,figures>\\sourcefileline{WHITE}* {
245 yy_push_state (sourcefileline);
247 <version>\"[^"]*\" { /* got the version number */
248 string s (YYText () + 1);
249 s = s.substr (0, s.rfind ('\"'));
253 SCM top_scope = scm_car (scm_last_pair (scopes_));
254 scm_module_define (top_scope, ly_symbol2scm ("version-seen"), SCM_BOOL_T);
256 if (!is_valid_version (s))
261 <sourcefilename>\"[^"]*\" {
262 string s (YYText () + 1);
263 s = s.substr (0, s.rfind ('\"'));
266 this->here_input().get_source_file ()->name_ = s;
267 message (_f ("Renaming input to: `%s'", s.c_str ()));
268 progress_indication ("\n");
269 scm_module_define (scm_car (scopes_),
270 ly_symbol2scm ("input-file-name"),
275 <sourcefileline>{INT} {
277 sscanf (YYText (), "%d", &i);
280 this->here_input ().get_source_file ()->set_line (here_input ().start (), i);
283 <version>{ANY_CHAR} {
284 LexerError (_ ("quoted string expected after \\version").c_str ());
287 <sourcefilename>{ANY_CHAR} {
288 LexerError (_ ("quoted string expected after \\sourcefilename").c_str ());
291 <sourcefileline>{ANY_CHAR} {
292 LexerError (_ ("integer expected after \\sourcefileline").c_str ());
307 <INITIAL,chords,lyrics,notes,figures>\\maininput {
311 is_main_input_ = true;
314 error (_ ("\\maininput not allowed outside init files"));
317 <INITIAL,chords,lyrics,figures,notes>\\include {
318 yy_push_state (incl);
320 <incl>\"[^""]*\" { /* got the include file name */
321 string s (YYText ()+1);
322 s = s.substr (0, s.rfind ('"'));
324 new_input (s, sources_);
327 <incl>\\{BLACK}*{WHITE}? { /* got the include identifier */
328 string s = YYText () + 1;
329 strip_trailing_white (s);
330 if (s.length () && (s[s.length () - 1] == ';'))
331 s = s.substr (0, s.length () - 1);
333 SCM sid = lookup_identifier (s);
334 if (scm_is_string (sid)) {
335 new_input (ly_scm2string (sid), sources_);
338 string msg (_f ("wrong or undefined identifier: `%s'", s ));
340 LexerError (msg.c_str ());
341 SCM err = scm_current_error_port ();
342 scm_puts ("This value was found in the table: ", err);
343 scm_display (sid, err);
346 <incl>(\$|#) { // scm for the filename
348 Input hi = here_input();
350 SCM sval = ly_parse_scm (hi.start (), &n, hi,
351 be_safe_global && is_main_input_, parser_);
352 sval = eval_scm (sval);
354 for (int i = 0; i < n; i++)
358 char_count_stack_.back () += n;
360 if (scm_is_string (sval)) {
361 new_input (ly_scm2string (sval), sources_);
364 LexerError (_ ("string expected after \\include").c_str ());
365 if (sval != SCM_UNDEFINED) {
366 SCM err = scm_current_error_port ();
367 scm_puts ("This value was found instead: ", err);
368 scm_display (sval, err);
373 <incl,version,sourcefilename>\"[^""]* { // backup rule
374 error (_ ("end quote missing"));
377 <chords,notes,figures>{RESTNAME} {
378 char const *s = YYText ();
379 yylval.scm = scm_from_locale_string (s);
382 <chords,notes,figures>R {
383 return MULTI_MEASURE_REST;
385 <INITIAL,chords,figures,lyrics,markup,notes># { //embedded scm
387 Input hi = here_input();
389 SCM sval = ly_parse_scm (hi.start (), &n, hi,
390 be_safe_global && is_main_input_, parser_);
392 if (sval == SCM_UNDEFINED)
395 for (int i = 0; i < n; i++)
399 char_count_stack_.back () += n;
405 <INITIAL,chords,figures,lyrics,markup,notes>\$ { //immediate scm
407 Input hi = here_input();
409 SCM sval = ly_parse_scm (hi.start (), &n, hi,
410 be_safe_global && is_main_input_, parser_);
412 for (int i = 0; i < n; i++)
416 char_count_stack_.back () += n;
418 sval = eval_scm (sval);
420 int token = scan_scm_id (sval);
421 if (!scm_is_eq (yylval.scm, SCM_UNSPECIFIED))
425 <INITIAL,notes,lyrics>{
427 return DOUBLE_ANGLE_OPEN;
430 return DOUBLE_ANGLE_CLOSE;
457 return scan_bare_word (YYText ());
461 return scan_escaped_word (YYText () + 1);
464 yylval.scm = scan_fraction (YYText ());
467 {UNSIGNED}/\/ | // backup rule
469 yylval.scm = scm_c_read_string (YYText ());
473 yylval.i = String_convert::dec2int (string (YYText () +1));
480 *yylval.string += to_string (escaped_char (YYText ()[1]));
483 *yylval.string += YYText ();
489 /* yylval is union. Must remember STRING before setting SCM*/
490 string *sp = yylval.string;
491 yylval.scm = ly_string2scm (*sp);
493 return is_lyric_state () ? LYRICS_STRING : STRING;
496 *yylval.string += YYText ();
502 start_lyric_quote ();
505 yylval.scm = scan_fraction (YYText ());
508 {UNSIGNED}/\/[^0-9] { // backup rule
509 yylval.scm = scm_c_read_string (YYText ());
512 {UNSIGNED}/\/ | // backup rule
514 yylval.scm = scm_c_read_string (YYText ());
518 return scan_escaped_word (YYText () + 1);
522 string s (YYText ());
524 return yylval.i = EXTENDER;
526 return yylval.i = HYPHEN;
529 char c = s[s.length () - 1];
530 if (c == '{' || c == '}') // brace open is for not confusing dumb tools.
531 here_input ().warning (
532 _ ("Brace found at end of lyric. Did you forget a space?"));
533 yylval.scm = ly_string2scm (s);
536 return LYRICS_STRING;
544 return scan_bare_word (YYText ());
547 return scan_escaped_word (YYText () + 1);
550 yylval.scm = scan_fraction (YYText ());
553 {UNSIGNED}/\/[^0-9] { // backup rule
554 yylval.scm = scm_c_read_string (YYText ());
557 {UNSIGNED}/\/ | // backup rule
559 yylval.scm = scm_c_read_string (YYText ());
588 string str (YYText () + 1);
590 int token_type = MARKUP_FUNCTION;
591 SCM s = lookup_markup_command (str);
593 // lookup-markup-command returns a pair with the car
594 // being the function to call, and the cdr being the
595 // call signature specified to define-markup-command,
596 // a list of predicates.
598 if (!scm_is_pair (s)) {
599 // If lookup-markup-command was not successful, we
600 // try lookup-markup-list-command instead.
601 // If this fails as well, we just scan and return
603 s = lookup_markup_list_command (str);
605 token_type = MARKUP_LIST_FUNCTION;
607 return scan_escaped_word (str);
610 // If the list of predicates is, say,
611 // (number? number? markup?), then tokens
612 // EXPECT_MARKUP EXPECT_SCM EXPECT_SCM EXPECT_NO_MORE_ARGS
613 // will be generated. Note that we have to push them
614 // in reverse order, so the first token pushed in the
615 // loop will be EXPECT_NO_MORE_ARGS.
617 yylval.scm = scm_car(s);
619 // yylval now contains the function to call as token
620 // value (for token type MARKUP_FUNCTION or
621 // MARKUP_LIST_FUNCTION).
623 push_extra_token(EXPECT_NO_MORE_ARGS);
625 for (; scm_is_pair(s); s = scm_cdr(s)) {
626 SCM predicate = scm_car(s);
628 if (predicate == ly_lily_module_constant ("markup-list?"))
629 push_extra_token(EXPECT_MARKUP_LIST);
630 else if (predicate == ly_lily_module_constant ("markup?"))
631 push_extra_token(EXPECT_MARKUP);
633 push_extra_token(EXPECT_SCM, predicate);
640 [^$#{}\"\\ \t\n\r\f]+ {
641 string s (YYText ());
643 char c = s[s.length () - 1];
644 /* brace open is for not confusing dumb tools. */
645 if (c == '{' || c == '}')
646 here_input ().warning (
647 _ ("Brace found at end of markup. Did you forget a space?"));
648 yylval.scm = ly_string2scm (s);
658 <longcomment><<EOF>> {
659 LexerError (_ ("EOF found inside a comment").c_str ());
660 is_main_input_ = false; // should be safe , can't have \include in --safe.
662 yyterminate (); // can't move this, since it actually rets a YY_NULL
665 <<EOF>> { if (is_main_input_)
667 /* 2 = init.ly + current file.
668 > because we're before closing, but is_main_input_ should
671 is_main_input_ = include_stack_.size () > 2;
673 /* Returns YY_NULL */
676 else if (!close_input ())
677 /* Returns YY_NULL */
683 return scan_bare_word (YYText ());
686 return scan_escaped_word (YYText () + 1);
690 -{UNSIGNED} | // backup rule
692 yylval.scm = scm_c_read_string (YYText ());
696 yylval.scm = scm_from_double (0.0);
701 yylval.scm = scm_c_read_string (YYText ());
711 char c = YYText ()[0];
716 <INITIAL,notes,figures>. {
720 <INITIAL,lyrics,notes,figures>\\. {
721 char c = YYText ()[1];
725 return E_ANGLE_CLOSE;
729 return E_EXCLAMATION;
735 return E_BRACKET_OPEN;
739 return E_BRACKET_CLOSE;
751 string msg = _f ("invalid character: `%c'", YYText ()[0]);
752 LexerError (msg.c_str ());
758 /* Make the lexer generate a token of the given type as the next token.
759 TODO: make it possible to define a value for the token as well */
761 Lily_lexer::push_extra_token (int token_type, SCM scm)
763 if (scm_is_null (extra_tokens_))
765 if (YY_START != extratoken)
766 hidden_state_ = YY_START;
767 yy_push_state (extratoken);
769 extra_tokens_ = scm_acons (scm_from_int (token_type), scm, extra_tokens_);
773 Lily_lexer::push_chord_state (SCM tab)
775 pitchname_tab_stack_ = scm_cons (tab, pitchname_tab_stack_);
776 yy_push_state (chords);
780 Lily_lexer::push_figuredbass_state ()
782 yy_push_state (figures);
786 Lily_lexer::push_initial_state ()
788 yy_push_state (INITIAL);
792 Lily_lexer::push_lyric_state ()
794 yy_push_state (lyrics);
798 Lily_lexer::push_markup_state ()
800 yy_push_state (markup);
804 Lily_lexer::push_note_state (SCM tab)
806 pitchname_tab_stack_ = scm_cons (tab, pitchname_tab_stack_);
807 yy_push_state (notes);
811 Lily_lexer::pop_state ()
813 if (YYSTATE == notes || YYSTATE == chords)
814 pitchname_tab_stack_ = scm_cdr (pitchname_tab_stack_);
820 Lily_lexer::identifier_type (SCM sid)
822 int k = try_special_identifiers (&yylval.scm , sid);
823 return k >= 0 ? k : SCM_IDENTIFIER;
828 Lily_lexer::scan_escaped_word (string str)
830 // use more SCM for this.
832 // SCM sym = ly_symbol2scm (str.c_str ());
834 int i = lookup_keyword (str);
835 if (i == MARKUP && is_lyric_state ())
840 SCM sid = lookup_identifier (str);
841 if (sid != SCM_UNDEFINED)
842 return scan_scm_id (sid);
844 string msg (_f ("unknown escaped string: `\\%s'", str));
845 LexerError (msg.c_str ());
847 yylval.scm = ly_string2scm (str);
853 Lily_lexer::scan_scm_id (SCM sid)
855 if (is_music_function (sid))
857 int funtype = SCM_FUNCTION;
859 yylval.scm = get_music_function_transform (sid);
861 SCM s = scm_object_property (yylval.scm, ly_symbol2scm ("music-function-signature"));
862 SCM cs = scm_car (s);
864 if (scm_is_pair (cs))
869 if (scm_is_eq (cs, ly_lily_module_constant ("ly:music?")))
870 funtype = MUSIC_FUNCTION;
871 else if (scm_is_eq (cs, ly_lily_module_constant ("ly:event?")))
872 funtype = EVENT_FUNCTION;
873 else if (ly_is_procedure (cs))
874 funtype = SCM_FUNCTION;
875 else programming_error ("Bad syntax function predicate");
877 push_extra_token (EXPECT_NO_MORE_ARGS);
878 for (s = scm_cdr (s); scm_is_pair (s); s = scm_cdr (s))
880 SCM optional = SCM_UNDEFINED;
883 if (scm_is_pair (cs))
885 optional = SCM_CDR (cs);
889 if (cs == Pitch_type_p_proc)
890 push_extra_token (EXPECT_PITCH);
891 else if (cs == Duration_type_p_proc)
892 push_extra_token (EXPECT_DURATION);
893 else if (ly_is_procedure (cs))
894 push_extra_token (EXPECT_SCM, cs);
897 programming_error ("Function parameter without type-checking predicate");
900 if (!scm_is_eq (optional, SCM_UNDEFINED))
901 push_extra_token (EXPECT_OPTIONAL, optional);
906 return identifier_type (sid);
910 Lily_lexer::scan_bare_word (string str)
912 SCM sym = ly_symbol2scm (str.c_str ());
913 if ((YYSTATE == notes) || (YYSTATE == chords)) {
914 SCM handle = SCM_BOOL_F;
915 if (scm_is_pair (pitchname_tab_stack_))
916 handle = scm_hashq_get_handle (scm_car (pitchname_tab_stack_), sym);
918 if (scm_is_pair (handle)) {
919 yylval.scm = scm_cdr (handle);
920 if (unsmob_pitch (yylval.scm))
921 return (YYSTATE == notes) ? NOTENAME_PITCH : TONICNAME_PITCH;
922 else if (scm_is_symbol (yylval.scm))
925 else if ((YYSTATE == chords)
926 && (handle = scm_hashq_get_handle (chordmodifier_tab_, sym))!= SCM_BOOL_F)
928 yylval.scm = scm_cdr (handle);
929 return CHORD_MODIFIER;
931 if ((chord_repetition_.repetition_symbol_ != SCM_EOL)
932 && to_boolean (scm_equal_p (chord_repetition_.repetition_symbol_, sym)))
933 return CHORD_REPETITION;
936 yylval.scm = ly_string2scm (str);
941 Lily_lexer::get_state () const
943 if (YY_START == extratoken)
944 return hidden_state_;
950 Lily_lexer::is_note_state () const
952 return get_state () == notes;
956 Lily_lexer::is_chord_state () const
958 return get_state () == chords;
962 Lily_lexer::is_lyric_state () const
964 return get_state () == lyrics;
968 Lily_lexer::is_figure_state () const
970 return get_state () == figures;
974 Lily_lexer::eval_scm (SCM readerdata)
976 SCM sval = SCM_UNDEFINED;
978 if (!SCM_UNBNDP (readerdata))
980 sval = ly_eval_scm (scm_car (readerdata),
981 *unsmob_input (scm_cdr (readerdata)),
982 be_safe_global && is_main_input_,
986 if (SCM_UNBNDP (sval))
989 return SCM_UNSPECIFIED;
997 urg, belong to string (_convert)
998 and should be generalised
1001 strip_leading_white (string&s)
1004 for (; i < s.length (); i++)
1005 if (!isspace (s[i]))
1012 strip_trailing_white (string&s)
1014 ssize i = s.length ();
1016 if (!isspace (s[i]))
1019 s = s.substr (0, i + 1);
1024 Lilypond_version oldest_version ("2.7.38");
1028 is_valid_version (string s)
1030 Lilypond_version current ( MAJOR_VERSION "." MINOR_VERSION "." PATCH_LEVEL );
1031 Lilypond_version ver (s);
1032 if (int (ver) < oldest_version)
1034 non_fatal_error (_f ("file too old: %s (oldest supported: %s)", ver.to_string (), oldest_version.to_string ()));
1035 non_fatal_error (_ ("consider updating the input with the convert-ly script"));
1041 non_fatal_error (_f ("program too old: %s (file requires: %s)", current.to_string (), ver.to_string ()));
1052 lyric_fudge (string s)
1056 while ((i = s.find ('_', i)) != string::npos)
1064 Convert "NUM/DEN" into a '(NUM . DEN) cons.
1067 scan_fraction (string frac)
1069 ssize i = frac.find ('/');
1070 string left = frac.substr (0, i);
1071 string right = frac.substr (i + 1, (frac.length () - i + 1));
1073 int n = String_convert::dec2int (left);
1074 int d = String_convert::dec2int (right);
1075 return scm_cons (scm_from_int (n), scm_from_int (d));
1079 lookup_markup_command (string s)
1081 SCM proc = ly_lily_module_constant ("lookup-markup-command");
1082 return scm_call_1 (proc, ly_string2scm (s));
1086 lookup_markup_list_command (string s)
1088 SCM proc = ly_lily_module_constant ("lookup-markup-list-command");
1089 return scm_call_1 (proc, ly_string2scm (s));
1092 /* Shut up lexer warnings. */
1112 silence_lexer_warnings ()
1114 (void) yy_start_stack_ptr;
1115 (void) yy_start_stack_depth;
1116 (void) yy_start_stack;
1117 (void) yy_push_state;
1118 (void) yy_pop_state;
1119 (void) yy_top_state;
1120 (void) silence_lexer_warnings;