lily/lexer.l

   1 %{ // -*-Fundamental-*-
   2 /*
   3   lexer.l -- implement the Flex lexer
   4
   5   source file of the LilyPond music typesetter
   6
   7   (c) 1996,1997 Han-Wen Nienhuys <hanwen@stack.nl>
   8 */
   9
  10
  11 /*
  12   backup rules
  13
  14   after making a change to the lexer rules, run
  15       flex -b <this lexer file>
  16   and make sure that
  17       lex.backup
  18   contains no backup states, but only the reminder
  19       Compressed tables always back up.
  20   (don-t forget to rm lex.yy.cc :-)
  21  */
  22
  23
  24 #include <stdio.h>
  25 #include <ctype.h>
  26
  27 #include "string.hh"
  28 #include "string-convert.hh"
  29 #include "my-lily-lexer.hh"
  30 #include "varray.hh"
  31 #include "parser.hh"
  32 #include "debug.hh"
  33 #include "parseconstruct.hh"
  34 #include "main.hh"
  35 #include "identifier.hh"
  36 void strip_trailing_white(String&);
  37 void strip_leading_white(String&);
  38
  39
  40 #define start_quote()   \
  41         yy_push_state(quote);\
  42         yylval.string = new String
  43
  44 #define yylval (*(YYSTYPE*)lexval_l)
  45
  46 #define YY_USER_ACTION  add_lexed_char(YYLeng());
  47 %}
  48
  49 %option c++
  50 %option noyywrap
  51 %option nodefault
  52 %option debug
  53 %option yyclass="My_lily_lexer"
  54 %option stack
  55 %option never-interactive
  56 %option warn
  57
  58 %x incl
  59 %x header
  60 %x lyrics
  61 %x notes
  62 %x quote
  63 %x longcomment
  64
  65
  66 A               [a-zA-Z]
  67 AA              {A}|_
  68 N               [0-9]
  69 AN              {AA}|{N}
  70 PUNCT           [?!,.:;']
  71 ACCENT          \\[`'"^]
  72 NATIONAL        [\241-\377]
  73 TEX             {AA}|-|{PUNCT}|{ACCENT}|{NATIONAL}
  74
  75 WORD            {A}{AN}*
  76 ALPHAWORD       {A}+
  77 INT             -?{N}+
  78 REAL            ({INT}\.{N}*)|(-?\.{N}+)
  79 KEYWORD         \\{WORD}
  80 WHITE           [ \n\t\f]
  81 HORIZONTALWHITE         [ \t]
  82 BLACK           [^ \n\t\f]
  83 RESTNAME        [rs]
  84 NOTECOMMAND     \\{A}+
  85 LYRICS          ({AA}|{NATIONAL})[^0-9 \t\n\f]*
  86
  87 %%
  88
  89
  90 <notes,incl,INITIAL,lyrics>{
  91   "%{"  {
  92         yy_push_state(longcomment);
  93   }
  94   %[^{\n].*\n   {
  95   }
  96   %[^{\n]       { // backup rule
  97   }
  98   %\n   {
  99   }
 100   %[^{\n].*     {
 101   }
 102   {WHITE}+      {
 103
 104   }
 105 }
 106
 107 <longcomment>{
 108         [^\%]*          {
 109         }
 110         \%*[^}%]*               {
 111
 112         }
 113         "%"+"}"         {
 114                 yy_pop_state();
 115         }
 116         <<EOF>>         {
 117                 LexerError("EOF found inside a comment");
 118                 if (! close_input())
 119                   yyterminate(); // can't move this, since it actually rets a YY_NULL
 120         }
 121 }
 122 <header>{
 123         [\{\}]  {
 124                 return YYText()[0];
 125         }
 126         ^{WORD}         {
 127                 String s=YYText();
 128                 yylval.string = new String(s);
 129                 return FIELDNAME;
 130         }
 131         {HORIZONTALWHITE}+{BLACK}.*\n           {
 132                 String s=YYText();
 133                 strip_leading_white(s);
 134                 strip_trailing_white(s);
 135                 yylval.string = new String(s);
 136                 return RECORDLINE;
 137         }
 138         {WHITE}*        {
 139         }
 140
 141         .               {
 142                 return YYText()[0];
 143         }
 144 }
 145
 146
 147 <notes,INITIAL,lyrics>\\include           {
 148         yy_push_state(incl);
 149 }
 150 <incl>\"[^"]*\"   { /* got the include file name */
 151         String s (YYText()+1);
 152         s = s.left_str(s.length_i()-1);
 153         DOUT << "#include `" << s << "\'\n";
 154         new_input(s,source_l_g);
 155         yy_pop_state();
 156 }
 157 <incl>\"[^"]*   { // backup rule
 158         cerr << "missing end quote" << endl;
 159         exit( 1 );
 160 }
 161 <notes>{RESTNAME}       {
 162         const char *s = YYText();
 163         yylval.string = new String (s);
 164         DOUT << "rest:"<< yylval.string;
 165         return RESTNAME;
 166 }
 167 <INITIAL,lyrics,notes>\\\${BLACK}*{WHITE}       {
 168         String s=YYText() + 2;
 169         s=s.left_str(s.length_i() - 1);
 170         return scan_escaped_word(s);
 171 }
 172 <INITIAL,lyrics,notes>\${BLACK}*{WHITE}         {
 173         String s=YYText() + 1;
 174         s=s.left_str(s.length_i() - 1);
 175         return scan_bare_word(s);
 176 }
 177 <INITIAL,lyrics,notes>\\\${BLACK}*              { // backup rule
 178         cerr << "white expected" << endl;
 179         exit( 1 );
 180 }
 181 <INITIAL,lyrics,notes>\${BLACK}*                { // backup rule
 182         cerr << "white expected" << endl;
 183         exit( 1 );
 184 }
 185 <notes>{
 186         {ALPHAWORD}/\'  {
 187                 post_quotes_b_ = true;
 188                 return scan_bare_word(YYText());
 189         }
 190         \'+             {
 191                 yylval.i = YYLeng();
 192                 if (post_quotes_b_) {
 193                         post_quotes_b_ = false;
 194                         return POST_QUOTES;
 195                 } else
 196                         return PRE_QUOTES;
 197         }
 198         {ALPHAWORD}     {
 199                 return scan_bare_word(YYText());
 200
 201         }
 202
 203         {NOTECOMMAND}   {
 204                 return scan_escaped_word(YYText()+1);
 205         }
 206
 207         {INT}           {
 208                 yylval.i = String_convert::dec2_i( String( YYText() ) );
 209                 return INT;
 210         }
 211
 212         \" {
 213                 start_quote();
 214         }
 215 }
 216
 217 \"              {
 218         start_quote();
 219 }
 220 <quote>{
 221         \\\\    {
 222                 *yylval.string += '\\';
 223         }
 224         \\\"    {
 225                 *yylval.string +='\"';
 226         }
 227         [^"]+   {
 228                 *yylval.string += YYText();
 229         }
 230         \"      {
 231                 DOUT << "quoted string: `" << *yylval.string << "'\n";
 232                 yy_pop_state();
 233                 return STRING;
 234         }
 235 }
 236
 237 <lyrics>{
 238
 239         \" {
 240                 start_quote();
 241         }
 242         {INT}           {
 243                 yylval.i = String_convert::dec2_i( String( YYText() ) );
 244                 return INT;
 245         }
 246         {NOTECOMMAND}   {
 247                 return scan_escaped_word(YYText()+1);
 248         }
 249         {LYRICS} {
 250                 /* ugr. This sux. */
 251                 String s (YYText());
 252                 int i = 0;
 253                 while ((i=s.index_i("_")) != -1) // change word binding "_" to " "
 254                         *(s.ch_l() + i) = ' ';
 255                 if ((i=s.index_i("\\,")) != -1)   // change "\," to TeX's "\c "
 256                         {
 257                         *(s.ch_l() + i + 1) = 'c';
 258                         s = s.left_str(i+2) + " " + s.right_str(s.length_i()-i-2);
 259                         }
 260                 yylval.string = new String(s);
 261                 DOUT << "lyric : `" << s << "'\n";
 262                 return STRING;
 263         }
 264         . {
 265                 return yylval.c = YYText()[0];
 266         }
 267 }
 268
 269 <<EOF>> {
 270         DOUT << "<<eof>>";
 271
 272         if (! close_input()) {
 273           yyterminate(); // can't move this, since it actually rets a YY_NULL
 274         }
 275 }
 276 {WORD}  {
 277         return scan_bare_word(YYText());
 278 }
 279 {KEYWORD}       {
 280         return scan_escaped_word(YYText()+1);
 281 }
 282 {REAL}          {
 283         Real r;
 284         int cnv=sscanf (YYText(), "%lf", &r);
 285         assert(cnv == 1);
 286         DOUT  << "REAL" << r<<'\n';
 287         yylval.real = r;
 288         return REAL;
 289 }
 290
 291 {INT}   {
 292         yylval.i = String_convert::dec2_i( String( YYText() ) );
 293         return INT;
 294 }
 295
 296 [{}]    {
 297
 298         DOUT << "parens\n";
 299         return YYText()[0];
 300 }
 301 [*:=]           {
 302         char c = YYText()[0];
 303         DOUT << "misc char" <<c<<"\n";
 304         return c;
 305 }
 306
 307 <INITIAL,notes>.        {
 308         return yylval.c = YYText()[0];
 309 }
 310
 311 <INITIAL,lyrics,notes>\\. {
 312     char c= YYText()[1];
 313     yylval.c = c;
 314     switch (c) {
 315     case '>':
 316         return E_BIGGER;
 317     case '<':
 318         return E_SMALLER;
 319     case '!':
 320         return E_EXCLAMATION;
 321     default:
 322         return E_CHAR;
 323     }
 324 }
 325
 326 <*>.            {
 327         LexerError( String( "illegal character: " ) +String( YYText()[0] ));
 328         return YYText()[0];
 329 }
 330
 331 %%
 332
 333 void
 334 My_lily_lexer::push_note_state()
 335 {
 336         yy_push_state(notes);
 337 }
 338
 339 void
 340 My_lily_lexer::push_lyric_state()
 341 {
 342         yy_push_state(lyrics);
 343 }
 344 void
 345 My_lily_lexer::pop_state()
 346 {
 347         yy_pop_state();
 348 }
 349
 350 int
 351 My_lily_lexer::scan_escaped_word(String str)
 352 {
 353         DOUT << "\\word: `" << str<<"'\n";
 354         int l = lookup_keyword(str);
 355         if (l != -1) {
 356                 DOUT << "(keyword)\n";
 357                 return l;
 358         }
 359         Identifier * id = lookup_identifier(str);
 360         if (id) {
 361                 DOUT << "(identifier)\n";
 362                 yylval.id = id;
 363                 return id->token_code_i_;
 364         }
 365         if ( YYSTATE != notes ) {
 366                 Melodic_req * mel_l = lookup_melodic_req_l(str);
 367                 if (mel_l) {
 368                     DOUT << "(notename)\n";
 369                     yylval.melreq = mel_l;
 370                     return NOTENAME_ID;
 371                 }
 372         }
 373         LexerError( "Unknown escaped string: `" + str + "'");
 374         DOUT << "(string)";
 375         String *sp = new String( str);
 376         yylval.string=sp;
 377         return STRING;
 378 }
 379
 380 int
 381 My_lily_lexer::scan_bare_word(String str)
 382 {
 383         DOUT << "word: `" << str<< "'\n";
 384         if (YYSTATE == notes){
 385                 Melodic_req * mel_l = lookup_melodic_req_l(str);
 386                 if (mel_l) {
 387                     DOUT << "(notename)\n";
 388                     yylval.melreq = mel_l;
 389                     return NOTENAME_ID;
 390                 }
 391         }
 392
 393         yylval.string=new String( str );
 394         return STRING;
 395 }
 396
 397 bool
 398 My_lily_lexer::note_state_b() const
 399 {
 400         return YY_START == notes;
 401 }
 402
 403 bool
 404 My_lily_lexer::lyric_state_b() const
 405 {
 406         return YY_START == lyrics;
 407 }
 408
 409 void
 410 My_lily_lexer::push_header_state()
 411 {
 412         yy_push_state(header);
 413 }
 414
 415 void strip_trailing_white(String&s)
 416 {
 417         int i=0;
 418         for (;  i < s.length_i(); i++)
 419                 if (!isspace(s[i]))
 420                         break;
 421
 422         s = s.nomid_str(0, i);
 423 }
 424 void strip_leading_white(String&s)
 425 {
 426         int i=s.length_i();
 427         while (i--)
 428                 if (!isspace(s[i]))
 429                         break;
 430
 431         s = s.left_str(i+1);
 432
 433 }