lexer.ll: Allow push_extra_token to take a Scheme value as well.

diff --git a/lily/include/lily-lexer.hh b/lily/include/lily-lexer.hh
index 9729ca701664d8cbaa28277408e62c6cc1e434aa..54e068ee7afb3a8a8e8e3263c070ba13ae75e582 100644 (file)
--- a/lily/include/lily-lexer.hh
+++ b/lily/include/lily-lexer.hh
@@ -62,7 +62,7 @@ private:
   SCM start_module_;
   int hidden_state_;
 public:
-  vector<int> extra_token_types_;
+  SCM extra_tokens_;
   void *lexval_;
   Input *lexloc_;
   bool is_main_input_;
@@ -101,7 +101,7 @@ public:
   SCM keyword_list () const;
   SCM lookup_identifier (string s);
   SCM lookup_identifier_symbol (SCM s);
-  void push_extra_token (int token_type);
+  void push_extra_token (int token_type, SCM scm = SCM_UNDEFINED);
   void push_chord_state (SCM tab);
   void push_figuredbass_state ();
   void push_lyric_state ();

diff --git a/lily/lexer.ll b/lily/lexer.ll
index 6b19c8f67dd8aed1d205736a1fe8a182afb5af78..8a07f1b2c00e64cd74208bdc6fdffc479b711746 100644 (file)
--- a/lily/lexer.ll
+++ b/lily/lexer.ll
@@ -173,9 +173,10 @@ BOM_UTF8   \357\273\277
   yyless (0);
 
   /* produce requested token */
-  int type = extra_token_types_.back ();
-  extra_token_types_.pop_back ();
-  if (extra_token_types_.empty ())
+  int type = scm_to_int (scm_caar (extra_tokens_));
+  yylval.scm = scm_cdar (extra_tokens_);
+  extra_tokens_ = scm_cdr (extra_tokens_);
+  if (scm_is_null (extra_tokens_))
     yy_pop_state ();
 
   return type;
@@ -185,9 +186,10 @@ BOM_UTF8   \357\273\277
   /* Generate a token without swallowing anything */
 
   /* produce requested token */
-  int type = extra_token_types_.back ();
-  extra_token_types_.pop_back ();
-  if (extra_token_types_.empty ())
+  int type = scm_to_int (scm_caar (extra_tokens_));
+  yylval.scm = scm_cdar (extra_tokens_);
+  extra_tokens_ = scm_cdr (extra_tokens_);
+  if (scm_is_null (extra_tokens_))
     yy_pop_state ();
 
   return type;
@@ -721,15 +723,15 @@ BOM_UTF8  \357\273\277
 /* Make the lexer generate a token of the given type as the next token. 
  TODO: make it possible to define a value for the token as well */
 void
-Lily_lexer::push_extra_token (int token_type)
+Lily_lexer::push_extra_token (int token_type, SCM scm)
 {
-       if (extra_token_types_.empty ())
+       if (scm_is_null (extra_tokens_))
        {
                if (YY_START != extratoken)
                        hidden_state_ = YY_START;
                yy_push_state (extratoken);
        }
-       extra_token_types_.push_back (token_type);
+       extra_tokens_ = scm_acons (scm_from_int (token_type), scm, extra_tokens_);
 }
 
 void

diff --git a/lily/lily-lexer.cc b/lily/lily-lexer.cc
index 4bba139d3fd516581c2309d1b0b75b04c5771f5b..f80ea67703010594d088202f86e12702d27879c8 100644 (file)
--- a/lily/lily-lexer.cc
+++ b/lily/lily-lexer.cc
@@ -105,6 +105,7 @@ Lily_lexer::Lily_lexer (Sources *sources, Lily_parser *parser)
   is_main_input_ = false;
   start_module_ = SCM_EOL;
   chord_repetition_ = Chord_repetition ();
+  extra_tokens_ = SCM_EOL;
   smobify_self ();
 
   add_scope (ly_make_module (false));
@@ -127,6 +128,7 @@ Lily_lexer::Lily_lexer (Lily_lexer const &src, Lily_parser *parser)
   is_main_input_ = src.is_main_input_;
 
   scopes_ = SCM_EOL;
+  extra_tokens_ = SCM_EOL;
 
   smobify_self ();
 
@@ -388,6 +390,7 @@ Lily_lexer::mark_smob (SCM s)
     scm_gc_mark (lexer->parser_->self_scm ());
   scm_gc_mark (lexer->pitchname_tab_stack_);
   scm_gc_mark (lexer->start_module_);
+  scm_gc_mark (lexer->extra_tokens_);
   return lexer->scopes_;
 }