fix issue c109, 's s(..)(.)sss'

diff --git a/dev-bin/run_tokenizer_tests.pl.data b/dev-bin/run_tokenizer_tests.pl.data
index db0a24fc64e47685d7b243aa520ecfefa6e169e0..2e1779cf31523d61417aa50ccd23b2b99f80086c 100644 (file)
--- a/dev-bin/run_tokenizer_tests.pl.data
+++ b/dev-bin/run_tokenizer_tests.pl.data
@@ -149,3 +149,8 @@ print $fh <6>;         # error
 
 ==> c106.in <==
 my$seed=$$^$^T;
+
+==> c109.in <==
+s s(..)(.)sss
+;
+

diff --git a/lib/Perl/Tidy/Tokenizer.pm b/lib/Perl/Tidy/Tokenizer.pm
index 54c7733c1eb6c29429a0798e2050ff6550b2af71..73739d0c47ef7acacf9abe593410cf72acf11180 100644 (file)
--- a/lib/Perl/Tidy/Tokenizer.pm
+++ b/lib/Perl/Tidy/Tokenizer.pm
@@ -8821,7 +8821,8 @@ sub follow_quoted_string {
                 "Note: alphanumeric quote delimiter ($beginning_tok) \n");
         }
 
-        while ( $i < $max_token_index ) {
+        # Note: changed < to <= here to fix c109. Relying on extra end blanks.
+        while ( $i <= $max_token_index ) {
 
             if ( $quote_pos == 0 || ( $i < 0 ) ) {
                 $tok = $rtokens->[ ++$i ];
@@ -8849,6 +8850,11 @@ sub follow_quoted_string {
                 $quoted_string .=
                   substr( $tok, $old_pos, $quote_pos - $old_pos - 1 );
 
+                # NOTE: any quote modifiers will be at the end of '$tok'. If we
+                # wanted to check them, this is the place to get them.  But
+                # this quote form is rarely used in practice, so it isn't
+                # worthwhile.
+
                 $quote_depth--;
 
                 if ( $quote_depth == 0 ) {

diff --git a/local-docs/BugLog.pod b/local-docs/BugLog.pod
index 790e995034a2cad82f1b40372ba2865e1beb16e9..c4539402d34d39fb71d19832c6422c2243cbfed7 100644 (file)
--- a/local-docs/BugLog.pod
+++ b/local-docs/BugLog.pod
@@ -2,6 +2,23 @@
 
 =over 4
 
+=item B<Fix tokenization issue c109>
+
+Automated random testing produced an error tokenizing the following code
+fragment:
+
+    s s(..)(.)sss
+    ;
+
+This is equivalent to 's/(..)(.)//s' with 's' as the delimiter instead of '/'.
+It was tokenized correctly except when the final 's' was followed by a newline,
+as in the example.  When the delimiter is a letter rather than a punctuation
+character, perltidy exercises some seldom-used code which had an off-by-one
+loop limit.  This has been fixed.
+
+12 Nov 2021.
+
+
 =item B<Fix tokenization of $$^, issue c106>
 
 Automated random testing produced an error tokenizing the following fragment: