From: Steve Hancock <perltidy@users.sourceforge.net>
Date: Sun, 18 Jul 2021 14:27:54 +0000 (-0700)
Subject: Fix mis-tokenization before pointer
X-Git-Tag: 20210717.01~4
X-Git-Url: https://git.donarmstrong.com/?a=commitdiff_plain;h=4aa13189c33cf98c9032f8dc50e3754f100478fb;p=perltidy.git

Fix mis-tokenization before pointer
---

diff --git a/dev-bin/run_convergence_tests.pl.data b/dev-bin/run_convergence_tests.pl.data
index c4b29812..1d8cda4c 100644
--- a/dev-bin/run_convergence_tests.pl.data
+++ b/dev-bin/run_convergence_tests.pl.data
@@ -6548,6 +6548,54 @@ ok('[{"1":[5]}]' eq
 --vertical-tightness=2
 --weld-nested-containers
 
+==> b1175.in <==
+# S1
+
+my $json_response
+  = decode_json $t->tx
+  ->res->content
+  ->get_body_chunk;
+
+# S2
+
+my $json_response
+  = decode_json $t->
+  tx->res->content
+  ->get_body_chunk;
+
+==> b1175.par <==
+--break-before-all-operators
+--ignore-old-breakpoints
+--maximum-line-length=22
+
+==> b1176.in <==
+# S1
+                if ( $enter->flags &
+                        OPf_STACKED
+                        and not
+                        null $ary->first
+                        ->sibling
+                        ->sibling )
+                {
+                        ...;
+                }
+
+# S2
+                if ( $enter->flags &
+                        OPf_STACKED
+                        and
+                        not null $ary->
+                        first->sibling
+                        ->sibling )
+                {
+                        ...;
+                }
+
+==> b1176.par <==
+--ignore-old-breakpoints
+--indent-columns=8
+--maximum-line-length=40
+
 ==> b120.in <==
 # Same as bug96
 # State 1
diff --git a/lib/Perl/Tidy/Tokenizer.pm b/lib/Perl/Tidy/Tokenizer.pm
index c012e47f..9ec72e78 100644
--- a/lib/Perl/Tidy/Tokenizer.pm
+++ b/lib/Perl/Tidy/Tokenizer.pm
@@ -2073,7 +2073,12 @@ EOM
                     || $last_nonblank_type eq 'U' )    # possible object
               )
             {
-                $type = 'Z';
+
+                # An identifier followed by '->' is not indirect object;
+                # fixes b1175, b1176
+                my ( $next_nonblank_type, $i_next ) =
+                  find_next_noncomment_type( $i, $rtokens, $max_token_index );
+                $type = 'Z' if ( $next_nonblank_type ne '->' );
             }
         },
         '(' => sub {
@@ -5963,8 +5968,9 @@ sub peek_ahead_for_nonblank_token {
         $line =~ s/^\s*//;                 # trim leading blanks
         next if ( length($line) <= 0 );    # skip blank
         next if ( $line =~ /^#/ );         # skip comment
-        my ( $rtok, $rmap, $rtype ) =
-          pre_tokenize( $line, 2 );        # only need 2 pre-tokens
+
+        # Updated from 2 to 3 to get trigraphs, added for case b1175
+        my ( $rtok, $rmap, $rtype ) = pre_tokenize( $line, 3 );
         my $j = $max_token_index + 1;
 
         foreach my $tok ( @{$rtok} ) {
@@ -7718,6 +7724,42 @@ sub find_next_nonblank_token {
     return ( $next_nonblank_token, $i );
 }
 
+sub find_next_noncomment_type {
+    my ( $i, $rtokens, $max_token_index ) = @_;
+
+    # Given the current character position, look ahead past any comments
+    # and blank lines and return the next token, including digraphs and
+    # trigraphs.
+
+    my ( $next_nonblank_token, $i_next ) =
+      find_next_nonblank_token( $i, $rtokens, $max_token_index );
+
+    # skip past any side comment
+    if ( $next_nonblank_token eq '#' ) {
+        ( $next_nonblank_token, $i_next ) =
+          find_next_nonblank_token( $i_next, $rtokens, $max_token_index );
+    }
+
+    goto RETURN if ( !$next_nonblank_token || $next_nonblank_token eq " " );
+
+    # check for possible a digraph
+    goto RETURN if ( !defined( $rtokens->[ $i_next + 1 ] ) );
+    my $test2 = $next_nonblank_token . $rtokens->[ $i_next + 1 ];
+    goto RETURN if ( !$is_digraph{$test2} );
+    $next_nonblank_token = $test2;
+    $i_next              = $i_next + 1;
+
+    # check for possible a trigraph
+    goto RETURN if ( !defined( $rtokens->[ $i_next + 1 ] ) );
+    my $test3 = $next_nonblank_token . $rtokens->[ $i_next + 1 ];
+    goto RETURN if ( !$is_trigraph{$test3} );
+    $next_nonblank_token = $test3;
+    $i_next              = $i_next + 1;
+
+  RETURN:
+    return ( $next_nonblank_token, $i_next );
+}
+
 sub is_possible_numerator {
 
     # Look at the next non-comment character and decide if it could be a
diff --git a/local-docs/BugLog.pod b/local-docs/BugLog.pod
index 1d5c05ba..ee15a157 100644
--- a/local-docs/BugLog.pod
+++ b/local-docs/BugLog.pod
@@ -2,6 +2,26 @@
 
 =over 4
 
+=item B<Fix mis-tokenization before pointer>
+
+Random testing produced the following case in which formatting was unstable because
+the variable '$t' was being mis-tokenized as a possible indirect object.
+
+    --break-before-all-operators
+    --ignore-old-breakpoints
+    --maximum-line-length=22
+    -sil=0
+
+    my $json_response
+      = decode_json $t
+      ->tx->res->content
+      ->get_body_chunk;
+
+
+This update fixes cases b1175, b1176.
+
+17 Jul 2021.
+
 =item B<Fix to make -wn and -bbxx=n flags work together>
 
 Testing with random parameters produced some cases where the combination of -wn
@@ -57,7 +77,7 @@ which is incorrect because it ignors the -bbsb flag.  The corrected result is
 This update fixes case b1173.  It works for any number of welded containers,
 and the -bbxxi=n flags also work correctly.
 
-16 Jul 2021.
+16 Jul 2021, c71f882.
 
 =item B<Fix problem with side comment after pattern>