Fix mis-tokenization before pointer

diff --git a/dev-bin/run_convergence_tests.pl.data b/dev-bin/run_convergence_tests.pl.data
index c4b29812a0724929b9c517fc03d4f8122d3cd794..1d8cda4c646d82f9ec7249ee268e5bf7f1f8fca0 100644 (file)
--- a/dev-bin/run_convergence_tests.pl.data
+++ b/dev-bin/run_convergence_tests.pl.data
@@ -6548,6 +6548,54 @@ ok('[{"1":[5]}]' eq
 --vertical-tightness=2
 --weld-nested-containers
 
+==> b1175.in <==
+# S1
+
+my $json_response
+  = decode_json $t->tx
+  ->res->content
+  ->get_body_chunk;
+
+# S2
+
+my $json_response
+  = decode_json $t->
+  tx->res->content
+  ->get_body_chunk;
+
+==> b1175.par <==
+--break-before-all-operators
+--ignore-old-breakpoints
+--maximum-line-length=22
+
+==> b1176.in <==
+# S1
+                if ( $enter->flags &
+                        OPf_STACKED
+                        and not
+                        null $ary->first
+                        ->sibling
+                        ->sibling )
+                {
+                        ...;
+                }
+
+# S2
+                if ( $enter->flags &
+                        OPf_STACKED
+                        and
+                        not null $ary->
+                        first->sibling
+                        ->sibling )
+                {
+                        ...;
+                }
+
+==> b1176.par <==
+--ignore-old-breakpoints
+--indent-columns=8
+--maximum-line-length=40
+
 ==> b120.in <==
 # Same as bug96
 # State 1

diff --git a/lib/Perl/Tidy/Tokenizer.pm b/lib/Perl/Tidy/Tokenizer.pm
index c012e47f9d89b0a0c068a375f21c8ece343ced89..9ec72e78c4eb5d5cc5742f0217722af3659811d8 100644 (file)
--- a/lib/Perl/Tidy/Tokenizer.pm
+++ b/lib/Perl/Tidy/Tokenizer.pm
@@ -2073,7 +2073,12 @@ EOM
                     || $last_nonblank_type eq 'U' )    # possible object
               )
             {
-                $type = 'Z';
+
+                # An identifier followed by '->' is not indirect object;
+                # fixes b1175, b1176
+                my ( $next_nonblank_type, $i_next ) =
+                  find_next_noncomment_type( $i, $rtokens, $max_token_index );
+                $type = 'Z' if ( $next_nonblank_type ne '->' );
             }
         },
         '(' => sub {
@@ -5963,8 +5968,9 @@ sub peek_ahead_for_nonblank_token {
         $line =~ s/^\s*//;                 # trim leading blanks
         next if ( length($line) <= 0 );    # skip blank
         next if ( $line =~ /^#/ );         # skip comment
-        my ( $rtok, $rmap, $rtype ) =
-          pre_tokenize( $line, 2 );        # only need 2 pre-tokens
+
+        # Updated from 2 to 3 to get trigraphs, added for case b1175
+        my ( $rtok, $rmap, $rtype ) = pre_tokenize( $line, 3 );
         my $j = $max_token_index + 1;
 
         foreach my $tok ( @{$rtok} ) {
@@ -7718,6 +7724,42 @@ sub find_next_nonblank_token {
     return ( $next_nonblank_token, $i );
 }
 
+sub find_next_noncomment_type {
+    my ( $i, $rtokens, $max_token_index ) = @_;
+
+    # Given the current character position, look ahead past any comments
+    # and blank lines and return the next token, including digraphs and
+    # trigraphs.
+
+    my ( $next_nonblank_token, $i_next ) =
+      find_next_nonblank_token( $i, $rtokens, $max_token_index );
+
+    # skip past any side comment
+    if ( $next_nonblank_token eq '#' ) {
+        ( $next_nonblank_token, $i_next ) =
+          find_next_nonblank_token( $i_next, $rtokens, $max_token_index );
+    }
+
+    goto RETURN if ( !$next_nonblank_token || $next_nonblank_token eq " " );
+
+    # check for possible a digraph
+    goto RETURN if ( !defined( $rtokens->[ $i_next + 1 ] ) );
+    my $test2 = $next_nonblank_token . $rtokens->[ $i_next + 1 ];
+    goto RETURN if ( !$is_digraph{$test2} );
+    $next_nonblank_token = $test2;
+    $i_next              = $i_next + 1;
+
+    # check for possible a trigraph
+    goto RETURN if ( !defined( $rtokens->[ $i_next + 1 ] ) );
+    my $test3 = $next_nonblank_token . $rtokens->[ $i_next + 1 ];
+    goto RETURN if ( !$is_trigraph{$test3} );
+    $next_nonblank_token = $test3;
+    $i_next              = $i_next + 1;
+
+  RETURN:
+    return ( $next_nonblank_token, $i_next );
+}
+
 sub is_possible_numerator {
 
     # Look at the next non-comment character and decide if it could be a

diff --git a/local-docs/BugLog.pod b/local-docs/BugLog.pod
index 1d5c05ba2c3e89f52a9ffa4ac26f02ab61e0c1ab..ee15a1578c59bd8df9a424adf19ff5932e3f1529 100644 (file)
--- a/local-docs/BugLog.pod
+++ b/local-docs/BugLog.pod
@@ -2,6 +2,26 @@
 
 =over 4
 
+=item B<Fix mis-tokenization before pointer>
+
+Random testing produced the following case in which formatting was unstable because
+the variable '$t' was being mis-tokenized as a possible indirect object.
+
+    --break-before-all-operators
+    --ignore-old-breakpoints
+    --maximum-line-length=22
+    -sil=0
+
+    my $json_response
+      = decode_json $t
+      ->tx->res->content
+      ->get_body_chunk;
+
+
+This update fixes cases b1175, b1176.
+
+17 Jul 2021.
+
 =item B<Fix to make -wn and -bbxx=n flags work together>
 
 Testing with random parameters produced some cases where the combination of -wn
@@ -57,7 +77,7 @@ which is incorrect because it ignors the -bbsb flag.  The corrected result is
 This update fixes case b1173.  It works for any number of welded containers,
 and the -bbxxi=n flags also work correctly.
 
-16 Jul 2021.
+16 Jul 2021, c71f882.
 
 =item B<Fix problem with side comment after pattern>