From: Steve Hancock <perltidy@users.sourceforge.net>
Date: Wed, 21 Jul 2021 22:26:06 +0000 (-0700)
Subject: revise token sequence numbering scheme
X-Git-Tag: 20210717.02~81
X-Git-Url: https://git.donarmstrong.com/?a=commitdiff_plain;h=83be417f4297e3fd1e0d2fccba399bd87dabd30c;p=perltidy.git

revise token sequence numbering scheme
---

diff --git a/lib/Perl/Tidy/Formatter.pm b/lib/Perl/Tidy/Formatter.pm
index c55445f9..f648a53e 100644
--- a/lib/Perl/Tidy/Formatter.pm
+++ b/lib/Perl/Tidy/Formatter.pm
@@ -4471,15 +4471,121 @@ sub make_closing_side_comment_prefix {
     my $In_format_skipping_section;
     my $Saw_VERSION_in_this_file;
 
+    # Variables used by sub check_sequence_numbers:
+    my $last_seqno;
+    my %saw_opening_seqno;
+    my %saw_closing_seqno;
+    my $initial_seqno;
+
     sub initialize_write_line {
 
         $Last_line_had_side_comment = 0;
         $In_format_skipping_section = 0;
         $Saw_VERSION_in_this_file   = 0;
 
+        $last_seqno        = SEQ_ROOT;
+        %saw_opening_seqno = ();
+        %saw_closing_seqno = ();
+
         return;
     }
 
+    sub check_sequence_numbers {
+
+        # Routine for checking sequence numbers.  This only needs to be
+        # done occasionally in DEVEL_MODE to be sure everything is working
+        # correctly.
+        my ( $rtokens, $rtoken_type, $rtype_sequence, $input_line_no ) = @_;
+        my $jmax = @{$rtokens} - 1;
+        return unless ( $jmax >= 0 );
+        foreach my $j ( 0 .. $jmax ) {
+            my $seqno = $rtype_sequence->[$j];
+            my $token = $rtokens->[$j];
+            my $type  = $rtoken_type->[$j];
+            my $err_msg =
+"Error at j=$j, line number $input_line_no, seqno='$seqno', type='$type', tok='$token':\n";
+
+            if ( !$seqno ) {
+
+           # Sequence numbers are generated for opening tokens, so every opening
+           # token should be sequenced.  Closing tokens will be unsequenced
+           # if they do not have a matching opening toke.
+                if (   $is_opening_sequence_token{$token}
+                    && $type ne 'q'
+                    && $type ne 'Q' )
+                {
+                    Fault(
+                        <<EOM
+$err_msg Unexpected opening token without sequence number
+EOM
+                    );
+                }
+            }
+            else {
+
+                # Save starting seqno to identify sequence method:
+                # New method starts with 2 and has continuous numbering
+                # Old method starts with >2 and may have gaps
+                if ( !defined($initial_seqno) ) { $initial_seqno = $seqno }
+
+                if ( $is_opening_sequence_token{$token} ) {
+
+                    # New method should have continuous numbering
+                    if ( $initial_seqno == 2 && $seqno != $last_seqno + 1 ) {
+                        Fault(
+                            <<EOM
+$err_msg Unexpected opening sequence number: previous seqno=$last_seqno, but seqno= $seqno
+EOM
+                        );
+                    }
+                    $last_seqno = $seqno;
+
+                    # Numbers must be unique
+                    if ( $saw_opening_seqno{$seqno} ) {
+                        my $lno = $saw_opening_seqno{$seqno};
+                        Fault(
+                            <<EOM
+$err_msg Already saw an opening tokens at line $lno with this sequence number
+EOM
+                        );
+                    }
+                    $saw_opening_seqno{$seqno} = $input_line_no;
+                }
+
+                # only one closing item per seqno
+                elsif ( $is_closing_sequence_token{$token} ) {
+                    if ( $saw_closing_seqno{$seqno} ) {
+                        my $lno = $saw_closing_seqno{$seqno};
+                        Fault(
+                            <<EOM
+$err_msg Already saw a closing token with this seqno  at line $lno
+EOM
+                        );
+                    }
+                    $saw_closing_seqno{$seqno} = $input_line_no;
+
+                    # Every closing seqno must have an opening seqno
+                    if ( !$saw_opening_seqno{$seqno} ) {
+                        Fault(
+                            <<EOM
+$err_msg Saw a closing token but no opening token with this seqno
+EOM
+                        );
+                    }
+                }
+
+                # Sequenced items must be opening or closing
+                else {
+                    Fault(
+                        <<EOM
+$err_msg Unexpected token type with a sequence number
+EOM
+                    );
+                }
+            }
+        }
+    }
+
     sub write_line {
 
       # This routine originally received lines of code and immediately processed
@@ -4554,6 +4660,11 @@ sub make_closing_side_comment_prefix {
             my $jmax = @{$rtokens} - 1;
             if ( $jmax >= 0 ) {
                 $Kfirst = defined($Klimit) ? $Klimit + 1 : 0;
+
+                DEVEL_MODE
+                  && check_sequence_numbers( $rtokens, $rtoken_type,
+                    $rtype_sequence, $input_line_no );
+
                 foreach my $j ( 0 .. $jmax ) {
 
                  # Clip negative nesting depths to zero to avoid problems.
diff --git a/lib/Perl/Tidy/Tokenizer.pm b/lib/Perl/Tidy/Tokenizer.pm
index 09c8665f..42915e5c 100644
--- a/lib/Perl/Tidy/Tokenizer.pm
+++ b/lib/Perl/Tidy/Tokenizer.pm
@@ -55,6 +55,7 @@ use vars qw{
   @current_depth
   @total_depth
   $total_depth
+  $next_sequence_number
   @nesting_sequence_number
   @current_sequence_number
   @paren_type
@@ -1344,6 +1345,7 @@ sub prepare_for_a_new_file {
     @total_depth             = ();
     @nesting_sequence_number = ( 0 .. @closing_brace_names - 1 );
     @current_sequence_number = ();
+    $next_sequence_number    = 2;    # The value 1 is reserved for SEQ_ROOT
 
     @paren_type                     = ();
     @paren_semicolon_count          = ();
@@ -1646,7 +1648,7 @@ sub prepare_for_a_new_file {
             @brace_package,                  @square_bracket_type,
             @square_bracket_structural_type, @depth_array,
             @starting_line_of_current_depth, @nested_ternary_flag,
-            @nested_statement_type,
+            @nested_statement_type,          $next_sequence_number,
         );
 
         # save all lexical variables
@@ -5751,8 +5753,18 @@ sub increase_nesting_depth {
     # Sequence numbers increment by number of items.  This keeps
     # a unique set of numbers but still allows the relative location
     # of any type to be determined.
-    $nesting_sequence_number[$aa] += scalar(@closing_brace_names);
-    my $seqno = $nesting_sequence_number[$aa];
+
+    ########################################################################
+    # OLD SEQNO METHOD for incrementing sequence numbers.
+    # Keep this coding awhile for possible testing.
+    ## $nesting_sequence_number[$aa] += scalar(@closing_brace_names);
+    ## my $seqno = $nesting_sequence_number[$aa];
+
+    # NEW SEQNO METHOD, continuous sequence numbers. This allows sequence
+    # numbers to be used as array indexes, and allows them to be compared.
+    my $seqno = $next_sequence_number++;
+    ########################################################################
+
     $current_sequence_number[$aa][ $current_depth[$aa] ] = $seqno;
 
     $starting_line_of_current_depth[$aa][ $current_depth[$aa] ] =