update comments

diff --git a/lib/Perl/Tidy.pm b/lib/Perl/Tidy.pm
index 1f18d133f9a9bf89f8f0f93502a1146ec83fa845..a16050b0bdacf703ef9688040659867c3ef73449 100644 (file)
--- a/lib/Perl/Tidy.pm
+++ b/lib/Perl/Tidy.pm
@@ -5190,8 +5190,8 @@ sub read_config_file {
         #     name { body }   or  name {   or    name { body
         # See rules in perltidy's perldoc page
         # Section: Other Controls - Creating a new abbreviation
-        if ( $line =~ /^((\w+)\s*\{)(.*)?$/ ) {
-            ( $name, $body ) = ( $2, $3 );
+        if ( $line =~ /^(?: (\w+) \s* \{ ) (.*)? $/x ) {
+            ( $name, $body ) = ( $1, $2 );
 
             # Cannot start new abbreviation unless old abbreviation is complete
             last if ($opening_brace_line);

diff --git a/lib/Perl/Tidy/Formatter.pm b/lib/Perl/Tidy/Formatter.pm
index db943a85bccdec8cc6f3978a8d381b7fc2e61b8d..7f2b1e80306d2a0db0eab297c9e2f1beb644c1d9 100644 (file)
--- a/lib/Perl/Tidy/Formatter.pm
+++ b/lib/Perl/Tidy/Formatter.pm
@@ -9058,7 +9058,7 @@ sub respace_tokens_inner_loop {
                     }
                 }
                 else {
-                    # it is rare to arrive here (identifier with spaces)
+                    # Could be something like '* STDERR' or '$ debug'
                 }
             }
         }
@@ -9149,7 +9149,7 @@ EOM
             # safety (the tokenizer should have done this).
             # To avoid trimming qw quotes use -ntqw; this causes the
             # tokenizer to set them as type 'Q' instead of 'q'.
-            $token =~ s/^ \s+ | \s+ $//x;
+            $token =~ s/^ \s+ | \s+ $//gx;
             $rtoken_vars->[_TOKEN_] = $token;
             if ( $self->[_save_logfile_] && $token =~ /\t/ ) {
                 $self->note_embedded_tab($input_line_number);
@@ -10560,22 +10560,18 @@ sub copy_token_as_type {
     # slightly modifying an existing token.
     my ( $rold_token, $type, $token ) = @_;
     if ( !defined($token) ) {
-        if ( $type eq 'b' ) {
-            $token = SPACE;
-        }
-        elsif ( $type eq 'q' ) {
-            $token = EMPTY_STRING;
-        }
-        elsif ( $type eq '->' ) {
-            $token = '->';
-        }
-        elsif ( $type eq ';' ) {
-            $token = ';';
-        }
-        elsif ( $type eq ',' ) {
-            $token = ',';
-        }
-        else {
+
+        $token =
+            $type eq 'b'  ? SPACE
+          : $type eq 'q'  ? EMPTY_STRING
+          : $type eq '->' ? $type
+          : $type eq ';'  ? $type
+          : $type eq ','  ? $type
+          :                 undef;
+
+        if ( !defined($token) ) {
+
+            $token = $type;
 
             # Unexpected type ... this sub will work as long as both $token and
             # $type are defined, but we should catch any unexpected types during
@@ -10586,8 +10582,6 @@ sub 'copy_token_as_type' received token type '$type' but expects just one of: 'b
 EOM
             }
 
-            # Shouldn't get here
-            $token = $type;
         }
     }
 
@@ -14806,7 +14800,7 @@ EOM
                     }
                 }
 
-                # it is a ternary - no special processing for these yet
+                # it is a ternary or input file is unbalanced
                 else {
 
                 }

diff --git a/lib/Perl/Tidy/Tokenizer.pm b/lib/Perl/Tidy/Tokenizer.pm
index dee0404742d62a2a444ff49f6437fffcad16063a..a27b8abc6083d29b1f90ac66ee07c1fc4ff33aa7 100644 (file)
--- a/lib/Perl/Tidy/Tokenizer.pm
+++ b/lib/Perl/Tidy/Tokenizer.pm
@@ -1,17 +1,28 @@
 #####################################################################
 #
-# The Perl::Tidy::Tokenizer package is essentially a filter which
-# reads lines of perl source code from a source object and provides
-# corresponding tokenized lines through its get_line() method.  Lines
-# flow from the source_object to the caller like this:
+# Perl::Tidy::Tokenizer reads a source and breaks it into a stream of tokens
 #
-# source_object --> Tokenizer -->  calling routine
-#   get_line()      get_line()     line_of_tokens
+# Usage:
+#
+#   STEP 1: initialize or re-initialze Tokenizer with user options
+#     Perl::Tidy::Tokenizer::check_options($rOpts);
+#
+#   STEP 2: create a tokenizer for a specific input source object
+#     my $tokenizer = Perl::Tidy::Tokenizer->new(
+#         source_object      => $source,
+#         ...
+#     );
+#
+#   STEP 3: get and process each tokenized 'line' (a hash ref of token info)
+#    while ( my $line = $tokenizer->get_line() ) {
+#        $formatter->write_line($line);
+#    }
+#
+#   STEP 4: report errors
+#    my $severe_error = $tokenizer->report_tokenization_errors();
 #
 # The source object can be a STRING ref, an ARRAY ref, or an object with a
 # get_line() method which supplies one line (a character string) perl call.
-# The Tokenizer returns a reference to a data structure 'line_of_tokens'
-# containing one tokenized line for each call to its get_line() method.
 #
 # NOTE: This is not a real class.  Only one tokenizer my be used.
 #
@@ -631,6 +642,8 @@ EOM
 
     # handle an object - must have a get_line method
     else {
+
+        # This will die if user's object does have a 'get_line' method
         while ( my $line = $line_source_object->get_line() ) {
             push( @{$rinput_lines}, $line );
         }
@@ -1157,7 +1170,7 @@ sub get_line {
         # check for error of extra whitespace
         # note for PERL6: leading whitespace is allowed
         else {
-            $candidate_target =~ s/^ \s+ | \s+ $//x;
+            $candidate_target =~ s/^ \s+ | \s+ $//gx;    # trim both ends
             if ( $candidate_target eq $here_doc_target ) {
                 $self->[_nearly_matched_here_target_at_] = $input_line_number;
             }