handle command failure; fix write_command_to_file

git-svn-id: file:///srv/svn/function2gene/trunk@10 a0738b58-4706-0410-8799-fb830574a030

diff --git a/bin/do_it_all b/bin/do_it_all
index 788319a03226fab077cda0ca67543ac5702340ee..9a681ecd6047aac77cccf84d37e62894b88abd6e 100755 (executable)
--- a/bin/do_it_all
+++ b/bin/do_it_all
@@ -5,6 +5,7 @@
 # Copyright 2007 by Don Armstrong <don@donarmstrong.com>.
 
 
 # Copyright 2007 by Don Armstrong <don@donarmstrong.com>.
 
 

+use threads;

 use warnings;
 use strict;
 
 use warnings;
 use strict;
 


@@ -81,7 +82,8 @@ use vars qw($DEBUG);
 use Cwd qw(abs_path);
 use IO::File;
 use Storable qw(thaw freeze);
 use Cwd qw(abs_path);
 use IO::File;
 use Storable qw(thaw freeze);

-use File::Basename qw(basename);
+use File::Basename qw(basename dirname);
+use Thread::Queue;

 
 my %options = (databases       => [],
               keywords        => [],
 
 my %options = (databases       => [],
               keywords        => [],


@@ -98,7 +100,7 @@ GetOptions(\%options,'keywords=s@','databases=s@',
 pod2usage() if $options{help};
 pod2usage({verbose=>2}) if $options{man};
 
 pod2usage() if $options{help};
 pod2usage({verbose=>2}) if $options{man};
 

-my $base_dir = basename($0);
+my $base_dir = dirname(abs_path($0));

 
 my $ERRORS='';
 
 
 my $ERRORS='';
 


@@ -107,7 +109,7 @@ $ERRORS.="restart-at must be one of get, parse or combine\n" if
 
 $ERRORS.="unknown database(s)" if
      @{$options{databases}} and
 
 $ERRORS.="unknown database(s)" if
      @{$options{databases}} and

-     grep {$_ !~ /^(?:ncbi|genecards|harvester)$/i} @{$options{databases}};
+     grep {$_ !~ /^(?:ncbi|genecard|harvester)$/i} @{$options{databases}};

 
 if (not length $options{results}) {
      $ERRORS.="results directory not specified";
 
 if (not length $options{results}) {
      $ERRORS.="results directory not specified";


@@ -119,7 +121,7 @@ elsif (not -d $options{results} or not -w $options{results}) {
 pod2usage($ERRORS) if length $ERRORS;
 
 if (not @{$options{databases}}) {
 pod2usage($ERRORS) if length $ERRORS;
 
 if (not @{$options{databases}}) {

-     $options{databases} = [qw(ncbi genecards harvester)]
+     $options{databases} = [qw(ncbi genecard harvester)]

 }
 
 $DEBUG = $options{debug};
 }
 
 $DEBUG = $options{debug};


@@ -165,9 +167,9 @@ if (@{$options{keywords}}) {
      for my $keyword_file (@{$options{keywords}}) {
          my $keyword_fh = IO::File->new($keyword_file,'r') or die
               "Unable to open $keyword_file for reading: $!";
      for my $keyword_file (@{$options{keywords}}) {
          my $keyword_fh = IO::File->new($keyword_file,'r') or die
               "Unable to open $keyword_file for reading: $!";

-         local $/;

          while (<$keyword_fh>) {
               next if /^\s*[#;]/;
          while (<$keyword_fh>) {
               next if /^\s*[#;]/;

+              next unless /\w+/;

               chomp;
               if (not $old_keywords{$_}) {
                    DEBUG("Adding new keyword '$_'");
               chomp;
               if (not $old_keywords{$_}) {
                    DEBUG("Adding new keyword '$_'");


@@ -178,6 +180,7 @@ if (@{$options{keywords}}) {
               }
          }
      }
               }
          }
      }

+     push @{$state{keywords}},@new_keywords;

 }
 
 if (exists $options{restart_at} and length $options{restart_at}) {
 }
 
 if (exists $options{restart_at} and length $options{restart_at}) {


@@ -228,22 +231,26 @@ for my $keyword (@{$state{keywords}}) {
      }
 }
 
      }
 }
 

-use threads;
-use Thread::Queue;

 
 for my $state (qw(get parse)) {
      my %databases;
      for my $database (keys %{$actions{$state}}) {
          next unless @{$actions{$state}{$database}};
 
 for my $state (qw(get parse)) {
      my %databases;
      for my $database (keys %{$actions{$state}}) {
          next unless @{$actions{$state}{$database}};

-         $databases{$database}{queue} = Thread::Queue->new;
-         $databases{$database}{thread} = threads->new(\&handle_action($state,$database,$databases{database}{queue}));
+         $databases{$database}{queue} = Thread::Queue->new
+              or die "Unable to create new thread queue";
+         $databases{$database}{thread} = threads->create(\&handle_action,$state,$database,$databases{$database}{queue})
+              or die "Unable to create new thread";

          $databases{$database}{queue}->enqueue(@{$actions{$state}{$database}});
          $databases{$database}{queue}->enqueue(undef);
      }
      my $ERRORS=0;
      for my $database (keys %databases) {
          $databases{$database}{queue}->enqueue(@{$actions{$state}{$database}});
          $databases{$database}{queue}->enqueue(undef);
      }
      my $ERRORS=0;
      for my $database (keys %databases) {

-         my ($actioned_keywords,$failed_keywords) = $databases{$database}{thread}->join;
-         if (@{$failed_keywords}) {
+         my ($actioned_keywords,$failed_keywords) = @{$databases{$database}{thread}->join||[]};
+         if (not defined $failed_keywords) {
+              ADVISE("Something bad happened during '$state' of '$database'");
+              $ERRORS = 1;
+         }
+         elsif (@{$failed_keywords}) {

               ADVISE("These keywords failed during '$state' of '$database':",@{$failed_keywords});
               $ERRORS=1;
          }
               ADVISE("These keywords failed during '$state' of '$database':",@{$failed_keywords});
               $ERRORS=1;
          }


@@ -286,32 +293,41 @@ else {
 sub handle_action{
      my ($state,$database,$queue) = @_;
      my $keyword;
 sub handle_action{
      my ($state,$database,$queue) = @_;
      my $keyword;

-     my $actioned_keywords = ();
-     my $failed_keywords = ();
+     my $actioned_keywords = [];
+     my $failed_keywords = [];
+     DEBUG("Beginning to handle actions for state '$state' database '$database'");

      while ($keyword = $queue->dequeue) {
      while ($keyword = $queue->dequeue) {

+         DEBUG("Handling state '$state' database '$database' keyword '$keyword'");

          # handle the action, baybee
          if ($state eq 'get') {
               my $command_fh;
          # handle the action, baybee
          if ($state eq 'get') {
               my $command_fh;

-              open($command_fh,'|-',
-                   "get_${database}_results",
-                  );
-              print {$command_fh} "$keyword\n";
-              close($command_fh);
-              if ($? != 0) {
-                   WARN("get_${database}_results with keyword $keyword failed with error code ".($?>>8));
+              eval {
+                   open($command_fh,'|-',
+                        "$base_dir/get_${database}_results",
+                       ) or die "unable to execute '$base_dir/get_${database}_results'";
+                   print {$command_fh} "$keyword\n" or die "unable to print $keyword to 'get_${database}_results'";
+                   close($command_fh) or die "Unable to close filehandle";
+                   if ($? != 0) {
+                        die "get_${database}_results with keyword $keyword failed with error code ".($?>>8);
+                   }
+              };
+              if ($@) {
+                   WARN($@);
+                   push @{$failed_keywords}, $keyword;

                    next;
               }
          }
          elsif ($state eq 'parse') {
               eval {
                    write_command_to_file("parsed_results_${database}_${keyword}.txt",
                    next;
               }
          }
          elsif ($state eq 'parse') {
               eval {
                    write_command_to_file("parsed_results_${database}_${keyword}.txt",

-                                         "parse_${database}_results",
+                                         "$base_dir/parse_${database}_results",

                                          '--keywords',
                                          $keyword,
                                         );
               };
               if ($@) {
                    WARN("parse_${database}_results failed with $@");
                                          '--keywords',
                                          $keyword,
                                         );
               };
               if ($@) {
                    WARN("parse_${database}_results failed with $@");

+                   push @{$failed_keywords}, $keyword;

                    next;
               }
          }
                    next;
               }
          }


@@ -321,7 +337,7 @@ sub handle_action{
          ADVISE("$state results from '$database' for '$keyword'");
          push @{$actioned_keywords},$keyword;
      }
          ADVISE("$state results from '$database' for '$keyword'");
          push @{$actioned_keywords},$keyword;
      }

-     return ($actioned_keywords,$failed_keywords);
+     return [$actioned_keywords,$failed_keywords];

 }
 
 sub save_state{
 }
 
 sub save_state{


@@ -333,7 +349,7 @@ sub save_state{
 }
 
 sub write_command_to_file{
 }
 
 sub write_command_to_file{

-     my ($file,@command);
+     my ($file,@command) = @_;

      my $fh = IO::File->new($file,'w') or
          die "Unable to open $file for writing: $!";
      my $command_fh;
      my $fh = IO::File->new($file,'w') or
          die "Unable to open $file for writing: $!";
      my $command_fh;