update everything again

git-svn-id: file:///srv/svn/function2gene/trunk@11 a0738b58-4706-0410-8799-fb830574a030

diff --git a/bin/get_genecard_results b/bin/get_genecard_results
index 634522700283854853b6fd7c8a0485cc1b154bea..051e39da7e7d54dfcdda7cac17abcb8df8a3b1ba 100755 (executable)
--- a/bin/get_genecard_results
+++ b/bin/get_genecard_results
@@ -68,8 +68,8 @@ BEGIN{
 }
 
 use IO::File;
 }
 
 use IO::File;

-use URI::ParamMunge;
-use LWP::UserAgent;
+use URI;
+use WWW::Mechanize;

 
 # XXX parse config file
 
 
 # XXX parse config file
 


@@ -82,7 +82,7 @@ my %options = (debug    => 0,
               name     => '${search}_results_genecard',
               terms    => '-',
               genecard_site => 'http://bioinfo.weizmann.ac.il/cards-bin/',
               name     => '${search}_results_genecard',
               terms    => '-',
               genecard_site => 'http://bioinfo.weizmann.ac.il/cards-bin/',

-              genecard_search_url  => 'cardsearch.pl?search_type=keyword%28s%29&search=complement',
+              genecard_search_url  => 'cardsearch.pl?search_type=kwd&mini=no&speed=fast&matches=999999',

              );
 
 GetOptions(\%options,'format|f=s','database|b=s','name|n=s',
              );
 
 GetOptions(\%options,'format|f=s','database|b=s','name|n=s',


@@ -106,24 +106,24 @@ else {
      $terms = new IO::File $options{terms}, 'r' or die "Unable to open file $options{terms}: $!";
 }
 
      $terms = new IO::File $options{terms}, 'r' or die "Unable to open file $options{terms}: $!";
 }
 

-my $ua = new LWP::UserAgent(agent=>"DA_get_harvester_results/$REVISION");
-

 #For every term
 while (<$terms>) {
      # Get uids to retrieve
      chomp;
      my $search = $_;
 #For every term
 while (<$terms>) {
      # Get uids to retrieve
      chomp;
      my $search = $_;

-     my $url = uri_param_munge($options{genecard_site}.$options{genecard_search_url},
-                              {search => $search,
-                              },
-                             );
-     my $request = HTTP::Request->new('GET', $url);
-     my $response = $ua->request($request);
-     $response = $response->content;
-     my @result_urls = $response =~ m#<a\s+href=\"(carddisp?[^\"]+)\"\s*>\s+<b>Display</b>\s*<font size=-1>\s*<br>\s*the<b>\s*complete#sg;
-
+     my $uri = URI->new($options{genecard_site}.$options{genecard_search_url});
+     $uri->query_form($uri->query_form(),
+                     search => $search,
+                    );
+     my $url = $uri->as_string;
+     my $mech = WWW::Mechanize->new(agent=>"DA_get_harvester_results/$REVISION");
+     $mech->get($url);
+     my $response = $mech->content();
+     my @result_urls = $response =~ m#<a\s+target\=\'card\'\s+href=\"(carddisp\.pl\?[^\"]+)\"\s*>#sg;

      my $dir_name = eval qq("$options{name}") or die $@;
      my $dir_name = eval qq("$options{name}") or die $@;

-     mkdir("$options{dir}/$dir_name") or die "Unable to make directory $options{dir}/$dir_name $!";
+     if (not -d "$options{dir}/$dir_name") {
+         mkdir("$options{dir}/$dir_name") or die "Unable to make directory $options{dir}/$dir_name $!";
+     }

      # Get XML file
      my @current_urls;
      while (@current_urls = map{$options{genecard_site}.$_} splice(@result_urls,0,30)) {
      # Get XML file
      my @current_urls;
      while (@current_urls = map{$options{genecard_site}.$_} splice(@result_urls,0,30)) {

diff --git a/bin/get_harvester_results b/bin/get_harvester_results
index 97734e9348963e9ede2a8e45a00a91d74b2a3d7b..7446e088c245d9d58adcafef59dd04f769edb6a8 100755 (executable)
--- a/bin/get_harvester_results
+++ b/bin/get_harvester_results
@@ -125,7 +125,9 @@ while (<$terms>) {
      my $url = $uri->as_string;
      my $queue = Thread::Queue->new();
      my $dir_name = eval qq("$options{name}") or die $@;
      my $url = $uri->as_string;
      my $queue = Thread::Queue->new();
      my $dir_name = eval qq("$options{name}") or die $@;

-     mkdir("$options{dir}/$dir_name") or die "Unable to make directory $options{dir}/$dir_name $!";
+     if (not -d "$options{dir}/$dir_name") {
+       mkdir("$options{dir}/$dir_name") or die "Unable to make directory $options{dir}/$dir_name $!";
+     }

      my $wget_thread = threads->new(\&get_url,"$options{dir}/$dir_name",$queue);
      push @threads,$wget_thread;
 
      my $wget_thread = threads->new(\&get_url,"$options{dir}/$dir_name",$queue);
      push @threads,$wget_thread;
 

diff --git a/bin/parse_genecard_results b/bin/parse_genecard_results
index 6024d9bd27e0f9d74618070735532cb6dd051da1..a6718c5434030239316d5e425ff666d7699188f5 100755 (executable)
--- a/bin/parse_genecard_results
+++ b/bin/parse_genecard_results
@@ -104,7 +104,7 @@ if ($options{keywords}) {
      if (@ARGV != 1) {
          pod2usage("If the --keywords option is used, exactly one argument (the keyword) must be passed");
      }
      if (@ARGV != 1) {
          pod2usage("If the --keywords option is used, exactly one argument (the keyword) must be passed");
      }

-     $option{dir} = "$ARGV[0]_results_genecard";
+     $options{dir} = "$ARGV[0]_results_genecard";

 }
 
 if (not -d $options{dir}) {
 }
 
 if (not -d $options{dir}) {

diff --git a/bin/parse_harvester_results b/bin/parse_harvester_results
index a35c9b902f5a9ff840d6bb3450862ae0989cd9e1..7a8cfd4a3ef37b40e7d63c36424d2e34c81ae59d 100755 (executable)
--- a/bin/parse_harvester_results
+++ b/bin/parse_harvester_results
@@ -80,7 +80,9 @@ my %options = (debug    => 0,
               keywords => 0,
              );
 
               keywords => 0,
              );
 

-GetOptions(\%options,'keyword|k=s','dir|D=s','debug|d+','help|h|?','man|m');
+GetOptions(\%options,'keyword|k=s','dir|D=s','debug|d+','help|h|?','man|m',
+          'keywords',
+         );

 
 
 pod2usage() if $options{help};
 
 
 pod2usage() if $options{help};


@@ -104,7 +106,7 @@ if ($options{keywords}) {
      if (@ARGV != 1) {
          pod2usage("If the --keywords option is used, exactly one argument (the keyword) must be passed");
      }
      if (@ARGV != 1) {
          pod2usage("If the --keywords option is used, exactly one argument (the keyword) must be passed");
      }

-     $option{dir} = "$ARGV[0]_results_harvester";
+     $options{dir} = "$ARGV[0]_results_harvester";

 }
 
 
 }