skip ENSG results, and use average weight

[function2gene.git] / bin / get_genecard_results

diff --git a/bin/get_genecard_results b/bin/get_genecard_results
index 634522700283854853b6fd7c8a0485cc1b154bea..1b94f626c1f1bab04f51ffd9dfd68754aa375fd7 100755 (executable)
--- a/bin/get_genecard_results
+++ b/bin/get_genecard_results
@@ -68,8 +68,9 @@ BEGIN{
 }
 
 use IO::File;
-use URI::ParamMunge;
-use LWP::UserAgent;
+use URI;
+use WWW::Mechanize;
+use Time::HiRes qw(usleep);
 
 # XXX parse config file
 
@@ -81,8 +82,8 @@ my %options = (debug    => 0,
               dir      => '.',
               name     => '${search}_results_genecard',
               terms    => '-',
-              genecard_site => 'http://bioinfo.weizmann.ac.il/cards-bin/',
-              genecard_search_url  => 'cardsearch.pl?search_type=keyword%28s%29&search=complement',
+              genecard_site => 'http://www.genecards.org/cgi-bin/',
+              genecard_search_url  => 'cardsearch.pl?search_type=kwd&mini=no&speed=fast&matches=999999',
              );
 
 GetOptions(\%options,'format|f=s','database|b=s','name|n=s',
@@ -106,28 +107,44 @@ else {
      $terms = new IO::File $options{terms}, 'r' or die "Unable to open file $options{terms}: $!";
 }
 
-my $ua = new LWP::UserAgent(agent=>"DA_get_harvester_results/$REVISION");
-
 #For every term
 while (<$terms>) {
      # Get uids to retrieve
      chomp;
      my $search = $_;
-     my $url = uri_param_munge($options{genecard_site}.$options{genecard_search_url},
-                              {search => $search,
-                              },
-                             );
-     my $request = HTTP::Request->new('GET', $url);
-     my $response = $ua->request($request);
-     $response = $response->content;
-     my @result_urls = $response =~ m#<a\s+href=\"(carddisp?[^\"]+)\"\s*>\s+<b>Display</b>\s*<font size=-1>\s*<br>\s*the<b>\s*complete#sg;
-
+     my $uri = URI->new($options{genecard_site}.$options{genecard_search_url});
+     $uri->query_form($uri->query_form(),
+                     search => $search,
+                    );
+     my $url = $uri->as_string;
+     my $mech = WWW::Mechanize->new(agent=>"DA_get_harvester_results/$REVISION");
+     $mech->get($url);
+     my $response = $mech->content();
+     my @result_urls = $response =~ m#<a\s+target\=\'card\'\s+href=\"(carddisp\.pl\?[^\"]+)\"\s*>#sg;
+     @result_urls = grep {$_ !~ /gene=ENSG\d+\&/i} @result_urls;
      my $dir_name = eval qq("$options{name}") or die $@;
-     mkdir("$options{dir}/$dir_name") or die "Unable to make directory $options{dir}/$dir_name $!";
+     if (not -d "$options{dir}/$dir_name") {
+         mkdir("$options{dir}/$dir_name") or die "Unable to make directory $options{dir}/$dir_name $!";
+     }
      # Get XML file
      my @current_urls;
      while (@current_urls = map{$options{genecard_site}.$_} splice(@result_urls,0,30)) {
-         system(q(wget),'-nd','-nH','-w','2','--random-wait','-P',qq($options{dir}/$dir_name),@current_urls) == 0 or warn "$!";
+         for my $url (@current_urls) {
+              # sleep for around 2 seconds
+              usleep((0.5+rand)*2*1000000);
+              $mech->get($url);
+              my $cleaned_url = $url;
+              $cleaned_url =~ s{http://}{}g;
+              $cleaned_url =~ s/[^\w]//g;
+              eval {
+                   $mech->save_content($options{dir}.'/'.$dir_name.'/'.$cleaned_url);
+                   print "retreived $url\n";
+              };
+              if ($@) {
+                   warn $@;
+              }
+         }
+         #system(q(wget),'-nd','-nH','-w','2','--random-wait','-P',qq($options{dir}/$dir_name),@current_urls) == 0 or warn "$!";
      }
 }