3 # get_harvester_results retreives files of search results from ncbi,
4 # and is released under the terms of the GPL version 2, or any later
5 # version, at your option. See the file README and COPYING for more
8 # Copyright 2004 by Don Armstrong <don@donarmstrong.com>.
10 # $Id: ss,v 1.1 2004/06/29 05:26:35 don Exp $
23 get_harvester_results [options]
29 --dir, -D directory to stick results into [default .]
30 --name, -n file naming scheme [default ${search}_results.$format]
31 --terms, -t file of search terms [default -]
32 --debug, -d debugging level [default 0]
33 --help, -h display this help
34 --man, -m display manual
42 Debug verbosity. (Default 0)
46 Display brief useage information.
56 get_harvester_results -D ./harvester_results/ -n '${search}_name.html' < search_parameters
58 Will pretty much do what you want
64 use vars qw($DEBUG $REVISION);
67 ($REVISION) = q$LastChangedRevision: 1$ =~ /LastChangedRevision:\s+([^\s]+)/;
68 $DEBUG = 0 unless defined $DEBUG;
76 # XXX parse config file
78 my %options = (debug => 0,
84 name => '${search}_results_harvester',
87 harvester_site => 'http://harvester.fzk.de',
90 GetOptions(\%options,'format|f=s','database|b=s','name|n=s',
91 'terms|t=s','dir|D=s','debug|d+','help|h|?','man|m');
93 pod2usage() if $options{help};
94 pod2usage({verbose=>2}) if $options{man};
96 $DEBUG = $options{debug};
98 if (not -d $options{dir}) {
99 die "$options{dir} does not exist or is not a directory";
103 $options{harvester_search_url} = '/cgi-bin/'.$options{orgn}.'/search.cgi?zoom_query=golgi&zoom_per_page=100&zoom_and=1&zoom_sort=0';
105 #open search terms file
107 if ($options{terms} eq '-') {
111 $terms = new IO::File $options{terms}, 'r' or die "Unable to open file $options{terms}: $!";
117 # Get uids to retrieve
120 my $uri = URI->new($options{harvester_site}.$options{harvester_search_url});
121 $uri->query_form(zoom_query =>[],
123 $uri->query_form(zoom_query => $search,
125 my $url = $uri->as_string;
126 my $queue = Thread::Queue->new();
127 my $dir_name = eval qq("$options{name}") or die $@;
128 if (not -d "$options{dir}/$dir_name") {
129 mkdir("$options{dir}/$dir_name") or die "Unable to make directory $options{dir}/$dir_name $!";
131 my $wget_thread = threads->new(\&get_url,"$options{dir}/$dir_name",$queue);
132 push @threads,$wget_thread;
134 my $mech = WWW::Mechanize->new(agent => "DA_get_harvester_results/$REVISION");
136 #HTTP::Request->new('GET', $url);
140 my @links = $mech->links;
142 for my $link (@links) {
143 if ($link->text() =~ /Next /) {
146 elsif ($link->url =~ m#http://harvester.fzk.de/harvester/human/[^\/]+/[^.]+.htm#) {
147 $queue->enqueue($link->url());
150 $mech->follow_link(url=>$next_link->url) if defined $next_link;
151 } while ($next_link);
152 $queue->enqueue(undef);
154 for my $thread (@threads) {
159 my ($dir,$queue) = @_;
163 while (my $url = $queue->dequeue) {
164 push @current_urls,$url;
165 if (@current_urls >= 30) {
166 wget_urls($dir,@current_urls);
170 wget_urls($dir,@current_urls) if @current_urls;
173 my ($dir,@urls) = @_;
175 system(q(wget),'-nd','-nH','-w','2','--random-wait','-P',$dir,@urls) == 0 or warn "$!";