- # Get XML file
- my @current_urls;
- while (@current_urls = splice(@result_urls,0,30)) {
- system(q(wget),'-nd','-nH','-w','2','--random-wait','-P',qq($options{dir}/$dir_name),@current_urls) == 0 or warn "$!";
- }
+ my $wget_thread = threads->new(\&get_url,"$options{dir}/$dir_name",$queue);
+ push @threads,$wget_thread;
+
+ my $mech = WWW::Mechanize->new(agent => "DA_get_harvester_results/$REVISION");
+
+ #HTTP::Request->new('GET', $url);
+ $mech->get($url);
+ my $next_link;
+ do {
+ my @links = $mech->links;
+ $next_link = undef;
+ for my $link (@links) {
+ if ($link->text() =~ /Next /) {
+ $next_link = $link;
+ }
+ elsif ($link->url =~ m#http://harvester.fzk.de/harvester/human/[^\/]+/[^.]+.htm#) {
+ $queue->enqueue($link->url());
+ }
+ }
+ $mech->follow_link(url=>$next_link->url) if defined $next_link;
+ } while ($next_link);
+ $queue->enqueue(undef);
+}
+for my $thread (@threads) {
+ $thread->join;