3 # get_ensembl_results retreives files of search results from ensembl,
4 # and is released under the terms of the GPL version 2, or any later
5 # version, at your option. See the file README and COPYING for more
8 # Copyright 2008 by Don Armstrong <don@donarmstrong.com>.
20 get_ensembl_results [options]
26 --dir, -D directory to stick results into [default .]
27 --name, -n file naming scheme [default ${search}_results.$format]
28 --terms, -t file of search terms [default -]
29 --debug, -d debugging level [default 0]
30 --help, -h display this help
31 --man, -m display manual
39 Debug verbosity. (Default 0)
43 Display brief useage information.
53 get_ensembl_results -D ./ensembl_results/ -n '${search}_name.html' < search_parameters
55 Will pretty much do what you want
61 use vars qw($DEBUG $REVISION);
64 ($REVISION) = q$LastChangedRevision: 1$ =~ /LastChangedRevision:\s+([^\s]+)/;
65 $DEBUG = 0 unless defined $DEBUG;
71 use Time::HiRes qw(usleep);
73 # XXX parse config file
75 my %options = (debug => 0,
79 name => '${search}_results_ensembl',
81 ensembl_site => 'http://www.ensembl.org',
82 ensembl_search_url => '/Homo_sapiens/searchview?species=Homo_sapiens&idx=&',
85 GetOptions(\%options,'name|n=s',
86 'terms|t=s','dir|D=s','debug|d+','help|h|?','man|m');
88 pod2usage() if $options{help};
89 pod2usage({verbose=>2}) if $options{man};
91 $DEBUG = $options{debug};
93 if (not -d $options{dir}) {
94 die "$options{dir} does not exist or is not a directory";
97 #open search terms file
99 if ($options{terms} eq '-') {
103 $terms = new IO::File $options{terms}, 'r' or die "Unable to open file $options{terms}: $!";
108 # Get uids to retrieve
112 my $dir_name = eval qq("$options{name}") or die $@;
113 if (not -d "$options{dir}/$dir_name") {
114 mkdir("$options{dir}/$dir_name") or die "Unable to make directory $options{dir}/$dir_name $!";
116 my $uri = URI->new($options{ensembl_site}.$options{ensembl_search_url});
117 $uri->query_form($uri->query_form(),
120 my $url = $uri->as_string;
121 my $mech = WWW::Mechanize->new(agent=>"DA_get_ensembl_results/$REVISION");
122 #print STDERR $url,qq(\n);
124 #print STDERR $mech->content();
125 my @links = $mech->find_link(text_regex=>qr/Gene/);
126 $mech->follow_link(text_regex=>qr/Gene/);
127 #print STDERR $mech->content();
128 # now we need to walk through all of the pages
129 push @links,$mech->find_all_links(url_regex => qr/_s=\d{2,}$/);
130 for my $link (@links) {
131 my $link_url = $link->url_abs->as_string;
132 print STDERR "getting $link_url\n";
133 $mech->get($link_url);
134 my @gene_links = $mech->find_all_links(url_regex => qr/geneview/);
135 for my $gene_link (@gene_links) {
136 my $gene_url = $gene_link->url_abs->as_string;
137 print STDERR "saving $gene_url\n";
138 $mech->get($gene_url);
139 my $cleaned_url = $gene_url;
140 $cleaned_url =~ s{http://}{}g;
141 $cleaned_url =~ s/[^\w]//g;
143 $mech->save_content($options{dir}.'/'.$dir_name.'/'.$cleaned_url);
144 print "retreived $url\n";