From 1e24111af3bd3cb2b5795a64756c8b2fb05be8b5 Mon Sep 17 00:00:00 2001 From: Don Armstrong Date: Mon, 25 Feb 2008 23:18:43 +0000 Subject: [PATCH] add get and parse uniprot results git-svn-id: file:///srv/svn/function2gene/trunk@33 a0738b58-4706-0410-8799-fb830574a030 --- bin/get_uniprot_results | 147 ++++++++++++++++++++++++++ bin/parse_uniprot_results | 214 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 361 insertions(+) create mode 100755 bin/get_uniprot_results create mode 100755 bin/parse_uniprot_results diff --git a/bin/get_uniprot_results b/bin/get_uniprot_results new file mode 100755 index 0000000..680e850 --- /dev/null +++ b/bin/get_uniprot_results @@ -0,0 +1,147 @@ +#! /usr/bin/perl + +# get_uniprot_results retreives files of search results from uniprot, +# and is released under the terms of the GPL version 2, or any later +# version, at your option. See the file README and COPYING for more +# information. + +# Copyright 2008 by Don Armstrong . + + +use warnings; +use strict; + + +use Getopt::Long; +use Pod::Usage; + +=head1 NAME + + get_uniprot_results [options] + +=head1 SYNOPSIS + + + Options: + --dir, -D directory to stick results into [default .] + --name, -n file naming scheme [default ${search}_results.$format] + --terms, -t file of search terms [default -] + --debug, -d debugging level [default 0] + --help, -h display this help + --man, -m display manual + +=head1 OPTIONS + +=over + +=item B<--debug, -d> + +Debug verbosity. (Default 0) + +=item B<--help, -h> + +Display brief useage information. + +=item B<--man, -m> + +Display this manual. + +=back + +=head1 EXAMPLES + + get_uniprot_results -D ./uniprot_results/ -n '${search}_name.html' < search_parameters + +Will pretty much do what you want + +=cut + + + +use vars qw($DEBUG $REVISION); + +BEGIN{ + ($REVISION) = q$LastChangedRevision: 1$ =~ /LastChangedRevision:\s+([^\s]+)/; + $DEBUG = 0 unless defined $DEBUG; +} + +use IO::File; +use URI; +use WWW::Mechanize; +use Time::HiRes qw(usleep); + +# XXX parse config file + +my %options = (debug => 0, + help => 0, + man => 0, + dir => '.', + name => '${search}_results_uniprot', + terms => '-', + uniprot_site => 'http://ca.expasy.org', + uniprot_search_url => '/cgi-bin/sprot-search-ful?S=on&T=on', + ); + +GetOptions(\%options,'name|n=s', + 'terms|t=s','dir|D=s','debug|d+','help|h|?','man|m'); + +pod2usage() if $options{help}; +pod2usage({verbose=>2}) if $options{man}; + +$DEBUG = $options{debug}; + +if (not -d $options{dir}) { + die "$options{dir} does not exist or is not a directory"; +} + +#open search terms file +my $terms; +if ($options{terms} eq '-') { + $terms = \*STDIN; +} +else { + $terms = new IO::File $options{terms}, 'r' or die "Unable to open file $options{terms}: $!"; +} + +#For every term +while (<$terms>) { + # Get uids to retrieve + chomp; + s/\r$//g; + my $search = $_; + my $dir_name = eval qq("$options{name}") or die $@; + if (not -d "$options{dir}/$dir_name") { + mkdir("$options{dir}/$dir_name") or die "Unable to make directory $options{dir}/$dir_name $!"; + } + my $uri = URI->new($options{uniprot_site}.$options{uniprot_search_url}); + $uri->query_form($uri->query_form(), + SEARCH => $search, + ); + my $url = $uri->as_string; + my $mech = WWW::Mechanize->new(agent=>"DA_get_uniprot_results/$REVISION"); + print "retreived $url\n"; + $mech->get($url); + my @links = $mech->find_all_links(text_regex=>qr/_HUMAN/); + for my $gene_link (@links) { + my $gene_url = $gene_link->url_abs->as_string; + print STDERR "saving $gene_url\n"; + $mech->get($gene_url); + $mech->follow_link(text_regex => qr/View\s*entry\s*in\s*raw\s*text\s*format/); + my $cleaned_url = $gene_url; + $cleaned_url =~ s{http://}{}g; + $cleaned_url =~ s/[^\w]//g; + eval { + $mech->save_content($options{dir}.'/'.$dir_name.'/'.$cleaned_url); + }; + if ($@) { + warn $@; + } + } +} + + + + + + +__END__ diff --git a/bin/parse_uniprot_results b/bin/parse_uniprot_results new file mode 100755 index 0000000..6b90eb7 --- /dev/null +++ b/bin/parse_uniprot_results @@ -0,0 +1,214 @@ +#! /usr/bin/perl + +# parse_uniprot_results retreives files of search results from ncbi, +# and is released under the terms of the GPL version 2, or any later +# version, at your option. See the file README and COPYING for more +# information. + +# Copyright 2004 by Don Armstrong . + +# $Id: ss,v 1.1 2004/06/29 05:26:35 don Exp $ + + +use warnings; +use strict; + + +use Getopt::Long; +use Pod::Usage; + +=head1 NAME + + parse_uniprot_results [options] + +=head1 SYNOPSIS + + + Options: + --dir, -D directory to stick results into [default .] + --name, -n file naming scheme [default ${search}_results.$format] + --terms, -t file of search terms [default -] + --debug, -d debugging level [default 0] + --help, -h display this help + --man, -m display manual + +=head1 OPTIONS + +=over + +=item B<--debug, -d> + +Debug verbosity. (Default 0) + +=item B<--help, -h> + +Display brief useage information. + +=item B<--man, -m> + +Display this manual. + +=back + +=head1 EXAMPLES + + parse_uniprot_results -D ./uniprot_results/ -n '${search}_name.html' < search_parameters + +Will pretty much do what you want + +=cut + + + +use vars qw($DEBUG $REVISION); + +BEGIN{ + ($REVISION) = q$LastChangedRevision: 1$ =~ /LastChangedRevision:\s+([^\s]+)/; + $DEBUG = 0 unless defined $DEBUG; +} + +use IO::File; +use IO::Dir; + +use HTML::TreeBuilder; +use HTML::ElementTable; + +my %options = (debug => 0, + help => 0, + man => 0, + dir => '.', + keyword => undef, + keywords => 0, + ); + +GetOptions(\%options,'keyword|k=s','dir|D=s','debug|d+','help|h|?','man|m', + 'keywords', + ); + + +pod2usage() if $options{help}; +pod2usage({verbose=>2}) if $options{man}; + +$DEBUG = $options{debug}; + +# CSV columns +use constant {NAME => 0, + REFSEQ => 1, + LOCATION => 2, + ALIAS => 3, + FUNCTION => 4, + DESCRIPTION => 5, + KEYWORD => 6, + DBNAME => 7, + FILENAME => 8, + }; + +if ($options{keywords}) { + if (@ARGV != 1) { + pod2usage("If the --keywords option is used, exactly one argument (the keyword) must be passed"); + } + $options{dir} = "$ARGV[0]_results_uniprot"; +} + +if (not -d $options{dir}) { + die "$options{dir} does not exist or is not a directory"; +} + +my $dir = new IO::Dir $options{dir} or die "Unable to open dir $options{dir}: $!"; + +print join(",", map {qq("$_");} qw(Name RefSeq Location Alias Function Description Keyword DBName Filename)),qq(\n); + +my ($keyword) = $options{keyword} || $options{dir} =~ m#(?:^|/)([^\/]+)_results_uniprot#; + +FILE: while ($_ = $dir->read) { + my $file_name = $_; + next if $file_name =~ /^\./; + next unless -f "$options{dir}/$file_name" and -r "$options{dir}/$file_name"; + + my $file = new IO::File "$options{dir}/$file_name", 'r' or die "Unable to open file $file_name"; + + my @fields; + my $current_field = undef; + LINE: while (<$file>) { + my ($type,$data) = $_ =~ /^(\w{2})\ {3}(.+)/s; + $type ||= 'SQ'; + next LINE if not defined $data; + if (not defined $current_field and $type ne 'ID') { + next FILE; + } + if (not defined $current_field or $current_field->{type} ne $type) { + if (defined $current_field) { + push @fields,$current_field; + } + $current_field = {type => $type, + data => '', + }; + } + $current_field->{data} .= $data; + } + if (defined $current_field) { + push @fields,$current_field; + } + + my @results; + + for my $field (@fields) { + my $type = $field->{type}; + if ($type eq 'GN') { + # Find gene name + ($results[NAME]) = $field->{data} =~ m{Name=(.+?);}xis; + } + elsif ($type eq 'DR') { + # Find REF SEQ number + ($results[REFSEQ]) = $field->{data} =~ m{RefSeq;\s*([^;]+)}xis; + if (not defined $results[REFSEQ]) { + ($results[REFSEQ]) = $field->{data} =~ m{Ensembl;\s*([^;]+)}xis; + } + } + elsif ($type eq 'DE') { + # Find gene aliases; these are odd bits separated by () + # and such. + my $alias = $field->{data}; + $alias =~ s/\n/ /g; + my @aliases = split /(?:\)\s+\(|\s+\(|\)(?:\s+|\.\s*$))/, $alias; + $results[ALIAS] = join('; ',map {s/(\s)\s+/$1/g; $_;} @aliases); + } + elsif ($type eq 'CC') { + my ($function) = $field->{data} =~ m{-!-\s*FUNCTION:\s*(.+?)(?:-!-|$)}xs; + if (defined $function) { + $function =~ s/\n//g; + $function =~ s/(\s)\s+/$1/g; + $results[FUNCTION] = $function; + } + my $description = $field->{data}; + $description =~ s/\n/ /g; + $description =~ s/-!-//g; + $description =~ s/(\s)\s+/$1/g; + $description =~ s/-----{5,}.+$//; + $results[DESCRIPTION] = $description; + } + } + + $results[NAME] ||= 'NO NAME'; + $results[REFSEQ] ||= 'NO REFSEQ'; + + $results[LOCATION] ||= 'NO LOCATION'; + $results[ALIAS] ||= 'NO ALIASES'; + $results[FUNCTION] ||= 'NO FUNCTION'; + + # Figure out the keyword used + $results[KEYWORD] ||= $keyword || 'NO KEYWORD'; + + # Database searched + $results[DBNAME] = 'uniprot'; + $results[FILENAME] = $file_name; + + print join(',',map {qq("$_")} @results),qq(\n); +} + + + + + + +__END__ -- 2.39.2