* Fix genecard and harvester parsers

 * Add headers for results_to_table
 * Call results_to_table in function2gene

git-svn-id: file:///srv/svn/function2gene/trunk@15 a0738b58-4706-0410-8799-fb830574a030

diff --git a/bin/function2gene b/bin/function2gene
index 07dd2aed5f40500c9c7fc59fbae01c8d58444b28..ed9a38ca23a3b948c7c07913c62efde34d1949a5 100755 (executable)
--- a/bin/function2gene
+++ b/bin/function2gene
@@ -56,7 +56,8 @@ databases.]
 =item B<--restart-at>
 
 If you need to restart the process at a particular state (which has
 =item B<--restart-at>
 
 If you need to restart the process at a particular state (which has

-already been completed) specify this option.
+already been completed) specify this option. Valid values are get,
+parse, or combine.

 
 =item B<--debug, -d>
 
 
 =item B<--debug, -d>
 


@@ -78,6 +79,10 @@ Display this manual.
    echo 'transferrin' > keywords.txt
    function2gene --keywords keywords.txt --results keyword_results
 
    echo 'transferrin' > keywords.txt
    function2gene --keywords keywords.txt --results keyword_results
 

+   # reparse the results
+   function2gene --keywords keywords.txt --results keyword_results \
+       --restart-at parse
+

 =cut
 
 
 =cut
 
 


@@ -287,7 +292,11 @@ if ($actions{combine}) {
          $state{done_keywords}{combined}{$db}{$keyword} = 1;
      }
      save_state(\%state);
          $state{done_keywords}{combined}{$db}{$keyword} = 1;
      }
      save_state(\%state);

-     ADVISE("Finished; results in $options{results}/combined_results");
+     write_command_to_file('combined_results_table.txt',
+                          "$base_dir/results_to_table",
+                          'combined_results.txt',
+                         );
+     ADVISE("Finished; results in $options{results}/combined_results.txt");

 }
 else {
      ADVISE('Nothing to do. [Perhaps you wanted --restart-at?]');
 }
 else {
      ADVISE('Nothing to do. [Perhaps you wanted --restart-at?]');

diff --git a/bin/parse_genecard_results b/bin/parse_genecard_results
index 4b7d514d7df0b5352a4fb71aa50f0d8cfd55365e..b42371957a229b4423cd8b89ac152eb2094f7134 100755 (executable)
--- a/bin/parse_genecard_results
+++ b/bin/parse_genecard_results
@@ -70,6 +70,9 @@ BEGIN{
 use IO::File;
 use IO::Dir;
 
 use IO::File;
 use IO::Dir;
 

+use HTML::TreeBuilder;
+use HTML::ElementTable;
+

 my %options = (debug    => 0,
               help     => 0,
               man      => 0,
 my %options = (debug    => 0,
               help     => 0,
               man      => 0,


@@ -139,29 +142,26 @@ while ($_ = $dir->read) {
      $results[REFSEQ] ||= 'NO REFSEQ';
 
      # Find Gene Location
      $results[REFSEQ] ||= 'NO REFSEQ';
 
      # Find Gene Location

-     ($results[LOCATION]) = $result =~ m&<I>LocusLink\s+cytogenetic\s+band:</I><b>\s+
-                                        <a\s+href="[^\"]+"\s+target\s+=\s+"aaa">\s*([^\<]+?)\s*</a>&xis;
+     ($results[LOCATION]) = $result =~ m{cytogenetic\s+band:</I><b>\s+
+                                        <a\s+href="[^\"]+"\s+target\s*=\s*"aaa"[^>]*>\s*([^\<]+?)\s*</a>}xis;

 
      $results[LOCATION] ||= 'NO LOCATION';
 
      # Find gene aliases
 
      $results[LOCATION] ||= 'NO LOCATION';
 
      # Find gene aliases

-     my ($alias_table) = $result =~ m|<b>Aliases and Descriptions</b>(.+?)</TR>|is;
-     $alias_table ||='';
-
-     my @gene_aliases = $alias_table =~ m|<li>\s*([^\(]{0,20}?)\s*\(<FONT|gis;
+     my ($alias_table) = $result =~ m{(<table[^>]+><tr><th[^>]+>Aliases.+?</table>)}is;
+     $alias_table ||= '';
+     my @gene_aliases = map {s/\s*$//; $_;} $alias_table =~ m{<td(?: nowrap)?>\s*([^<]+)<}gis;

 
      $results[ALIAS] = join('; ', @gene_aliases);
      $results[ALIAS] ||= 'NO ALIASES';
 
      # Find gene function(s)
 
 
      $results[ALIAS] = join('; ', @gene_aliases);
      $results[ALIAS] ||= 'NO ALIASES';
 
      # Find gene function(s)
 

-     # Swiss prot functions
-     my @functions = $result =~ m&<li><b>Function:</b>\s+(.+?)(?:<li>)|(?:</ul>)&gis;
-
+     my @functions;

      # GO Functions
      # GO Functions

-     push @functions, (map {s/\n//g}
-                      map {s#\s*\s*</a>(?:</td><td>\s*)?\s*# #g; $_;}
-                      $result =~ m{(GO:\d+\s*</a>(?:</td><td>\s*)?.+?)(?:</td><dd>|<p>)}gis
+     push @functions, (map {s/\n//g; $_;}
+                      map {s#\s*</a>(?:</td><td>\s*)?\s*# #g; $_;}
+                      $result =~ m{(GO:\d+\s*</a>(?:</td><td>\s*)?.+?)(?:</font>|</td>|<dd>|<p>)}gis

                      );
      $results[FUNCTION] = join('; ', map {(defined $_)?($_):()} @functions);
      $results[FUNCTION] ||= 'NO FUNCTION';
                      );
      $results[FUNCTION] = join('; ', map {(defined $_)?($_):()} @functions);
      $results[FUNCTION] ||= 'NO FUNCTION';


@@ -171,8 +171,19 @@ while ($_ = $dir->read) {
 
      $results[KEYWORD] ||= 'NO KEYWORD';
 
 
      $results[KEYWORD] ||= 'NO KEYWORD';
 

+     # Swiss prot functions
+     my @description = (map {s/<[^>]+>/ /g;
+                          s/\s+/ /g;
+                          $_;
+                     }
+                     $result =~ m{<(?:dd|li)><b>Function(?::</b>|</b>:)\s+
+                                  (.+?)<(?:/dd|li)>}xgis
+                    );
+

      # Figure out what the description is
      # Figure out what the description is

-     $results[DESCRIPTION] = '';
+     $results[DESCRIPTION] = join('; ',
+                                 map {(defined $_)?($_):()}
+                                 @description);

 
      # Database searched
      $results[DBNAME] = 'genecard';
 
      # Database searched
      $results[DBNAME] = 'genecard';

diff --git a/bin/parse_harvester_results b/bin/parse_harvester_results
index 30fc0113271433674f6f178a5da882334d533c4f..e52066127ceab16107b04bfc68c919ee150adb86 100755 (executable)
--- a/bin/parse_harvester_results
+++ b/bin/parse_harvester_results
@@ -142,6 +142,13 @@ while ($_ = $dir->read) {
          ($results[NAME]) = $result =~ m&<TR>\s*<TD\s*BGCOLOR="\#FEFE99"\s*VALIGN="top"\s*NOWRAP>Entry\s*name</TD>\s*
                                          <TD\s*VALIGN="top"\s*COLSPAN="5">\s*<b>\s*([^<]+?)\s*</b></TD>\s*</TR>&xis;
      }
          ($results[NAME]) = $result =~ m&<TR>\s*<TD\s*BGCOLOR="\#FEFE99"\s*VALIGN="top"\s*NOWRAP>Entry\s*name</TD>\s*
                                          <TD\s*VALIGN="top"\s*COLSPAN="5">\s*<b>\s*([^<]+?)\s*</b></TD>\s*</TR>&xis;
      }

+     if (not defined $results[NAME]) {
+         ($results[NAME]) = $result =~ m{<TITLE>[^:]+:\s*[^\*]+\*[^\*]+\*\s*([^-]+)}xis;
+         $results[NAME] =~ s/\s*$// if defined $results[NAME];
+         $results[NAME] =~ s/^\s*$// if defined $results[NAME];
+         $results[NAME] =~ s/\d+\s*kDa\s*protein// if defined $results[NAME];
+         $results[NAME] =~ s/\s*similar to .+// if defined $results[NAME];
+     }

 
      $results[NAME] ||= 'NO NAME';
      $results[NAME] =~ s/_HUMAN//;
 
      $results[NAME] ||= 'NO NAME';
      $results[NAME] =~ s/_HUMAN//;

diff --git a/bin/results_to_table b/bin/results_to_table
index 04f2f6df3a01419025bba29390888401e7a80df3..24c94004380c993627d9b3819574c340c840b64b 100755 (executable)
--- a/bin/results_to_table
+++ b/bin/results_to_table
@@ -75,12 +75,14 @@ while (<$search_results_fh>) {
      $databases{total}{total}{count}++;
 }
 
      $databases{total}{total}{count}++;
 }
 

-our ($keyword,$gct,$hvt,$nct,$t);
+our ($keyword,$gct,$hvt,$nct,$t) = ('Keyword','GeneCards','Harvester','NCBI','Total');

      format STDOUT =
 @<<<<<<<<<<<<<<<<<<<<<< & @>>>>>>>>>> & @>>>>>>>>>> & @>>>>>>>>>> & @>>>>>>>>>> \\
 $keyword,                 $gct,     $hvt,     $nct,     $t
 .
      format STDOUT =
 @<<<<<<<<<<<<<<<<<<<<<< & @>>>>>>>>>> & @>>>>>>>>>> & @>>>>>>>>>> & @>>>>>>>>>> \\
 $keyword,                 $gct,     $hvt,     $nct,     $t
 .

-for$keyword (sort keys %terms) {
+write;
+
+for $keyword (sort keys %terms) {

      ($gct,$hvt,$nct,$t) =
          map {
               if (not defined $_) {
      ($gct,$hvt,$nct,$t) =
          map {
               if (not defined $_) {