X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=code_perl%2FMaasha%2FBiopieces.pm;h=29a914a230ef1bccb22aa55409bc9fc06b55fb34;hb=7c9609e8a165301350118cde21387c8a2e63dd42;hp=06d4b3d8a02e9db84e287b9a92dcf71ecb23830e;hpb=e2ee381ee079a78605a6381d5545a741a99c34c8;p=biopieces.git diff --git a/code_perl/Maasha/Biopieces.pm b/code_perl/Maasha/Biopieces.pm index 06d4b3d..29a914a 100644 --- a/code_perl/Maasha/Biopieces.pm +++ b/code_perl/Maasha/Biopieces.pm @@ -51,6 +51,7 @@ use Maasha::NCBI; use Maasha::GFF; use Maasha::TwoBit; use Maasha::Solid; +use Maasha::Solexa; use Maasha::SQL; use Maasha::Gwiki; @@ -161,6 +162,8 @@ sub run_script $options = get_options( $script ); + $options->{ "SCRIPT" } = $script; + if ( $script ne "list_biopieces" and $script ne "list_genomes" ) { $script = "print_usage" if ( -t STDIN and keys %{ $options } <= 1 or $options->{ 'help' } ); } @@ -216,6 +219,7 @@ sub run_script elsif ( $script eq "create_blast_db" ) { script_create_blast_db( $in, $out, $options ) } elsif ( $script eq "blast_seq" ) { script_blast_seq( $in, $out, $options ) } elsif ( $script eq "blat_seq" ) { script_blat_seq( $in, $out, $options ) } + elsif ( $script eq "soap_seq" ) { script_soap_seq( $in, $out, $options ) } elsif ( $script eq "match_seq" ) { script_match_seq( $in, $out, $options ) } elsif ( $script eq "create_vmatch_index" ) { script_create_vmatch_index( $in, $out, $options ) } elsif ( $script eq "vmatch_seq" ) { script_vmatch_seq( $in, $out, $options ) } @@ -230,9 +234,11 @@ sub run_script elsif ( $script eq "write_solid" ) { script_write_solid( $in, $out, $options ) } elsif ( $script eq "head_records" ) { script_head_records( $in, $out, $options ) } elsif ( $script eq "remove_keys" ) { script_remove_keys( $in, $out, $options ) } + elsif ( $script eq "remove_adaptor" ) { script_remove_adaptor( $in, $out, $options ) } elsif ( $script eq "rename_keys" ) { script_rename_keys( $in, $out, $options ) } elsif ( $script eq "uniq_vals" ) { script_uniq_vals( $in, $out, $options ) } elsif ( $script eq "merge_vals" ) { script_merge_vals( $in, $out, $options ) } + elsif ( $script eq "merge_records" ) { script_merge_records( $in, $out, $options ) } elsif ( $script eq "grab" ) { script_grab( $in, $out, $options ) } elsif ( $script eq "compute" ) { script_compute( $in, $out, $options ) } elsif ( $script eq "flip_tab" ) { script_flip_tab( $in, $out, $options ) } @@ -260,9 +266,6 @@ sub run_script close $in if defined $in; close $out; - - # unset status - missing - # write log file - missing } @@ -277,7 +280,7 @@ sub get_options # Returns hash - my ( %options, @options, $opt, @genomes ); + my ( %options, @options, $opt, @genomes, $real ); if ( $script eq "print_usage" ) { @@ -363,6 +366,7 @@ sub get_options { @options = qw( data_in|i=s + samples|s=s num|n=s ); } @@ -386,6 +390,7 @@ sub get_options @options = qw( data_in|i=s num|n=s + format|f=s quality|q=s ); } @@ -569,6 +574,17 @@ sub get_options ooc|c ); } + elsif ( $script eq "soap_seq" ) + { + @options = qw( + in_file|i=s + genome|g=s + seed_size|s=s + mismatches|m=s + gap_size|G=s + cpus|c=s + ); + } elsif ( $script eq "match_seq" ) { @options = qw( @@ -718,6 +734,15 @@ sub get_options save_keys|K=s ); } + elsif ( $script eq "remove_adaptor" ) + { + @options = qw( + adaptor|a=s + mismatches|m=s + no_remove|n + offset|o=s + ); + } elsif ( $script eq "rename_keys" ) { @options = qw( @@ -738,6 +763,13 @@ sub get_options delimit|d=s ); } + elsif ( $script eq "merge_records" ) + { + @options = qw( + keys|k=s + merge|m=s + ); + } elsif ( $script eq "grab" ) { @options = qw( @@ -908,6 +940,8 @@ sub get_options use_score|u visibility|v=s wiggle|w + score|S + log10|L color|c=s chunk_size|C=s ); @@ -921,17 +955,15 @@ sub get_options ); # print STDERR Dumper( \@options ); - + GetOptions( \%options, @options, ); - $options{ "script" } = $script; - # print STDERR Dumper( \%options ); - if ( -t STDIN && scalar( keys %options ) == 1 or $options{ "help" } ) { + if ( -t STDIN && scalar( keys %options ) == 0 or $options{ "help" } ) { return wantarray ? %options : \%options; } @@ -943,6 +975,7 @@ sub get_options $options{ "feats" } = [ split ",", $options{ "feats" } ] if defined $options{ "feats" }; $options{ "frames" } = [ split ",", $options{ "frames" } ] if defined $options{ "frames" }; $options{ "formats" } = [ split ",", $options{ "formats" } ] if defined $options{ "formats" }; + $options{ "samples" } = [ split ",", $options{ "samples" } ] if defined $options{ "samples" }; # ---- check arguments ---- @@ -955,7 +988,9 @@ sub get_options map { Maasha::Common::error( qq(Argument to --cols must be a whole numbers - not "$_") ) if $_ !~ /^\d+$/ } @{ $options{ "cols" } } if $options{ "cols" }; -# print STDERR Dumper( \%options ); + # print STDERR Dumper( \%options ); + + $real = "beg|end|word_size|wrap|chunk_size|tile_size|len|prefix_length|mismatches|offset|num|skip|cpus|window_size|step_size"; foreach $opt ( keys %options ) { @@ -963,7 +998,7 @@ sub get_options { Maasha::Common::error( qq(Argument to --$opt must be a valid file or fileglob expression - not "$options{ $opt }") ); } - elsif ( $opt =~ /beg|end|word_size|wrap|chunk_size|tile_size|len|prefix_length|num|skip|cpus|window_size|step_size/ and $options{ $opt } !~ /^\d+$/ ) + elsif ( $opt =~ /$real/ and $options{ $opt } !~ /^\d+$/ ) { Maasha::Common::error( qq(Argument to --$opt must be a whole number - not "$options{ $opt }") ); } @@ -992,13 +1027,21 @@ sub get_options Maasha::Common::error( qq(Genome $options{ $opt } not found in "$ENV{ 'BP_DATA' }/genomes/") ); } } - elsif ( $opt eq "terminal" and not $options{ $opt } =~ /^(svg|post|dumb)/ ) + elsif ( $opt eq "terminal" and not $options{ $opt } =~ /^(svg|post|dumb|x11)/ ) { Maasha::Common::error( qq(Bad --$opt argument "$options{ $opt }") ); } - elsif ( $opt eq "table" and $options{ $opt } =~ /-\./ ) + elsif ( $opt eq "table" and $options{ $opt } =~ /(-|\.)/ ) { - Maasha::Common::error( qq(Character '$options{ $opt }' is not allowed in table names) ); + Maasha::Common::error( qq(Character '$1' is not allowed in table name: $options{ $opt }) ); + } + elsif ( $opt eq "merge" and $options{ $opt } !~ /^(AandB|AorB|BorA|AnotB|BnotA)$/ ) + { + Maasha::Common::error( qq(Argument to --$opt must be AandB, AorB, BorA, AnotB, or BnotA - not "$options{ $opt }") ); + } + elsif ( $opt eq "format" and $script eq "read_solexa" and $options{ $opt } !~ /octal|decimal/ ) + { + Maasha::Common::error( qq(Argument to --$opt must be octal or decimal - not "$options{ $opt }") ); } } @@ -1007,7 +1050,9 @@ sub get_options Maasha::Common::error( qq(no --database or --genome specified) ) if $script eq "blast_seq" and not $options{ "genome" } and not $options{ "database" }; Maasha::Common::error( qq(both --database and --genome specified) ) if $script eq "blast_seq" and $options{ "genome" } and $options{ "database" }; Maasha::Common::error( qq(no --index_name or --genome specified) ) if $script eq "vmatch_seq" and not $options{ "genome" } and not $options{ "index_name" }; - Maasha::Common::error( qq(both --index and --genome specified) ) if $script eq "vmatch_seq" and $options{ "genome" } and $options{ "index" }; + Maasha::Common::error( qq(both --index and --genome specified) ) if $script eq "vmatch_seq" and $options{ "genome" } and $options{ "index_name" }; + Maasha::Common::error( qq(no --in_file or --genome specified) ) if $script eq "soap_seq" and not $options{ "genome" } and not $options{ "in_file" }; + Maasha::Common::error( qq(both --in_file and --genome specified) ) if $script eq "soap_seq" and $options{ "genome" } and $options{ "in_file" }; Maasha::Common::error( qq(no --genome specified) ) if $script =~ /format_genome|get_genome_seq|get_genome_align|get_genome_phastcons|blat_seq|plot_phastcons_profiles|plot_karyogram/ and not $options{ "genome" }; Maasha::Common::error( qq(no --key specified) ) if $script =~ /plot_lendist|plot_histogram/ and not $options{ "key" }; Maasha::Common::error( qq(no --keys speficied) ) if $script =~ /sort_records|count_vals|sum_vals|mean_vals|median_vals|length_vals/ and not $options{ "keys" }; @@ -1044,7 +1089,7 @@ sub script_print_usage if ( $options->{ 'data_in' } ) { $file = $options->{ 'data_in' }; } else { - $file = join "", $ENV{ 'BP_DIR' }, "/bp_usage/", $options->{ 'script' }, ".wiki"; + $file = join "", $ENV{ 'BP_DIR' }, "/bp_usage/", $options->{ 'SCRIPT' }, ".wiki"; } $wiki = Maasha::Gwiki::gwiki_read( $file ); @@ -1090,6 +1135,7 @@ sub script_list_biopieces @{ $wiki } = grep { $_->[ 0 ]->{ 'FORMAT' } =~ /paragraph/ } @{ $wiki }; $synopsis = $wiki->[ 0 ]->[ 0 ]->{ 'TEXT' }; + $synopsis =~ s/!(\w)/$1/g; printf( "%-30s%s\n", $program, $synopsis ); } @@ -1298,7 +1344,7 @@ sub script_read_psl # Returns nothing. - my ( $record, @files, $file, $entries, $entry, $num ); + my ( $record, $file, $data_in, $num ); while ( $record = get_record( $in ) ) { put_record( $record, $out ); @@ -1308,11 +1354,11 @@ sub script_read_psl foreach $file ( @{ $options->{ "files" } } ) { - $entries = Maasha::UCSC::psl_get_entries( $file ); + $data_in = Maasha::Common::read_open( $file ); - foreach $entry ( @{ $entries } ) + while ( $record = Maasha::UCSC::psl_get_entry( $data_in ) ) { - put_record( $entry, $out ); + put_record( $record, $out ); goto NUM if $options->{ "num" } and $num == $options->{ "num" }; @@ -1398,10 +1444,11 @@ sub script_read_fixedstep if ( $head =~ /^chrom=([^ ]+) start=(\d+) step=(\d+)$/ ) { - $record->{ "CHR" } = $1; - $record->{ "CHR_BEG" } = $2; - $record->{ "STEP" } = $3; - $record->{ "VALS" } = join ",", @{ $entry }; + $record->{ "REC_TYPE" } = "fixed_step"; + $record->{ "CHR" } = $1; + $record->{ "CHR_BEG" } = $2; + $record->{ "STEP" } = $3; + $record->{ "VALS" } = join ",", @{ $entry }; } put_record( $record, $out ); @@ -1706,7 +1753,7 @@ sub script_read_soft # Returns nothing. - my ( $data_in, $file, $num, $records, $record, $soft_index, $fh, @platforms, $plat_table, @samples, $sample, $old_end ); + my ( $data_in, $file, $num, $records, $record, $soft_index, $fh, @platforms, $plat_table, @samples, $sample, $old_end, $skip ); while ( $record = get_record( $in ) ) { put_record( $record, $out ); @@ -1716,21 +1763,30 @@ sub script_read_soft foreach $file ( @{ $options->{ "files" } } ) { + print STDERR "Creating index for file: $file\n" if $options->{ "verbose" }; + $soft_index = Maasha::NCBI::soft_index_file( $file ); $fh = Maasha::Common::read_open( $file ); - @platforms = grep { $_->[ 0 ] =~ /PLATFORM/ } @{ $soft_index }; + @platforms = grep { $_->{ "SECTION" } =~ /PLATFORM/ } @{ $soft_index }; + + print STDERR "Getting platform tables for file: $file\n" if $options->{ "verbose" }; - $plat_table = Maasha::NCBI::soft_get_platform( $fh, $platforms[ 0 ]->[ 1 ], $platforms[ -1 ]->[ 2 ] ); + $plat_table = Maasha::NCBI::soft_get_platform( $fh, $platforms[ 0 ]->{ "LINE_BEG" }, $platforms[ -1 ]->{ "LINE_END" } ); - @samples = grep { $_->[ 0 ] =~ /SAMPLE/ } @{ $soft_index }; + @samples = grep { $_->{ "SECTION" } =~ /SAMPLE/ } @{ $soft_index }; - $old_end = $platforms[ -1 ]->[ 2 ]; + $old_end = $platforms[ -1 ]->{ "LINE_END" }; foreach $sample ( @samples ) { - $records = Maasha::NCBI::soft_get_sample( $fh, $plat_table, $sample->[ 1 ] - $old_end - 1, $sample->[ 2 ] - $old_end - 1 ); + $skip = 0; + $skip = 1 if ( $options->{ "samples" } and grep { $sample->{ "SECTION" } !~ /$_/ } @{ $options->{ "samples" } } ); + + print STDERR "Getting samples for dataset: $sample->{ 'SECTION' }\n" if $options->{ "verbose" } and not $skip; + + $records = Maasha::NCBI::soft_get_sample( $fh, $plat_table, $sample->{ "LINE_BEG" } - $old_end - 1, $sample->{ "LINE_END" } - $old_end - 1, $skip ); foreach $record ( @{ $records } ) { @@ -1741,7 +1797,7 @@ sub script_read_soft $num++; } - $old_end = $sample->[ 2 ]; + $old_end = $sample->{ "LINE_END" }; } close $fh; @@ -1862,8 +1918,9 @@ sub script_read_solexa # Returns nothing. - my ( $record, $file, $base_name, $data_in, $line, $num, @fields, @seqs, @scores, $i, $seq, $seq_count ); + my ( $record, $file, $data_in, $entry, $num, @seqs, @scores, $i ); + $options->{ "format" } ||= "octal"; $options->{ "quality" } ||= 20; while ( $record = get_record( $in ) ) { @@ -1874,35 +1931,33 @@ sub script_read_solexa foreach $file ( @{ $options->{ "files" } } ) { - $data_in = Maasha::Common::read_open( $file ); - $base_name = Maasha::Common::get_basename( $file ); - $base_name =~ s/\..*//; - - $seq_count = 0; + $data_in = Maasha::Common::read_open( $file ); - while ( $line = <$data_in> ) + if ( $options->{ "format" } eq "octal" ) { - @fields = split /:/, $line; - @seqs = split //, $fields[ 5 ]; - @scores = split / /, $fields[ -1 ]; + while ( $entry = Maasha::Solexa::solexa_get_entry_octal( $data_in ) ) + { + $record = Maasha::Solexa::solexa2biopiece( $entry, $options->{ "quality" } ); - for ( $i = 0; $i < @scores; $i++ ) { - $seqs[ $i ] = lc $seqs[ $i ] if $scores[ $i ] < $options->{ "quality" }; - } + put_record( $record, $out ); - $seq = join "", @seqs; + goto NUM if $options->{ "num" } and $num == $options->{ "num" }; - $record->{ "SEQ_NAME" } = sprintf( "%s_ID%08d", $base_name, $seq_count ); - $record->{ "SEQ" } = $seq; - $record->{ "SEQ_LEN" } = length $seq; - $record->{ "SCORE_MEAN" } = sprintf ( "%.2f", Maasha::Calc::mean( \@scores ) ); + $num++; + } + } + else + { + while ( $entry = Maasha::Solexa::solexa_get_entry_decimal( $data_in ) ) + { + $record = Maasha::Solexa::solexa2biopiece( $entry, $options->{ "quality" } ); - put_record( $record, $out ); + put_record( $record, $out ); - goto NUM if $options->{ "num" } and $num == $options->{ "num" }; + goto NUM if $options->{ "num" } and $num == $options->{ "num" }; - $seq_count++; - $num++; + $num++; + } } close $data_in; @@ -2024,7 +2079,7 @@ sub script_format_genome # Returns nothing. - my ( $dir, $genome, $fasta_dir, $phastcons_dir, $vals, $fh_out, $record, $format, $index ); + my ( $dir, $genome, $fasta_dir, $phastcons_dir, $vals, $fh_out, $record, $format, $index, $entry ); $dir = $options->{ 'dir' } || $ENV{ 'BP_DATA' }; $genome = $options->{ 'genome' }; @@ -2059,9 +2114,9 @@ sub script_format_genome while ( $record = get_record( $in ) ) { - if ( $fh_out and $record->{ "SEQ_NAME" } and $record->{ "SEQ" } ) + if ( $fh_out and $entry = record2fasta( $record ) ) { - Maasha::Fasta::put_entry( [ $record->{ "SEQ_NAME" }, $record->{ "SEQ" } ], $fh_out, $options->{ "wrap" } ); + Maasha::Fasta::put_entry( $entry, $fh_out, $options->{ "wrap" } ); } elsif ( $fh_out and $record->{ "CHR" } and $record->{ "CHR_BEG" } and $record->{ "STEP" } and $record->{ "VALS" } ) { @@ -3283,8 +3338,6 @@ sub script_patscan_seq $i++; } - -# put_record( $record, $out ); } close $fh_out; @@ -3354,7 +3407,7 @@ sub script_create_blast_db # Returns nothing. - my ( $fh, $seq_type, $path, $record ); + my ( $fh, $seq_type, $path, $record, $entry ); $path = $options->{ "database" }; @@ -3364,11 +3417,11 @@ sub script_create_blast_db { put_record( $record, $out ) if not $options->{ "no_stream" }; - if ( $record->{ "SEQ" } and $record->{ "SEQ_NAME" } ) + if ( $entry = record2fasta( $record ) ) { - $seq_type = Maasha::Seq::seq_guess_type( $record->{ "SEQ" } ) if not $seq_type; + $seq_type = Maasha::Seq::seq_guess_type( $entry->[ SEQ ] ) if not $seq_type; - Maasha::Fasta::put_entry( [ $record->{ "SEQ_NAME" }, $record->{ "SEQ" } ], $fh ); + Maasha::Fasta::put_entry( $entry, $fh ); } } @@ -3397,14 +3450,14 @@ sub script_blast_seq # Returns nothing. - my ( $genome, $q_type, $s_type, $tmp_in, $tmp_out, $fh_in, $fh_out, $record, $line, @fields ); + my ( $genome, $q_type, $s_type, $tmp_in, $tmp_out, $fh_in, $fh_out, $record, $line, @fields, $entry ); $options->{ "e_val" } = 10 if not defined $options->{ "e_val" }; $options->{ "filter" } = "F"; $options->{ "filter" } = "T" if $options->{ "filter" }; $options->{ "cpus" } ||= 1; - $options->{ "database" } = "$ENV{ 'BP_DATA' }/genomes/$options->{ 'genome' }/blast/$options->{ 'genome' }.fna"if $options->{ 'genome' }; + $options->{ "database" } = "$ENV{ 'BP_DATA' }/genomes/$options->{ 'genome' }/blast/$options->{ 'genome' }.fna" if $options->{ 'genome' }; $tmp_in = "$BP_TMP/blast_query.seq"; $tmp_out = "$BP_TMP/blast.result"; @@ -3413,11 +3466,11 @@ sub script_blast_seq while ( $record = get_record( $in ) ) { - if ( $record->{ "SEQ_NAME" } and $record->{ "SEQ" } ) + if ( $entry = record2fasta( $record ) ) { - $q_type = Maasha::Seq::seq_guess_type( $record->{ "SEQ" } ) if not $q_type; + $q_type = Maasha::Seq::seq_guess_type( $entry->[ SEQ ] ) if not $q_type; - Maasha::Fasta::put_entry( [ $record->{ "SEQ_NAME" }, $record->{ "SEQ" } ], $fh_out ); + Maasha::Fasta::put_entry( $entry, $fh_out ); } put_record( $record, $out ); @@ -3444,7 +3497,41 @@ sub script_blast_seq } } - Maasha::Common::run( "blastall", "-p $options->{ 'program' } -e $options->{ 'e_val' } -a $options->{ 'cpus' } -m 8 -i $tmp_in -d $options->{ 'database' } -F $options->{ 'filter' } -o $tmp_out > /dev/null 2>&1", 1 ); + if ( $options->{ 'verbose' } ) + { + Maasha::Common::run( + "blastall", + join( " ", + "-p $options->{ 'program' }", + "-e $options->{ 'e_val' }", + "-a $options->{ 'cpus' }", + "-m 8", + "-i $tmp_in", + "-d $options->{ 'database' }", + "-F $options->{ 'filter' }", + "-o $tmp_out", + ), + 1 + ); + } + else + { + Maasha::Common::run( + "blastall", + join( " ", + "-p $options->{ 'program' }", + "-e $options->{ 'e_val' }", + "-a $options->{ 'cpus' }", + "-m 8", + "-i $tmp_in", + "-d $options->{ 'database' }", + "-F $options->{ 'filter' }", + "-o $tmp_out", + "> /dev/null 2>&1" + ), + 1 + ); + } unlink $tmp_in; @@ -3507,7 +3594,7 @@ sub script_blat_seq # Returns nothing. - my ( $blat_args, $genome_file, $query_file, $fh_in, $fh_out, $type, $record, $result_file, $entries ); + my ( $blat_args, $genome_file, $query_file, $fh_in, $fh_out, $type, $record, $result_file, $entries, $entry ); $genome_file = "$ENV{ 'BP_DATA' }/genomes/$options->{ 'genome' }/fasta/$options->{ 'genome' }.fna"; @@ -3530,10 +3617,10 @@ sub script_blat_seq while ( $record = get_record( $in ) ) { - if ( $record->{ "SEQ_NAME" } and $record->{ "SEQ" } ) + if ( $entry = record2fasta( $record ) ) { - Maasha::Fasta::put_entry( [ $record->{ "SEQ_NAME" }, $record->{ "SEQ" } ], $fh_out, 80 ); - $type = Maasha::Seq::seq_guess_type( $record->{ "SEQ" } ) if not $type; + $type = Maasha::Seq::seq_guess_type( $entry->[ SEQ ] ) if not $type; + Maasha::Fasta::put_entry( $entry, $fh_out, 80 ); } put_record( $record, $out ); @@ -3558,6 +3645,98 @@ sub script_blat_seq } +sub script_soap_seq +{ + # Martin A. Hansen, July 2008. + + # soap sequences in stream against a given file or genome. + + my ( $in, # handle to in stream + $out, # handle to out stream + $options, # options hash + ) = @_; + + # Returns nothing. + + my ( $genome, $tmp_in, $tmp_out, $fh_in, $fh_out, $record, $line, @fields, $entry, $count, $args ); + + $options->{ "seed_size" } ||= 10; + $options->{ "mismatches" } ||= 2; + $options->{ "gap_size" } ||= 0; + $options->{ "cpus" } ||= 1; + + if ( $options->{ "genome" } ) { + $options->{ "in_file" } = "$ENV{ 'BP_DATA' }/genomes/$options->{ 'genome' }/fasta/$options->{ 'genome' }.fna"; + } + + $tmp_in = "$BP_TMP/soap_query.seq"; + $tmp_out = "$BP_TMP/soap.result"; + + $fh_out = Maasha::Common::write_open( $tmp_in ); + + $count = 0; + + while ( $record = get_record( $in ) ) + { + if ( $entry = record2fasta( $record ) ) + { + Maasha::Fasta::put_entry( $entry, $fh_out ); + + $count++; + } + + put_record( $record, $out ); + } + + close $fh_out; + + if ( $count > 0 ) + { + $args = join( " ", + "-s $options->{ 'seed_size' }", + "-r 2", + "-a $tmp_in", + "-v $options->{ 'mismatches' }", + "-g $options->{ 'gap_size' }", + "-p $options->{ 'cpus' }", + "-d $options->{ 'in_file' }", + "-o $tmp_out", + ); + + $args .= " > /dev/null 2>&1" if not $options->{ 'verbose' }; + + Maasha::Common::run( "soap", $args, 1 ); + + unlink $tmp_in; + + $fh_out = Maasha::Common::read_open( $tmp_out ); + + undef $record; + + while ( $line = <$fh_out> ) + { + chomp $line; + + @fields = split /\t/, $line; + + $record->{ "REC_TYPE" } = "SOAP"; + $record->{ "Q_ID" } = $fields[ 0 ]; + $record->{ "SCORE" } = $fields[ 3 ]; + $record->{ "STRAND" } = $fields[ 6 ]; + $record->{ "S_ID" } = $fields[ 7 ]; + $record->{ "S_BEG" } = $fields[ 8 ] - 1; # soap is 1-based + $record->{ "S_END" } = $fields[ 8 ] + $fields[ 5 ] - 2; + + put_record( $record, $out ); + } + + close $fh_out; + } + + unlink $tmp_out; +} + + sub script_match_seq { # Martin A. Hansen, August 2007. @@ -3613,7 +3792,7 @@ sub script_create_vmatch_index # Returns nothing. - my ( $record, $file_tmp, $fh_tmp, $type ); + my ( $record, $file_tmp, $fh_tmp, $type, $entry ); if ( $options->{ "index_name" } ) { @@ -3623,11 +3802,11 @@ sub script_create_vmatch_index while ( $record = get_record( $in ) ) { - if ( $options->{ "index_name" } and $record->{ "SEQ_NAME" } and $record->{ "SEQ" } ) + if ( $options->{ "index_name" } and $entry = record2fasta( $record ) ) { - Maasha::Fasta::put_entry( [ $record->{ "SEQ_NAME" }, $record->{ "SEQ" } ], $fh_tmp ); + Maasha::Fasta::put_entry( $entry, $fh_tmp ); - $type = Maasha::Seq::seq_guess_type( $record->{ "SEQ" } ) if not $type; + $type = Maasha::Seq::seq_guess_type( $entry->[ SEQ ] ) if not defined $type; } put_record( $record, $out ) if not $options->{ "no_stream" }; @@ -3714,14 +3893,14 @@ sub script_write_fasta # Returns nothing. - my ( $record, $fh ); + my ( $record, $fh, $entry ); $fh = write_stream( $options->{ "data_out" }, $options->{ "compress" } ); while ( $record = get_record( $in ) ) { - if ( $record->{ "SEQ_NAME" } and $record->{ "SEQ" } ) { - Maasha::Fasta::put_entry( [ $record->{ "SEQ_NAME" }, $record->{ "SEQ" } ], $fh, $options->{ "wrap" } ); + if ( $entry = record2fasta( $record ) ) { + Maasha::Fasta::put_entry( $entry, $fh, $options->{ "wrap" } ); } put_record( $record, $out ) if not $options->{ "no_stream" }; @@ -3953,6 +4132,17 @@ sub script_write_bed Maasha::UCSC::bed_put_entry( $new_record, $fh, 6 ); } + elsif ( $record->{ "REC_TYPE" } eq "SOAP" and $record->{ "S_ID" } =~ /^chr/i ) # ---- Hits from Vmatch ---- + { + $new_record->{ "CHR" } = $record->{ "S_ID" }; + $new_record->{ "CHR_BEG" } = $record->{ "S_BEG" }; + $new_record->{ "CHR_END" } = $record->{ "S_END" }; + $new_record->{ "Q_ID" } = $record->{ "Q_ID" }; + $new_record->{ "SCORE" } = $record->{ "SCORE" } || 999; + $new_record->{ "STRAND" } = $record->{ "STRAND" }; + + Maasha::UCSC::bed_put_entry( $new_record, $fh, 6 ); + } elsif ( $record->{ "CHR" } and defined $record->{ "CHR_BEG" } and $record->{ "CHR_END" } ) # ---- Generic data from tables ---- { Maasha::UCSC::bed_put_entry( $record, $fh ); @@ -4050,7 +4240,7 @@ sub script_write_2bit # Returns nothing. - my ( $record, $mask, $tmp_file, $fh_tmp, $fh_in, $fh_out ); + my ( $record, $mask, $tmp_file, $fh_tmp, $fh_in, $fh_out, $entry ); $mask = 1 if not $options->{ "no_mask" }; @@ -4061,8 +4251,8 @@ sub script_write_2bit while ( $record = get_record( $in ) ) { - if ( $record->{ "SEQ_NAME" } and $record->{ "SEQ" } ) { - Maasha::Fasta::put_entry( [ $record->{ "SEQ_NAME" }, $record->{ "SEQ" } ], $fh_tmp ); + if ( $entry = record2fasta( $record ) ) { + Maasha::Fasta::put_entry( $entry, $fh_tmp ); } put_record( $record, $out ) if not $options->{ "no_stream" }; @@ -4094,17 +4284,17 @@ sub script_write_solid # Returns nothing. - my ( $record, $fh, $seq_cs ); + my ( $record, $fh, $entry ); $fh = write_stream( $options->{ "data_out" }, $options->{ "compress" } ); while ( $record = get_record( $in ) ) { - if ( $record->{ "SEQ_NAME" } and $record->{ "SEQ" } ) + if ( $entry = record2fasta( $record ) ) { - $seq_cs = Maasha::Solid::seq2color_space( uc $record->{ "SEQ" } ); + $entry->[ SEQ ] = Maasha::Solid::seq2color_space( uc $entry->[ SEQ ] ); - Maasha::Fasta::put_entry( [ $record->{ "SEQ_NAME" }, $seq_cs ], $fh, $options->{ "wrap" } ); + Maasha::Fasta::put_entry( $entry, $fh, $options->{ "wrap" } ); } put_record( $record, $out ) if not $options->{ "no_stream" }; @@ -4376,6 +4566,57 @@ sub script_remove_keys } +sub script_remove_adaptor +{ + # Martin A. Hansen, August 2008. + + # Find and remove adaptor from sequences in the stream. + + my ( $in, # handle to in stream + $out, # handle to out stream + $options, # options hash + ) = @_; + + # Returns nothing. + + my ( $record, $adaptor, $seq, $adaptor_len, $seq_len, $offset, $max_match, $max_mismatch, $pos ); + + $max_mismatch = $options->{ "mismatches" } || 0; + $offset = $options->{ "offset" } || 15; + $adaptor = $options->{ "adaptor" }; + $adaptor_len = length $adaptor; + $adaptor = [ split //, uc $adaptor ]; + + $max_match = $adaptor_len - $max_mismatch; + + while ( $record = get_record( $in ) ) + { + if ( $record->{ "SEQ" } ) + { + $seq = $record->{ "SEQ" }; + $seq_len = length $seq; + $seq = [ split //, uc $seq ]; + + $pos = Maasha::Seq::find_adaptor( $adaptor, $seq, $adaptor_len, $seq_len, $offset, $max_match, $max_mismatch ); + + $record->{ "ADAPTOR_POS" } = $pos; + + if ( $pos >= 0 and not $options->{ "no_remove" } ) + { + $record->{ "SEQ" } = substr $record->{ "SEQ" }, 0, $pos; + $record->{ "SEQ_LEN" } = $pos; + } + + put_record( $record, $out ); + } + else + { + put_record( $record, $out ); + } + } +} + + sub script_rename_keys { # Martin A. Hansen, August 2007. @@ -4482,6 +4723,177 @@ sub script_merge_vals } +sub script_merge_records +{ + # Martin A. Hansen, July 2008. + + # Merges records in the stream based on identical values of two given keys. + + my ( $in, # handle to in stream + $out, # handle to out stream + $options, # options hash + ) = @_; + + # Returns nothing. + + my ( $merge, $record, $file1, $file2, $fh1, $fh2, $key1, $key2, @keys1, @keys2, @vals1, @vals2, + $num1, $num2, $num, $cmp, $i ); + + $merge = $options->{ "merge" } || "AandB"; + + $file1 = "$BP_TMP/merge_records1.tmp"; + $file2 = "$BP_TMP/merge_records2.tmp"; + + $fh1 = Maasha::Common::write_open( $file1 ); + $fh2 = Maasha::Common::write_open( $file2 ); + + $key1 = $options->{ "keys" }->[ 0 ]; + $key2 = $options->{ "keys" }->[ 1 ]; + + $num = $key2 =~ s/n$//; + $num1 = 0; + $num2 = 0; + + while ( $record = get_record( $in ) ) + { + if ( exists $record->{ $key1 } ) + { + @keys1 = $key1; + @vals1 = $record->{ $key1 }; + + delete $record->{ $key1 }; + + map { push @keys1, $_; push @vals1, $record->{ $_ } } keys %{ $record }; + + print $fh1 join( "\t", @vals1 ), "\n"; + + $num1++; + } + elsif ( exists $record->{ $key2 } ) + { + @keys2 = $key2; + @vals2 = $record->{ $key2 }; + + delete $record->{ $key2 }; + + map { push @keys2, $_; push @vals2, $record->{ $_ } } keys %{ $record }; + + print $fh2 join( "\t", @vals2 ), "\n"; + + $num2++; + } + } + + close $fh1; + close $fh2; + + if ( $num ) + { + Maasha::Common::run( "sort", "-k 1,1n $file1 > $file1.sort" ) and rename "$file1.sort", $file1; + Maasha::Common::run( "sort", "-k 1,1n $file2 > $file2.sort" ) and rename "$file2.sort", $file2; + } + else + { + Maasha::Common::run( "sort", "-k 1,1 $file1 > $file1.sort" ) and rename "$file1.sort", $file1; + Maasha::Common::run( "sort", "-k 1,1 $file2 > $file2.sort" ) and rename "$file2.sort", $file2; + } + + $fh1 = Maasha::Common::read_open( $file1 ); + $fh2 = Maasha::Common::read_open( $file2 ); + + @vals1 = Maasha::Common::get_fields( $fh1 ); + @vals2 = Maasha::Common::get_fields( $fh2 ); + + while ( $num1 > 0 and $num2 > 0 ) + { + undef $record; + + if ( $num ) { + $cmp = $vals1[ 0 ] <=> $vals2[ 0 ]; + } else { + $cmp = $vals1[ 0 ] cmp $vals2[ 0 ]; + } + + if ( $cmp < 0 ) + { + if ( $merge =~ /^(AorB|AnotB)$/ ) + { + for ( $i = 0; $i < @keys1; $i++ ) { + $record->{ $keys1[ $i ] } = $vals1[ $i ]; + } + + put_record( $record, $out ); + } + + @vals1 = Maasha::Common::get_fields( $fh1 ); + $num1--; + } + elsif ( $cmp > 0 ) + { + if ( $merge =~ /^(BorA|BnotA)$/ ) + { + for ( $i = 0; $i < @keys2; $i++ ) { + $record->{ $keys2[ $i ] } = $vals2[ $i ]; + } + + put_record( $record, $out ); + } + + @vals2 = Maasha::Common::get_fields( $fh2 ); + $num2--; + } + else + { + if ( $merge =~ /^(AandB|AorB|BorA)$/ ) + { + for ( $i = 0; $i < @keys1; $i++ ) { + $record->{ $keys1[ $i ] } = $vals1[ $i ]; + } + + for ( $i = 1; $i < @keys2; $i++ ) { + $record->{ $keys2[ $i ] } = $vals2[ $i ]; + } + + put_record( $record, $out ); + } + + @vals1 = Maasha::Common::get_fields( $fh1 ); + @vals2 = Maasha::Common::get_fields( $fh2 ); + $num1--; + $num2--; + } + } + + close $fh1; + close $fh2; + + unlink $file1; + unlink $file2; + + if ( $num1 > 0 and $merge =~ /^(AorB|AnotB)$/ ) + { + undef $record; + + for ( $i = 0; $i < @keys1; $i++ ) { + $record->{ $keys1[ $i ] } = $vals1[ $i ]; + } + + put_record( $record, $out ); + } + + if ( $num2 > 0 and $merge =~ /^(BorA|BnotA)$/ ) + { + undef $record; + + for ( $i = 0; $i < @keys2; $i++ ) { + $record->{ $keys2[ $i ] } = $vals2[ $i ]; + } + + put_record( $record, $out ); + } +} + + sub script_grab { # Martin A. Hansen, August 2007. @@ -4830,7 +5242,7 @@ sub script_count_records } } - $result = { "count_records" => $count }; + $result = { "RECORDS_COUNT" => $count }; $fh = write_stream( $options->{ "data_out" } ); @@ -4977,6 +5389,7 @@ sub script_count_vals $fh_out = Maasha::Common::write_open( $tmp_file ); + $cache = 0; $num = 0; while ( $record = get_record( $in ) ) @@ -5003,7 +5416,7 @@ sub script_count_vals if ( $cache ) { - $num = 0; + $num = 0; $fh_in = Maasha::Common::read_open( $tmp_file ); @@ -5499,6 +5912,8 @@ sub script_upload_to_ucsc # Calculate the mean of values of given keys. + # This routine has developed into the most ugly hack. Do something! + my ( $in, # handle to in stream $out, # handle to out stream $options, # options hash @@ -5507,7 +5922,7 @@ sub script_upload_to_ucsc # Returns nothing. my ( $record, $file, $wib_file, $wig_file, $wib_dir, $fh_in, $fh_out, $i, $first, $format, $args, $type, $columns, $append, %fh_hash, - $chr, $beg, $end, $block, $line, $max, $beg_block, $entry, $q_id, $clones ); + $chr, $beg, $end, $block, $line, $max, $beg_block, $entry, $q_id, $clones, $vals ); $options->{ "short_label" } ||= $options->{ 'table' }; $options->{ "long_label" } ||= $options->{ 'table' }; @@ -5527,8 +5942,6 @@ sub script_upload_to_ucsc if ( $options->{ 'wiggle' } ) { - $options->{ "visibility" } = "full"; - while ( $record = get_record( $in ) ) { put_record( $record, $out ) if not $options->{ "no_stream" }; @@ -5563,7 +5976,9 @@ sub script_upload_to_ucsc $end = $entry->{ 'CHR_END' }; $q_id = $entry->{ 'Q_ID' }; - if ( $q_id =~ /_(\d+)$/ ) { + if ( $options->{ "score" } ) { + $clones = $entry->{ 'SCORE' }; + } elsif ( $q_id =~ /_(\d+)$/ ) { $clones = $1; } else { $clones = 1; @@ -5599,29 +6014,13 @@ sub script_upload_to_ucsc close $fh_in; - Maasha::UCSC::fixedstep_put_entry( $chr, $beg_block, $block, $fh_out ); + Maasha::UCSC::fixedstep_put_entry( $chr, $beg_block, $block, $fh_out, $options->{ "log10" } ); unlink "$BP_TMP/$chr"; } close $fh_out; - $wig_file = "$options->{ 'table' }.wig"; - $wib_file = "$options->{ 'table' }.wib"; - - $wib_dir = "$ENV{ 'BP_DATA' }/genomes/$options->{ 'database' }/wib"; - - Maasha::Common::dir_create_if_not_exists( $wib_dir ); - - # Maasha::Common::run( "wigEncode", "$file $wig_file $wib_file > /dev/null 2>&1" ); - - `cd $BP_TMP && wigEncode $file $wig_file $wib_file > /dev/null 2>&1`; - Maasha::Common::run( "mv", "$BP_TMP/$wib_file $wib_dir" ); - - unlink $file; - - $file = $wig_file; - $format = "WIGGLE"; } else @@ -5632,7 +6031,17 @@ sub script_upload_to_ucsc { put_record( $record, $out ) if not $options->{ "no_stream" }; - if ( $record->{ "REC_TYPE" } eq "PSL" ) + if ( $record->{ "REC_TYPE" } eq "fixed_step" ) + { + $vals = $record->{ "VALS" }; + $vals =~ tr/,/\n/; + + print $fh_out "fixedStep chrom=$record->{ 'CHR' } start=$record->{ 'CHR_BEG' } step=$record->{ 'STEP' }\n"; + print $fh_out "$vals\n"; + + $format = "WIGGLE" if not $format; + } + elsif ( $record->{ "REC_TYPE" } eq "PSL" ) { Maasha::UCSC::psl_put_header( $fh_out ) if $first; Maasha::UCSC::psl_put_entry( $record, $fh_out ); @@ -5748,6 +6157,27 @@ sub script_upload_to_ucsc } elsif ( $format eq "WIGGLE" ) { + $options->{ "visibility" } = "full"; + + $wig_file = "$options->{ 'table' }.wig"; + $wib_file = "$options->{ 'table' }.wib"; + + $wib_dir = "$ENV{ 'HOME' }/ucsc/wib"; + + Maasha::Common::dir_create_if_not_exists( $wib_dir ); + + if ( $options->{ 'verbose' } ) { + `cd $BP_TMP && wigEncode $file $wig_file $wib_file`; + } else { + `cd $BP_TMP && wigEncode $file $wig_file $wib_file > /dev/null 2>&1`; + } + + Maasha::Common::run( "mv", "$BP_TMP/$wib_file $wib_dir" ); + + unlink $file; + + $file = $wig_file; + $type = "wig 0"; Maasha::UCSC::wiggle_upload_to_ucsc( $BP_TMP, $wib_dir, $file, $options ); @@ -5763,6 +6193,32 @@ sub script_upload_to_ucsc # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< +sub record2fasta +{ + # Martin A. Hansen, July 2008. + + # Given a biopiece record converts it to a FASTA record. + # If no generic SEQ or SEQ_NAME is found, the Q_* and S_* are + # tried in that order. + + my ( $record, # record + ) = @_; + + # Returns a tuple. + + my ( $seq_name, $seq ); + + $seq_name = $record->{ "SEQ_NAME" } || $record->{ "Q_ID" } || $record->{ "S_ID" }; + $seq = $record->{ "SEQ" } || $record->{ "Q_SEQ" } || $record->{ "S_SEQ" }; + + if ( defined $seq_name and defined $seq ) { + return wantarray ? ( $seq_name, $seq ) : [ $seq_name, $seq ]; + } else { + return; + } +} + + sub read_stream { # Martin A. Hansen, July 2007. @@ -5821,10 +6277,10 @@ sub get_record # Reads one record at a time and converts that record # to a Perl data structure (a hash) which is returned. - my ( $fh, + my ( $fh, # handle to stream ) = @_; - # Returns data structure. + # Returns a hash. my ( $block, @lines, $line, $key, $value, %record ); @@ -5840,7 +6296,7 @@ sub get_record foreach $line ( @lines ) { - ( $key, $value ) = split ": ", $line; + ( $key, $value ) = split ": ", $line, 2; $record{ $key } = $value; } @@ -5940,69 +6396,63 @@ sub sig_handler print STDERR "\nProgram '$script' died->$sig" . " - Please wait for temporary data to be removed\n"; } - # This is a really bad solution, potentially, anyone can include this module and set - # the BP_TMP to point at any dir and thus take out the machine !!! - - Maasha::Common::dir_remove( $BP_TMP ); + clean_tmp(); } exit( 0 ); } -END -{ - # This is a really bad solution, potentially, anyone can include this module and set - # the BP_TMP to point at any dir and thus take out the machine !!! - - Maasha::Common::dir_remove( $BP_TMP ); -} - - -# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< - -1; - -__END__ - - -sub script_read_soft +sub clean_tmp { - # Martin A. Hansen, December 2007. - - # Read soft format. - # http://www.ncbi.nlm.nih.gov/geo/info/soft2.html + # Martin A. Hansen, July 2008. - my ( $in, # handle to in stream - $out, # handle to out stream - $options, # options hash - ) = @_; + # Cleans out any unused temporary files and direcotries in BP_TMP. # Returns nothing. - my ( $data_in, $file, $num, $records, $record ); + my ( $tmpdir, @dirs, $curr_pid, $dir, $user, $sid, $pid ); - while ( $record = get_record( $in ) ) { - put_record( $record, $out ); - } + $tmpdir = $ENV{ 'BP_TMP' } || Maasha::Common::error( 'No BP_TMP variable in environment.' ); - $num = 1; + $curr_pid = Maasha::Common::get_processid(); - foreach $file ( @{ $options->{ "files" } } ) - { - $records = Maasha::NCBI::soft_parse( $file ); + @dirs = Maasha::Common::ls_dirs( $tmpdir ); - foreach $record ( @{ $records } ) + foreach $dir ( @dirs ) + { + if ( $dir =~ /^$tmpdir\/(.+)_(\d+)_(\d+)_bp_tmp$/ ) { - put_record( $record, $out ); - - goto NUM if $options->{ "num" } and $num == $options->{ "num" }; + $user = $1; + $sid = $2; + $pid = $3; - $num++; + if ( $user eq Maasha::Common::get_user() ) + { + if ( not Maasha::Common::process_running( $pid ) ) + { + # print STDERR "Removing stale dir: $dir\n"; + Maasha::Common::dir_remove( $dir ); + } + elsif ( $pid == $curr_pid ) + { + # print STDERR "Removing current dir: $dir\n"; + Maasha::Common::dir_remove( $dir ); + } + } } } +} - NUM: - close $data_in if $data_in; +END +{ + clean_tmp(); } + + +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + +1; + +__END__