From e0b1c18d1409ea7059d3efd22229d1334aeef6c6 Mon Sep 17 00:00:00 2001 From: martinahansen Date: Tue, 26 May 2009 11:18:29 +0000 Subject: [PATCH] migrated count_vals git-svn-id: http://biopieces.googlecode.com/svn/trunk@413 74ccb610-7750-0410-82ae-013aeee3265d --- bp_bin/compute | 100 ++++++++++++++++++++++++++- bp_bin/count_vals | 135 +++++++++++++++++++++++++++++++++++- code_perl/Maasha/BioRun.pm | 138 +------------------------------------ 3 files changed, 230 insertions(+), 143 deletions(-) diff --git a/bp_bin/compute b/bp_bin/compute index fdf5bd2..701c3d2 100755 --- a/bp_bin/compute +++ b/bp_bin/compute @@ -1,6 +1,100 @@ -#!/usr/bin/env perl +#!/usr/bin/env perl -w + +# Copyright (C) 2007-2009 Martin A. Hansen. + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# http://www.gnu.org/copyleft/gpl.html + + +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> DESCRIPTION <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + +# Performs computations on records in stream. + +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + -use warnings; use strict; +use Maasha::Common; +use Maasha::Biopieces; + + +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + + +my ( $run_time_beg, $run_time_end, $options, $in, $out, $record, $eval_key, @keys, $eval_val ); + +$options = Maasha::Biopieces::parse_options( + [ + { long => 'eval', short => 'e', type => 'string', mandatory => 'yes', default => undef, allowed => undef, disallowed => undef }, + ] +); + +$in = Maasha::Biopieces::read_stream( $options->{ "stream_in" } ); +$out = Maasha::Biopieces::write_stream( $options->{ "stream_out" } ); + +while ( $record = Maasha::Biopieces::get_record( $in ) ) +{ + if ( $options->{ "eval" } =~ /^(\S+)\s*=\s*(.+)$/ ) + { + $eval_key = $1; + $eval_val = $2; + + if ( not @keys ) + { + @keys = split /\s+|\+|-|\*|\/|\*\*/, $eval_val; + + @keys = grep { exists $record->{ $_ } } @keys; + } + + map { $eval_val =~ s/\Q$_\E/$record->{ $_ }/g } @keys; + + $record->{ $eval_key } = eval "$eval_val"; + Maasha::Common::error( qq(eval "$eval_key = $eval_val" failed -> $@) ) if $@; + } + else + { + Maasha::Common::error( qq(Bad compute expression: "$options->{ 'eval' }"\n) ); + } + + Maasha::Biopieces::put_record( $record, $out ); +} + + +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + + +BEGIN +{ + $run_time_beg = Maasha::Biopieces::run_time(); + + Maasha::Biopieces::log_biopiece(); +} + +END +{ + Maasha::Biopieces::close_stream( $in ); + Maasha::Biopieces::close_stream( $out ); + + $run_time_end = Maasha::Biopieces::run_time(); + + Maasha::Biopieces::run_time_print( $run_time_beg, $run_time_end, $options ); +} + + +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + -use Maasha::BioRun; +__END__ diff --git a/bp_bin/count_vals b/bp_bin/count_vals index fdf5bd2..3beaeed 100755 --- a/bp_bin/count_vals +++ b/bp_bin/count_vals @@ -1,6 +1,135 @@ -#!/usr/bin/env perl +#!/usr/bin/env perl -w + +# Copyright (C) 2007-2009 Martin A. Hansen. + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# http://www.gnu.org/copyleft/gpl.html + + +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> DESCRIPTION <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + +# Count the number of times values of given keys exists in stream. + +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + -use warnings; use strict; +use Maasha::Filesys; +use Maasha::Biopieces; + + +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + + +my ( $run_time_beg, $run_time_end, $options, $in, $out, $num, $record, %count_hash, @records, $tmp_dir, $tmp_file, $fh_out, $fh_in, $cache ); + +$options = Maasha::Biopieces::parse_options( + [ + { long => 'keys', short => 'k', type => 'list', mandatory => 'yes', default => undef, allowed => undef, disallowed => undef }, + ] +); + +$in = Maasha::Biopieces::read_stream( $options->{ "stream_in" } ); +$out = Maasha::Biopieces::write_stream( $options->{ "stream_out" } ); + +$tmp_dir = Maasha::Biopieces::get_tmpdir(); + +$tmp_file = "$tmp_dir/count_cache.tmp"; + +$fh_out = Maasha::Filesys::file_write_open( $tmp_file ); + +$cache = 0; +$num = 0; + +while ( $record = Maasha::Biopieces::get_record( $in ) ) +{ + map { $count_hash{ $_ }{ $record->{ $_ } }++ if exists $record->{ $_ } } @{ $options->{ "keys" } }; + + push @records, $record; + + if ( scalar @records > 5_000_000 ) # too many records to hold in memory - use disk cache + { + map { Maasha::Biopieces::put_record( $_, $fh_out ) } @records; + + undef @records; + + $cache = 1; + } + + print STDERR "verbose: records read $num\n" if ( $options->{ 'verbose' } and ( $num % 1_000_000 ) == 0 ); + + $num++; +} + +close $fh_out; + +if ( $cache ) +{ + $num = 0; + + $fh_in = Maasha::Common::read_open( $tmp_file ); + + while ( $record = Maasha::Biopieces::get_record( $fh_in ) ) + { + map { $record->{ $_ . "_COUNT" } = $count_hash{ $_ }{ $record->{ $_ } } if exists $record->{ $_ } } @{ $options->{ "keys" } }; + + Maasha::Biopieces::put_record( $record, $out ); + + print STDERR "verbose: cache read $num\n" if ( $options->{ 'verbose' } and ( $num % 1_000_000 ) == 0 ); + + $num++; + } + + close $fh_in; +} + +foreach $record ( @records ) +{ + map { $record->{ $_ . "_COUNT" } = $count_hash{ $_ }{ $record->{ $_ } } if exists $record->{ $_ } } @{ $options->{ "keys" } }; + + Maasha::Biopieces::put_record( $record, $out ); +} + +unlink $tmp_file; + +Maasha::Filesys::dir_remove( $tmp_dir ); + + +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + + +BEGIN +{ + $run_time_beg = Maasha::Biopieces::run_time(); + + Maasha::Biopieces::log_biopiece(); +} + +END +{ + Maasha::Biopieces::close_stream( $in ); + Maasha::Biopieces::close_stream( $out ); + + $run_time_end = Maasha::Biopieces::run_time(); + + Maasha::Biopieces::run_time_print( $run_time_beg, $run_time_end, $options ); +} + + +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + -use Maasha::BioRun; +__END__ diff --git a/code_perl/Maasha/BioRun.pm b/code_perl/Maasha/BioRun.pm index a43ce9c..b9049f2 100644 --- a/code_perl/Maasha/BioRun.pm +++ b/code_perl/Maasha/BioRun.pm @@ -163,10 +163,8 @@ sub run_script elsif ( $script eq "uniq_vals" ) { script_uniq_vals( $in, $out, $options ) } elsif ( $script eq "merge_vals" ) { script_merge_vals( $in, $out, $options ) } elsif ( $script eq "merge_records" ) { script_merge_records( $in, $out, $options ) } - elsif ( $script eq "compute" ) { script_compute( $in, $out, $options ) } elsif ( $script eq "flip_tab" ) { script_flip_tab( $in, $out, $options ) } elsif ( $script eq "sort_records" ) { script_sort_records( $in, $out, $options ) } - elsif ( $script eq "count_vals" ) { script_count_vals( $in, $out, $options ) } elsif ( $script eq "plot_histogram" ) { script_plot_histogram( $in, $out, $options ) } elsif ( $script eq "plot_lendist" ) { script_plot_lendist( $in, $out, $options ) } elsif ( $script eq "plot_chrdist" ) { script_plot_chrdist( $in, $out, $options ) } @@ -583,12 +581,6 @@ sub get_options merge|m=s ); } - elsif ( $script eq "compute" ) - { - @options = qw( - eval|e=s - ); - } elsif ( $script eq "sort_records" ) { @options = qw( @@ -596,12 +588,6 @@ sub get_options keys|k=s ); } - elsif ( $script eq "count_vals" ) - { - @options = qw( - keys|k=s - ); - } elsif ( $script eq "plot_histogram" ) { @options = qw( @@ -831,7 +817,7 @@ sub get_options Maasha::Common::error( qq(both --in_file and --genome specified) ) if $script eq "soap_seq" and $options{ "genome" } and $options{ "in_file" }; Maasha::Common::error( qq(no --genome specified) ) if $script =~ /get_genome_align|get_genome_phastcons|plot_phastcons_profiles|plot_karyogram/ and not $options{ "genome" }; Maasha::Common::error( qq(no --key specified) ) if $script =~ /plot_lendist|plot_histogram/ and not $options{ "key" }; - Maasha::Common::error( qq(no --keys speficied) ) if $script =~ /sort_records|count_vals|sum_vals|mean_vals|median_vals|length_vals/ and not $options{ "keys" }; + Maasha::Common::error( qq(no --keys speficied) ) if $script =~ /sort_records|sum_vals|mean_vals|median_vals|length_vals/ and not $options{ "keys" }; if ( $script eq "upload_to_ucsc" ) { @@ -3332,53 +3318,6 @@ sub script_merge_records } -sub script_compute -{ - # Martin A. Hansen, August 2007. - - # Evaluate extression for records in stream. - - my ( $in, # handle to in stream - $out, # handle to out stream - $options, # options hash - ) = @_; - - # Returns nothing. - - my ( $record, $eval_key, @keys, $eval_val ); - - while ( $record = Maasha::Biopieces::get_record( $in ) ) - { - if ( $options->{ "eval" } ) - { - if ( $options->{ "eval" } =~ /^(\S+)\s*=\s*(.+)$/ ) - { - $eval_key = $1; - $eval_val = $2; - - if ( not @keys ) - { - @keys = split /\s+|\+|-|\*|\/|\*\*/, $eval_val; - - @keys = grep { exists $record->{ $_ } } @keys; - } - - map { $eval_val =~ s/\Q$_\E/$record->{ $_ }/g } @keys; - - $record->{ $eval_key } = eval "$eval_val"; - Maasha::Common::error( qq(eval "$eval_key = $eval_val" failed -> $@) ) if $@; - } - else - { - Maasha::Common::error( qq(Bad compute expression: "$options->{ 'eval' }"\n) ); - } - } - - Maasha::Biopieces::put_record( $record, $out ); - } -} - - sub script_flip_tab { # Martin A. Hansen, June 2008. @@ -3470,81 +3409,6 @@ sub script_sort_records } -sub script_count_vals -{ - # Martin A. Hansen, August 2007. - - # Count records in stream. - - my ( $in, # handle to in stream - $out, # handle to out stream - $options, # options hash - ) = @_; - - # Returns nothing. - - my ( $num, $record, %count_hash, @records, $tmp_file, $fh_out, $fh_in, $cache ); - - $tmp_file = "$BP_TMP/count_cache.tmp"; - - $fh_out = Maasha::Common::write_open( $tmp_file ); - - $cache = 0; - $num = 0; - - while ( $record = Maasha::Biopieces::get_record( $in ) ) - { - map { $count_hash{ $_ }{ $record->{ $_ } }++ if exists $record->{ $_ } } @{ $options->{ "keys" } }; - - push @records, $record; - - if ( scalar @records > 5_000_000 ) # too many records to hold in memory - use disk cache - { - map { Maasha::Biopieces::put_record( $_, $fh_out ) } @records; - - undef @records; - - $cache = 1; - } - - print STDERR "verbose: records read $num\n" if ( $options->{ 'verbose' } and ( $num % 1_000_000 ) == 0 ); - - $num++; - } - - close $fh_out; - - if ( $cache ) - { - $num = 0; - - $fh_in = Maasha::Common::read_open( $tmp_file ); - - while ( $record = Maasha::Biopieces::get_record( $fh_in ) ) - { - map { $record->{ $_ . "_COUNT" } = $count_hash{ $_ }{ $record->{ $_ } } if exists $record->{ $_ } } @{ $options->{ "keys" } }; - - Maasha::Biopieces::put_record( $record, $out ); - - print STDERR "verbose: cache read $num\n" if ( $options->{ 'verbose' } and ( $num % 1_000_000 ) == 0 ); - - $num++; - } - - close $fh_in; - } - - foreach $record ( @records ) - { - map { $record->{ $_ . "_COUNT" } = $count_hash{ $_ }{ $record->{ $_ } } if exists $record->{ $_ } } @{ $options->{ "keys" } }; - - Maasha::Biopieces::put_record( $record, $out ); - } - - unlink $tmp_file; -} - - sub script_plot_histogram { # Martin A. Hansen, September 2007. -- 2.39.5