3 # Copyright (C) 2007-2009 Martin A. Hansen.
5 # This program is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU General Public License
7 # as published by the Free Software Foundation; either version 2
8 # of the License, or (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program; if not, write to the Free Software
17 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 # http://www.gnu.org/copyleft/gpl.html
22 # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> DESCRIPTION <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
24 # Count the number of times values of given keys exists in stream.
26 # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
32 use Maasha::Biopieces;
35 # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
38 my ( $options, $in, $out, $num, $record, %count_hash, @records, $tmp_dir, $tmp_file, $fh_out, $fh_in, $cache );
40 $options = Maasha::Biopieces::parse_options(
42 { long => 'keys', short => 'k', type => 'list', mandatory => 'yes', default => undef, allowed => undef, disallowed => undef },
46 $in = Maasha::Biopieces::read_stream( $options->{ "stream_in" } );
47 $out = Maasha::Biopieces::write_stream( $options->{ "stream_out" } );
49 $tmp_dir = Maasha::Biopieces::get_tmpdir();
51 $tmp_file = "$tmp_dir/count_cache.tmp";
53 $fh_out = Maasha::Filesys::file_write_open( $tmp_file );
58 while ( $record = Maasha::Biopieces::get_record( $in ) )
60 map { $count_hash{ $_ }{ $record->{ $_ } }++ if exists $record->{ $_ } } @{ $options->{ "keys" } };
62 push @records, $record;
64 if ( scalar @records > 5_000_000 ) # too many records to hold in memory - use disk cache
66 map { Maasha::Biopieces::put_record( $_, $fh_out ) } @records;
73 print STDERR "verbose: records read $num\n" if ( $options->{ 'verbose' } and ( $num % 1_000_000 ) == 0 );
84 $fh_in = Maasha::Common::read_open( $tmp_file );
86 while ( $record = Maasha::Biopieces::get_record( $fh_in ) )
88 map { $record->{ $_ . "_COUNT" } = $count_hash{ $_ }{ $record->{ $_ } } if exists $record->{ $_ } } @{ $options->{ "keys" } };
90 Maasha::Biopieces::put_record( $record, $out );
92 print STDERR "verbose: cache read $num\n" if ( $options->{ 'verbose' } and ( $num % 1_000_000 ) == 0 );
100 foreach $record ( @records )
102 map { $record->{ $_ . "_COUNT" } = $count_hash{ $_ }{ $record->{ $_ } } if exists $record->{ $_ } } @{ $options->{ "keys" } };
104 Maasha::Biopieces::put_record( $record, $out );
109 Maasha::Biopieces::close_stream( $in );
110 Maasha::Biopieces::close_stream( $out );
113 # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
118 Maasha::Biopieces::status_set();
124 Maasha::Biopieces::status_log();
128 # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<