2 * This file is part of fastq-tools.
4 * Copyright (c) 2011 by Daniel C. Jones <dcjones@cs.washington.edu>
7 * Sample reads with or without replacement from a FASTQ file.
22 static const char* prog_name = "fastq-sample";
24 static int replacement_flag;
30 "fastq-sample [OPTION]... FILE [FILE2]\n"
31 "Sample random reads from a FASTQ file."
33 " -n N the number of reads to sample (default: 10000)\n"
34 " -p N the proportion of the total reads to sample\n"
35 " -o, --output=PREFIX output file prefix\n (Default: \"sample\")"
36 " -c, --complement-output=PREFIX\n"
37 " output reads not included in the random sample to\n"
38 " a file (or files) with the given prefix (by default,\n"
39 " they are not output).\n"
40 " -r, --with-replacement sample with relpacement\n"
41 " -s, --seed=SEED a manual seed to the random number generator\n"
42 " -h, --help print this message\n"
43 " -V, --version output version information and exit\n"
48 /* count the number of entries in a fastq file */
49 unsigned long count_entries(fastq_t* fqf)
51 seq_t* seq = seq_create();
53 while (fastq_read(fqf, seq)) ++n;
60 /* compare two unsigned integers (for qsort) */
61 int cmpul( const void* a, const void* b ) {
62 if( *(unsigned long*) a < *(unsigned long*) b ) return -1;
63 else if( *(unsigned long*) a > *(unsigned long*) b ) return 1;
68 /* randomly shuffle an array of unsigned longs */
69 void shuffle(rng_t* rng, unsigned long* xs, unsigned long n)
71 unsigned long i, j, k;
72 for (i = n - 1; i > 0; --i) {
73 j = fastq_rng_uniform_int(rng, i + 1);
74 k = xs[j]; xs[j] = xs[i]; xs[i] = k; // swap
79 unsigned long* index_with_replacement(rng_t* rng, unsigned long n, unsigned long k)
81 unsigned long* xs = malloc_or_die(k * sizeof(unsigned long));
83 for (i = 0; i < k; ++i) xs[i] = fastq_rng_uniform_int(rng, n);
88 unsigned long* index_without_replacement(rng_t* rng, unsigned long n)
90 unsigned long* xs = malloc_or_die(n * sizeof(unsigned long));
92 for (i = 0; i < n; ++i) xs[i] = i;
98 void fastq_sample(unsigned long rng_seed,
99 const char* prefix, const char* cprefix,
100 FILE* file1, FILE* file2, unsigned long k, double p)
103 * The basic idea is this:
105 * 1. Count the number of lines in the file, n.
107 * 2a. If sampling with replacement, generate k random integers in [0, n-1].
109 * 2b. If sampling without replacement, generate a list of integers 0..(n-1),
110 * shuffle with fisher-yates, then consider the first k.
112 * 3. Sort the integer list.
114 * 3. Read through the file again, when the number at the front of the integer
115 * list matches the index of the fastq etry, print the entry, and pop the
122 fastq_t* f1 = fastq_create(file1);
123 fastq_t* f2 = file2 == NULL ? NULL : fastq_create(file2);
125 n = count_entries(f1);
127 n2 = count_entries(f2);
129 fprintf(stderr, "Input files have differing numbers of entries (%lu != %lu).\n", n, n2);
135 if (f2 != NULL) fastq_rewind(f2);
138 k = (unsigned long) round(p * (double) n);
139 if (!replacement_flag && k > n) k = n;
142 rng_t* rng = fastq_rng_alloc();
143 fastq_rng_seed(rng, rng_seed);
146 if (replacement_flag) xs = index_with_replacement(rng, n, k);
147 else xs = index_without_replacement(rng, n);
149 qsort(xs, k, sizeof(unsigned long), cmpul);
158 output_len = strlen(prefix) + 7;
159 output_name = malloc_or_die((output_len + 1) * sizeof(char));
161 snprintf(output_name, output_len, "%s.fastq", prefix);
162 fout1 = open_without_clobber(output_name);
164 fprintf(stderr, "Cannot open file %s for writing.\n", output_name);
173 output_len = strlen(prefix) + 9;
174 output_name = malloc_or_die((output_len + 1) * sizeof(char));
176 snprintf(output_name, output_len, "%s.1.fastq", prefix);
177 fout1 = open_without_clobber(output_name);
179 fprintf(stderr, "Cannot open file %s for writing.\n", output_name);
183 snprintf(output_name, output_len, "%s.2.fastq", prefix);
184 fout1 = open_without_clobber(output_name);
186 fprintf(stderr, "Cannot open file %s for writing.\n", output_name);
193 /* open complement output */
197 if (cprefix != NULL && file2 == NULL) {
198 output_len = strlen(cprefix) + 7;
199 output_name = malloc_or_die((output_len + 1) * sizeof(char));
201 snprintf(output_name, output_len, "%s.fastq", cprefix);
202 cfout1 = fopen(output_name, "wb");
203 if (cfout1 == NULL) {
204 fprintf(stderr, "Cannot open file %s for writing.\n", output_name);
212 else if (cprefix != NULL) {
213 output_len = strlen(cprefix) + 9;
214 output_name = malloc_or_die((output_len + 1) * sizeof(char));
216 snprintf(output_name, output_len, "%s.1.fastq", cprefix);
217 cfout1 = fopen(output_name, "wb");
218 if (cfout1 == NULL) {
219 fprintf(stderr, "Cannot open file %s for writing.\n", output_name);
223 snprintf(output_name, output_len, "%s.2.fastq", cprefix);
224 cfout2 = fopen(output_name, "wb");
225 if (cfout1 == NULL) {
226 fprintf(stderr, "Cannot open file %s for writing.\n", output_name);
233 unsigned long i = 0; // read number
234 unsigned long j = 0; // index into xs
237 seq_t* seq1 = seq_create();
238 seq_t* seq2 = seq_create();
240 while (j < k && fastq_read(f1, seq1)) {
242 ret = fastq_read(f2, seq2);
244 fputs("Input files have differing numbers of entries.\n", stderr);
250 while (j < k && xs[j] == i) {
251 fastq_print(fout1, seq1);
252 if (f2 != NULL) fastq_print(fout2, seq2);
256 else if (cfout1 != NULL) {
257 fastq_print(cfout1, seq1);
258 if (f2 != NULL) fastq_print(cfout2, seq2);
267 if (f2 != NULL) fastq_free(f2);
270 if (fout2 != NULL) fclose(fout2);
272 if (cfout1 != NULL) fclose(cfout1);
273 if (cfout2 != NULL) fclose(cfout2);
280 int main(int argc, char* argv[])
285 const char* prefix = "sample";
286 const char* cprefix = NULL;
287 unsigned long rng_seed = 4357;
288 unsigned long k = 10000; // number of reads to sample
289 double p = -1; // proportion of reads to sample
291 static struct option long_options[] =
293 {"with-replacement", no_argument, NULL, 'r'},
294 {"complement-output", required_argument, NULL, 'c'},
295 {"seed", required_argument, NULL, 's'},
296 {"output", no_argument, NULL, 'o'},
297 {"help", no_argument, NULL, 'h'},
298 {"version", no_argument, NULL, 'V'},
302 replacement_flag = 0;
305 opt = getopt_long(argc, argv, "n:p:o:rs:hV", long_options, &opt_idx);
307 if( opt == -1 ) break;
311 if (long_options[opt_idx].flag != 0) break;
317 k = strtoul(optarg, NULL, 10);
323 fputs("Sample proportion ('-p') is less than zero.\n", stderr);
329 replacement_flag = 1;
333 rng_seed = strtoul(optarg, NULL, 10);
349 print_version(stdout, prog_name);
363 if (optind >= argc) {
364 fputs("An input file must be given.\n", stderr);
369 file1 = fopen(argv[optind], "rb");
371 fprintf(stderr, "Cannot open '%s' for reading.\n", argv[optind]);
375 if (++optind < argc) {
376 file2 = fopen(argv[optind], "rb");
378 fprintf(stderr, "Cannot open '%s' for reading.\n", argv[optind]);
383 fastq_sample(rng_seed, prefix, cprefix, file1, file2, k, p);