#include <assert.h>
#include <string.h>
#include <ctype.h>
+#include <math.h>
#include "faidx.h"
#include "sam.h"
#include "kstring.h"
bam_fillmd1_core(b, ref, is_equal, 0);
}
+int bam_cap_mapQ(bam1_t *b, char *ref)
+{
+ uint8_t *seq = bam1_seq(b), *qual = bam1_qual(b);
+ uint32_t *cigar = bam1_cigar(b);
+ bam1_core_t *c = &b->core;
+ int i, x, y, mm, q, len, clip_l, clip_q;
+ double t;
+
+ mm = q = len = clip_l = clip_q = 0;
+ for (i = y = 0, x = c->pos; i < c->n_cigar; ++i) {
+ int j, l = cigar[i]>>4, op = cigar[i]&0xf;
+ if (op == BAM_CMATCH) {
+ for (j = 0; j < l; ++j) {
+ int z = y + j;
+ int c1 = bam1_seqi(seq, z), c2 = bam_nt16_table[(int)ref[x+j]];
+ if (ref[x+j] == 0) break; // out of boundary
+ if (c2 != 15 && c1 != 15 && qual[z] >= 13) { // not ambiguous
+ ++len;
+ if (c1 && c1 != c2 && qual[z] >= 13) { // mismatch
+ ++mm;
+ q += qual[z] > 33? 33 : qual[z];
+ }
+ }
+ }
+ if (j < l) break;
+ x += l; y += l; len += l;
+ } else if (op == BAM_CDEL) {
+ for (j = 0; j < l; ++j)
+ if (ref[x+j] == 0) break;
+ if (j < l) break;
+ x += l;
+ } else if (op == BAM_CSOFT_CLIP) {
+ for (j = 0; j < l; ++j) clip_q += qual[y+j];
+ clip_l += l;
+ y += l;
+ } else if (op == BAM_CHARD_CLIP) {
+ clip_q += 13 * l;
+ clip_l += l;
+ } else if (op == BAM_CINS) y += l;
+ else if (op == BAM_CREF_SKIP) x += l;
+ }
+ for (i = 0, t = 1; i < mm; ++i)
+ t *= (double)len / (i+1);
+ t = q - 4.343 * log(t) + clip_q / 5.;
+ if (t > 40) return -1;
+ if (t < 0) t = 0;
+ t = sqrt((40 - t) / 40) * 40;
+ fprintf(stderr, "%s %lf %d\n", bam1_qname(b), t, q);
+ return (int)(t + .499);
+}
+
// local realignment
#define MIN_REF_LEN 10
int bam_realn(bam1_t *b, const char *ref)
{
- int k, l_ref, score, n_cigar;
+ int k, score, q[2], r[2], kk[2], kl[2], x, y, max, j, n_cigar, endx = -1;
uint32_t *cigar = bam1_cigar(b);
- uint8_t *s_ref = 0, *s_read = 0, *seq;
- ka_param_t par;
+ uint8_t *seq = bam1_seq(b);
bam1_core_t *c = &b->core;
- // set S/W parameters
- par = ka_param_blast;
- par.gap_open = 4; par.gap_ext = 1; par.gap_end_open = par.gap_end_ext = 0;
- if (c->n_cigar > 1) { // set band width
- int sumi, sumd;
- sumi = sumd = 0;
- for (k = 0; k < c->n_cigar; ++k)
- if ((cigar[k]&0xf) == BAM_CINS) sumi += cigar[k]>>4;
- else if ((cigar[k]&0xf) == BAM_CDEL) sumd += cigar[k]>>4;
- par.band_width = (sumi > sumd? sumi : sumd) + MIN_BAND_WIDTH;
- } else par.band_width = MIN_BAND_WIDTH;
- // calculate the length of the reference in the alignment
- for (k = l_ref = 0; k < c->n_cigar; ++k) {
- if ((cigar[k]&0xf) == BAM_CREF_SKIP) break; // do not do realignment if there is an `N' operation
- if ((cigar[k]&0xf) == BAM_CMATCH || (cigar[k]&0xf) == BAM_CDEL)
- l_ref += cigar[k]>>4;
+ q[0] = q[1] = r[0] = r[1] = kk[0] = kk[1] = kl[0] = kl[1] = -1;
+ // find the right boundary
+ for (k = 0, score = max = 0, x = c->pos, y = 0; k < c->n_cigar; ++k) {
+ int op = cigar[k]&0xf;
+ int ol = cigar[k]>>4;
+ if (op == BAM_CMATCH) {
+ for (j = 0; j < ol; ++j) {
+ int c1, c2, z = y + j;
+ c1 = bam_nt16_nt4_table[bam1_seqi(seq, z)];
+ if (ref[x+j] == 0) return -1;
+ c2 = bam_nt16_nt4_table[(int)bam_nt16_table[(int)ref[x+j]]];
+ if (c1 < 3 && c2 < 3)
+ score += c1 == c2? 5 : -4;
+ if (score < 0) score = 0;
+ if (score > max) max = score, q[1] = z, r[1] = x+j, kk[1] = k, kl[1] = j + 1;
+ }
+ x += ol; y += ol;
+ } else if (op == BAM_CINS) {
+ score -= 4 - ol * 3;
+ y += ol;
+ if (score < 0) score = 0;
+ } else if (op == BAM_CDEL) {
+ score -= 4 - ol * 3;
+ x += ol;
+ if (score < 0) score = 0;
+ } else if (op == BAM_CSOFT_CLIP) y += ol;
+ else if (op == BAM_CREF_SKIP) x += ol;
+ }
+ if (score < 0) return -1; // no high scoring segments
+ endx = x - 1;
+ // find the left boundary
+ for (k = c->n_cigar - 1, score = max = 0, x = x-1, y = y-1; k >= 0; --k) {
+ int op = cigar[k]&0xf;
+ int ol = cigar[k]>>4;
+ if (op == BAM_CMATCH) {
+ for (j = 0; j < ol; ++j) {
+ int c1, c2, z = y - j;
+ c1 = bam_nt16_nt4_table[bam1_seqi(seq, z)];
+ if (ref[x+j] == 0) return -1;
+ c2 = bam_nt16_nt4_table[(int)bam_nt16_table[(int)ref[x-j]]];
+ if (c1 < 3 && c2 < 3)
+ score += c1 == c2? 5 : -4;
+ if (score < 0) score = 0;
+ if (score > max) max = score, q[0] = z, r[0] = x-j, kk[0] = k, kl[0] = j + 1;
+ }
+ x -= ol; y -= ol;
+ } else if (op == BAM_CINS) {
+ score -= 4 - ol * 3;
+ y -= ol;
+ if (score < 0) score = 0;
+ } else if (op == BAM_CDEL) {
+ score -= 4 - ol * 3;
+ x -= ol;
+ if (score < 0) score = 0;
+ } else if (op == BAM_CSOFT_CLIP) y -= ol;
+ else if (op == BAM_CREF_SKIP) x -= ol;
}
- if (k != c->n_cigar || l_ref < MIN_REF_LEN) return -1;
- for (k = 0; k < l_ref; ++k)
- if (ref[c->pos + k] == 0) return -1; // the read stands out of the reference
- // allocate
- s_ref = calloc(l_ref, 1);
- s_read = calloc(c->l_qseq, 1);
- for (k = 0, seq = bam1_seq(b); k < c->l_qseq; ++k)
- s_read[k] = bam_nt16_nt4_table[bam1_seqi(seq, k)];
- for (k = 0; k < l_ref; ++k)
- s_ref[k] = bam_nt16_nt4_table[(int)bam_nt16_table[(int)ref[c->pos + k]]];
- // do alignment
- cigar = ka_global_core(s_ref, l_ref, s_read, c->l_qseq, &par, &score, &n_cigar);
- if (score <= 0) { // realignment failed
- free(cigar); free(s_ref); free(s_read);
- return -1;
+ if (q[1] - q[0] < 15) return -1; // the high-scoring segment is too short
+ // modify CIGAR
+ n_cigar = 0;
+ cigar = calloc(c->n_cigar + 4, 4);
+ if (q[0] != 0) cigar[n_cigar++] = (uint32_t)q[0]<<4 | BAM_CSOFT_CLIP;
+ if (r[0] != c->pos) cigar[n_cigar++] = (uint32_t)(r[0] - c->pos)<<4 | BAM_CREF_SKIP;
+ if (kk[0] == kk[1]) {
+ cigar[n_cigar++] = (uint32_t)(kl[0] + kl[1] - (bam1_cigar(b)[kk[0]]>>4))<<4 | BAM_CMATCH;
+ } else {
+ cigar[n_cigar++] = (uint32_t)kl[0]<<4 | BAM_CMATCH;
+ for (k = kk[0] + 1; k < kk[1]; ++k)
+ cigar[n_cigar++] = bam1_cigar(b)[k];
+ cigar[n_cigar++] = (uint32_t)kl[1]<<4 | BAM_CMATCH; // FIXME: add ref_skip after this line
}
+ if (q[1] + 1 != c->l_qseq)
+ cigar[n_cigar++] = (uint32_t)(c->l_qseq - q[1] - 1)<<4 | BAM_CSOFT_CLIP;
// copy over the alignment
if (4 * (n_cigar - (int)c->n_cigar) + b->data_len > b->m_data) { // enlarge b->data
b->m_data = 4 * (n_cigar - (int)c->n_cigar) + b->data_len;
}
memcpy(bam1_cigar(b), cigar, n_cigar * 4);
c->n_cigar = n_cigar;
- free(s_ref); free(s_read); free(cigar);
+ free(cigar);
return 0;
}
int bam_fillmd(int argc, char *argv[])
{
- int c, is_equal = 0, tid = -2, ret, len, is_bam_out, is_sam_in, is_uncompressed, max_nm = 0, is_realn;
+ int c, is_equal = 0, tid = -2, ret, len, is_bam_out, is_sam_in, is_uncompressed, max_nm = 0, is_realn, is_capQ;
samfile_t *fp, *fpout = 0;
faidx_t *fai;
char *ref = 0, mode_w[8], mode_r[8];
bam1_t *b;
- is_bam_out = is_sam_in = is_uncompressed = is_realn = 0;
+ is_bam_out = is_sam_in = is_uncompressed = is_realn = is_capQ = 0;
mode_w[0] = mode_r[0] = 0;
strcpy(mode_r, "r"); strcpy(mode_w, "w");
- while ((c = getopt(argc, argv, "reubSn:")) >= 0) {
+ while ((c = getopt(argc, argv, "reubSCn:")) >= 0) {
switch (c) {
case 'r': is_realn = 1; break;
case 'e': is_equal = 1; break;
case 'u': is_uncompressed = is_bam_out = 1; break;
case 'S': is_sam_in = 1; break;
case 'n': max_nm = atoi(optarg); break;
+ case 'C': is_capQ = 1; break;
default: fprintf(stderr, "[bam_fillmd] unrecognized option '-%c'\n", c); return 1;
}
}
fp->header->target_name[tid]);
}
if (is_realn) bam_realn(b, ref);
+ if (is_capQ) {
+ int q = bam_cap_mapQ(b, ref);
+ if (b->core.qual > q) b->core.qual = q;
+ }
if (ref) bam_fillmd1_core(b, ref, is_equal, max_nm);
}
samwrite(fpout, b);
#define MPLP_NO_COMP 0x20
#define MPLP_NO_ORPHAN 0x40
#define MPLP_REALN 0x80
-#define MPLP_NO_HALFTRIM 0x100
+#define MPLP_CAPQ 0x100
typedef struct {
int max_mq, min_mq, flag, min_baseQ;
typedef struct {
bamFile fp;
bam_iter_t iter;
- int min_mq, flag;
+ int min_mq, flag, ref_id;
char *ref;
} mplp_aux_t;
static int mplp_func(void *data, bam1_t *b)
{
extern int bam_realn(bam1_t *b, const char *ref);
+ extern int bam_cap_mapQ(bam1_t *b, char *ref);
mplp_aux_t *ma = (mplp_aux_t*)data;
- int ret, cond = 0;
+ int ret, skip = 0;
do {
- cond = 0;
+ int has_ref = (ma->ref && ma->ref_id == b->core.tid)? 1 : 0;
ret = ma->iter? bam_iter_read(ma->fp, ma->iter, b) : bam_read1(ma->fp, b);
if (ret < 0) break;
- if (b->core.flag&BAM_FUNMAP) cond = 1;
- else if (b->core.qual < ma->min_mq) cond = 1;
- else if ((ma->flag&MPLP_NO_ORPHAN) && (b->core.flag&1) && !(b->core.flag&2)) cond = 1;
- if (ma->ref && !cond && (ma->flag&MPLP_REALN)) bam_realn(b, ma->ref);
- } while (cond);
+ skip = 0;
+ if (has_ref && (ma->flag&MPLP_REALN)) bam_realn(b, ma->ref);
+ if ((ma->flag&MPLP_CAPQ) && has_ref) {
+ int q = bam_cap_mapQ(b, ma->ref);
+ if (q < 0) skip = 1;
+ else if (b->core.qual > q) b->core.qual = q;
+ } else if (b->core.flag&BAM_FUNMAP) skip = 1;
+ else if (b->core.qual < ma->min_mq) skip = 1;
+ else if ((ma->flag&MPLP_NO_ORPHAN) && (b->core.flag&1) && !(b->core.flag&2)) skip = 1;
+ } while (skip);
return ret;
}
if (k == kh_end(hash)) continue;
}
if (tid != ref_tid) {
- free(ref);
+ free(ref); ref = 0;
if (conf->fai) ref = fai_fetch(conf->fai, h->target_name[tid], &ref_len);
- for (i = 0; i < n; ++i) data[i]->ref = ref;
+ for (i = 0; i < n; ++i) data[i]->ref = ref, data[i]->ref_id = tid;
ref_tid = tid;
}
if (conf->flag & MPLP_GLF) {
mplp.max_mq = 60;
mplp.theta = 1e-3;
mplp.min_baseQ = 13;
- while ((c = getopt(argc, argv, "gf:r:l:M:q:t:Q:uaORH")) >= 0) {
+ while ((c = getopt(argc, argv, "gf:r:l:M:q:t:Q:uaORC")) >= 0) {
switch (c) {
case 't': mplp.theta = atof(optarg); break;
case 'f':
case 'l': mplp.fn_pos = strdup(optarg); break;
case 'g': mplp.flag |= MPLP_GLF; break;
case 'u': mplp.flag |= MPLP_NO_COMP; break;
- case 'a': mplp.flag |= MPLP_NO_ORPHAN | MPLP_REALN | MPLP_NO_HALFTRIM; break;
+ case 'a': mplp.flag |= MPLP_NO_ORPHAN | MPLP_REALN; break;
case 'O': mplp.flag |= MPLP_NO_ORPHAN; break;
- case 'H': mplp.flag |= MPLP_NO_HALFTRIM; break;
+ case 'C': mplp.flag |= MPLP_CAPQ; break;
case 'R': mplp.flag |= MPLP_REALN; break;
case 'M': mplp.max_mq = atoi(optarg); break;
case 'q': mplp.min_mq = atoi(optarg); break;
my $version = '0.1.0';
&usage if (@ARGV < 1);
my $command = shift(@ARGV);
- my %func = (subsam=>\&subsam, listsam=>\&listsam, fillac=>\&fillac, qstats=>\&qstats);
+ my %func = (subsam=>\&subsam, listsam=>\&listsam, fillac=>\&fillac, qstats=>\&qstats, varFilter=>\&varFilter);
die("Unknown command \"$command\".\n") if (!defined($func{$command}));
&{$func{$command}};
}
next if (/^#/);
my @t = split;
next if (length($t[3]) != 1 || uc($t[3]) eq 'N');
+ $t[3] = uc($t[3]); $t[4] = uc($t[4]);
my @s = split(',', $t[4]);
$t[5] = 3 if ($t[5] < 0);
- $t[3] = uc($t[3]); $t[4] = uc($t[4]);
next if (length($s[0]) != 1);
push(@a, [$t[5], ($t[4] eq '.' || $t[4] eq $t[3])? 0 : 1, $ts{$t[3].$s[0]}? 1 : 0, $h{$t[0],$t[1]}? 1 : 0]);
}
}
}
+sub varFilter {
+ my %opts = (d=>1, D=>10000, l=>30, Q=>25, q=>10, G=>25, s=>100, w=>10, W=>10, N=>2, p=>undef, F=>.001);
+ getopts('pq:d:D:l:Q:w:W:N:G:F:', \%opts);
+ die(qq/
+Usage: vcfutils.pl varFilter [options] <in.vcf>
+
+Options: -Q INT minimum RMS mapping quality for SNPs [$opts{Q}]
+ -q INT minimum RMS mapping quality for gaps [$opts{q}]
+ -d INT minimum read depth [$opts{d}]
+ -D INT maximum read depth [$opts{D}]
+
+ -G INT min indel score for nearby SNP filtering [$opts{G}]
+ -w INT SNP within INT bp around a gap to be filtered [$opts{w}]
+
+ -W INT window size for filtering dense SNPs [$opts{W}]
+ -N INT max number of SNPs in a window [$opts{N}]
+
+ -l INT window size for filtering adjacent gaps [$opts{l}]
+
+ -p print filtered variants
+\n/) if (@ARGV == 0 && -t STDIN);
+
+ # calculate the window size
+ my ($ol, $ow, $oW) = ($opts{l}, $opts{w}, $opts{W});
+ my $max_dist = $ol > $ow? $ol : $ow;
+ $max_dist = $oW if ($max_dist < $oW);
+ # the core loop
+ my @staging; # (indel_filtering_score, flt_tag)
+ while (<>) {
+ my @t = split;
+ next if (/^#/);
+ next if ($t[4] eq '.'); # skip non-var sites
+ my $is_snp = 1;
+ if (length($t[3]) > 1) {
+ $is_snp = 0;
+ } else {
+ my @s = split(',', $t[4]);
+ for (@s) {
+ $is_snp = 0 if (length > 1);
+ }
+ }
+ # clear the out-of-range elements
+ while (@staging) {
+ # Still on the same chromosome and the first element's window still affects this position?
+ last if ($staging[0][3] eq $t[0] && $staging[0][4] + $staging[0][2] + $max_dist >= $t[1]);
+ varFilter_aux(shift(@staging), $opts{p}); # calling a function is a bit slower, not much
+ }
+ my ($flt, $score) = (0, -1);
+
+ # collect key annotations
+ my ($dp, $mq, $af) = (-1, -1, 1);
+ if ($t[7] =~ /DP=(\d+)/i) {
+ $dp = $1;
+ } elsif ($t[7] =~ /DP4=(\d+),(\d+),(\d+),(\d+)/i) {
+ $dp = $1 + $2 + $3 + $4;
+ }
+ if ($t[7] =~ /MQ=(\d+)/i) {
+ $mq = $1;
+ }
+ if ($t[7] =~ /AF=([^\s;=]+)/i) {
+ $af = $1;
+ } elsif ($t[7] =~ /AF1=([^\s;=]+)/i) {
+ $af = $1;
+ }
+ # the depth filter
+ if ($dp >= 0) {
+ if ($dp < $opts{d}) {
+ $flt = 2;
+ } elsif ($dp > $opts{D}) {
+ $flt = 3;
+ }
+ }
+
+ # site dependent filters
+ my $dlen = 0;
+ if ($flt == 0) {
+ if (!$is_snp) { # an indel
+ # If deletion, remember the length of the deletion
+ $dlen = length($t[3]) - 1;
+ $flt = 1 if ($mq < $opts{q});
+ # filtering SNPs
+ if ($t[5] >= $opts{G}) {
+ for my $x (@staging) {
+ # Is it a SNP and is it outside the SNP filter window?
+ next if ($x->[0] >= 0 || $x->[4] + $x->[2] + $ow < $t[1]);
+ $x->[1] = 5 if ($x->[1] == 0);
+ }
+ }
+ # the indel filtering score
+ $score = $t[5];
+ # check the staging list for indel filtering
+ for my $x (@staging) {
+ # Is it a SNP and is it outside the gap filter window
+ next if ($x->[0] < 0 || $x->[4] + $x->[2] + $ol < $t[1]);
+ if ($x->[0] < $score) {
+ $x->[1] = 6;
+ } else {
+ $flt = 6; last;
+ }
+ }
+ } else { # a SNP
+ $flt = 1 if ($mq < $opts{Q});
+ # check adjacent SNPs
+ my $k = 1;
+ for my $x (@staging) {
+ ++$k if ($x->[0] < 0 && -($x->[0] + 1) > $opts{F} && $x->[4] + $x->[2] + $oW >= $t[1] && ($x->[1] == 0 || $x->[1] == 4 || $x->[1] == 5));
+ }
+ # filtering is necessary
+ if ($k > $opts{N}) {
+ $flt = 4;
+ for my $x (@staging) {
+ $x->[1] = 4 if ($x->[0] < 0 && $x->[4] + $x->[2] + $oW >= $t[1] && $x->[1] == 0);
+ }
+ } else { # then check gap filter
+ for my $x (@staging) {
+ next if ($x->[0] < 0 || $x->[4] + $x->[2] + $ow < $t[1]);
+ if ($x->[0] >= $opts{G}) {
+ $flt = 5; last;
+ }
+ }
+ }
+ }
+ }
+ push(@staging, [$score < 0? -$af-1 : $score, $flt, $dlen, @t]);
+ }
+ # output the last few elements in the staging list
+ while (@staging) {
+ varFilter_aux(shift @staging, $opts{p});
+ }
+}
+
+sub varFilter_aux {
+ my ($first, $is_print) = @_;
+ if ($first->[1] == 0) {
+ print join("\t", @$first[3 .. @$first-1]), "\n";
+ } elsif ($is_print) {
+ print STDERR join("\t", substr("UQdDWGgsiX", $first->[1], 1), @$first[3 .. @$first-1]), "\n";
+ }
+}
+
sub usage {
die(qq/
Usage: vcfutils.pl <command> [<arguments>]\n
listsam list the samples
fillac fill the allele count field
qstats SNP stats stratified by QUAL
+ varFilter filtering short variants
\n/);
}
projects / samtools.git / commitdiff