From 51f014165fb63b2b9b86db80ea4681f9351453a1 Mon Sep 17 00:00:00 2001 From: Heng Li Date: Thu, 16 Sep 2010 17:38:43 +0000 Subject: [PATCH] * added varFilter to vcfutils.pl * reimplement realn(). now it performs a local alignment * added cap_mapQ() to cap mapping quality when there are many substitutions --- bam_md.c | 172 +++++++++++++++++++++++++++++++++---------- bam_plcmd.c | 34 +++++---- bcftools/vcfutils.pl | 145 +++++++++++++++++++++++++++++++++++- misc/samtools.pl | 1 - 4 files changed, 297 insertions(+), 55 deletions(-) diff --git a/bam_md.c b/bam_md.c index c6c5a65..eee767e 100644 --- a/bam_md.c +++ b/bam_md.c @@ -2,6 +2,7 @@ #include #include #include +#include #include "faidx.h" #include "sam.h" #include "kstring.h" @@ -109,6 +110,57 @@ void bam_fillmd1(bam1_t *b, char *ref, int is_equal) bam_fillmd1_core(b, ref, is_equal, 0); } +int bam_cap_mapQ(bam1_t *b, char *ref) +{ + uint8_t *seq = bam1_seq(b), *qual = bam1_qual(b); + uint32_t *cigar = bam1_cigar(b); + bam1_core_t *c = &b->core; + int i, x, y, mm, q, len, clip_l, clip_q; + double t; + + mm = q = len = clip_l = clip_q = 0; + for (i = y = 0, x = c->pos; i < c->n_cigar; ++i) { + int j, l = cigar[i]>>4, op = cigar[i]&0xf; + if (op == BAM_CMATCH) { + for (j = 0; j < l; ++j) { + int z = y + j; + int c1 = bam1_seqi(seq, z), c2 = bam_nt16_table[(int)ref[x+j]]; + if (ref[x+j] == 0) break; // out of boundary + if (c2 != 15 && c1 != 15 && qual[z] >= 13) { // not ambiguous + ++len; + if (c1 && c1 != c2 && qual[z] >= 13) { // mismatch + ++mm; + q += qual[z] > 33? 33 : qual[z]; + } + } + } + if (j < l) break; + x += l; y += l; len += l; + } else if (op == BAM_CDEL) { + for (j = 0; j < l; ++j) + if (ref[x+j] == 0) break; + if (j < l) break; + x += l; + } else if (op == BAM_CSOFT_CLIP) { + for (j = 0; j < l; ++j) clip_q += qual[y+j]; + clip_l += l; + y += l; + } else if (op == BAM_CHARD_CLIP) { + clip_q += 13 * l; + clip_l += l; + } else if (op == BAM_CINS) y += l; + else if (op == BAM_CREF_SKIP) x += l; + } + for (i = 0, t = 1; i < mm; ++i) + t *= (double)len / (i+1); + t = q - 4.343 * log(t) + clip_q / 5.; + if (t > 40) return -1; + if (t < 0) t = 0; + t = sqrt((40 - t) / 40) * 40; + fprintf(stderr, "%s %lf %d\n", bam1_qname(b), t, q); + return (int)(t + .499); +} + // local realignment #define MIN_REF_LEN 10 @@ -116,44 +168,83 @@ void bam_fillmd1(bam1_t *b, char *ref, int is_equal) int bam_realn(bam1_t *b, const char *ref) { - int k, l_ref, score, n_cigar; + int k, score, q[2], r[2], kk[2], kl[2], x, y, max, j, n_cigar, endx = -1; uint32_t *cigar = bam1_cigar(b); - uint8_t *s_ref = 0, *s_read = 0, *seq; - ka_param_t par; + uint8_t *seq = bam1_seq(b); bam1_core_t *c = &b->core; - // set S/W parameters - par = ka_param_blast; - par.gap_open = 4; par.gap_ext = 1; par.gap_end_open = par.gap_end_ext = 0; - if (c->n_cigar > 1) { // set band width - int sumi, sumd; - sumi = sumd = 0; - for (k = 0; k < c->n_cigar; ++k) - if ((cigar[k]&0xf) == BAM_CINS) sumi += cigar[k]>>4; - else if ((cigar[k]&0xf) == BAM_CDEL) sumd += cigar[k]>>4; - par.band_width = (sumi > sumd? sumi : sumd) + MIN_BAND_WIDTH; - } else par.band_width = MIN_BAND_WIDTH; - // calculate the length of the reference in the alignment - for (k = l_ref = 0; k < c->n_cigar; ++k) { - if ((cigar[k]&0xf) == BAM_CREF_SKIP) break; // do not do realignment if there is an `N' operation - if ((cigar[k]&0xf) == BAM_CMATCH || (cigar[k]&0xf) == BAM_CDEL) - l_ref += cigar[k]>>4; + q[0] = q[1] = r[0] = r[1] = kk[0] = kk[1] = kl[0] = kl[1] = -1; + // find the right boundary + for (k = 0, score = max = 0, x = c->pos, y = 0; k < c->n_cigar; ++k) { + int op = cigar[k]&0xf; + int ol = cigar[k]>>4; + if (op == BAM_CMATCH) { + for (j = 0; j < ol; ++j) { + int c1, c2, z = y + j; + c1 = bam_nt16_nt4_table[bam1_seqi(seq, z)]; + if (ref[x+j] == 0) return -1; + c2 = bam_nt16_nt4_table[(int)bam_nt16_table[(int)ref[x+j]]]; + if (c1 < 3 && c2 < 3) + score += c1 == c2? 5 : -4; + if (score < 0) score = 0; + if (score > max) max = score, q[1] = z, r[1] = x+j, kk[1] = k, kl[1] = j + 1; + } + x += ol; y += ol; + } else if (op == BAM_CINS) { + score -= 4 - ol * 3; + y += ol; + if (score < 0) score = 0; + } else if (op == BAM_CDEL) { + score -= 4 - ol * 3; + x += ol; + if (score < 0) score = 0; + } else if (op == BAM_CSOFT_CLIP) y += ol; + else if (op == BAM_CREF_SKIP) x += ol; + } + if (score < 0) return -1; // no high scoring segments + endx = x - 1; + // find the left boundary + for (k = c->n_cigar - 1, score = max = 0, x = x-1, y = y-1; k >= 0; --k) { + int op = cigar[k]&0xf; + int ol = cigar[k]>>4; + if (op == BAM_CMATCH) { + for (j = 0; j < ol; ++j) { + int c1, c2, z = y - j; + c1 = bam_nt16_nt4_table[bam1_seqi(seq, z)]; + if (ref[x+j] == 0) return -1; + c2 = bam_nt16_nt4_table[(int)bam_nt16_table[(int)ref[x-j]]]; + if (c1 < 3 && c2 < 3) + score += c1 == c2? 5 : -4; + if (score < 0) score = 0; + if (score > max) max = score, q[0] = z, r[0] = x-j, kk[0] = k, kl[0] = j + 1; + } + x -= ol; y -= ol; + } else if (op == BAM_CINS) { + score -= 4 - ol * 3; + y -= ol; + if (score < 0) score = 0; + } else if (op == BAM_CDEL) { + score -= 4 - ol * 3; + x -= ol; + if (score < 0) score = 0; + } else if (op == BAM_CSOFT_CLIP) y -= ol; + else if (op == BAM_CREF_SKIP) x -= ol; } - if (k != c->n_cigar || l_ref < MIN_REF_LEN) return -1; - for (k = 0; k < l_ref; ++k) - if (ref[c->pos + k] == 0) return -1; // the read stands out of the reference - // allocate - s_ref = calloc(l_ref, 1); - s_read = calloc(c->l_qseq, 1); - for (k = 0, seq = bam1_seq(b); k < c->l_qseq; ++k) - s_read[k] = bam_nt16_nt4_table[bam1_seqi(seq, k)]; - for (k = 0; k < l_ref; ++k) - s_ref[k] = bam_nt16_nt4_table[(int)bam_nt16_table[(int)ref[c->pos + k]]]; - // do alignment - cigar = ka_global_core(s_ref, l_ref, s_read, c->l_qseq, &par, &score, &n_cigar); - if (score <= 0) { // realignment failed - free(cigar); free(s_ref); free(s_read); - return -1; + if (q[1] - q[0] < 15) return -1; // the high-scoring segment is too short + // modify CIGAR + n_cigar = 0; + cigar = calloc(c->n_cigar + 4, 4); + if (q[0] != 0) cigar[n_cigar++] = (uint32_t)q[0]<<4 | BAM_CSOFT_CLIP; + if (r[0] != c->pos) cigar[n_cigar++] = (uint32_t)(r[0] - c->pos)<<4 | BAM_CREF_SKIP; + if (kk[0] == kk[1]) { + cigar[n_cigar++] = (uint32_t)(kl[0] + kl[1] - (bam1_cigar(b)[kk[0]]>>4))<<4 | BAM_CMATCH; + } else { + cigar[n_cigar++] = (uint32_t)kl[0]<<4 | BAM_CMATCH; + for (k = kk[0] + 1; k < kk[1]; ++k) + cigar[n_cigar++] = bam1_cigar(b)[k]; + cigar[n_cigar++] = (uint32_t)kl[1]<<4 | BAM_CMATCH; // FIXME: add ref_skip after this line } + if (q[1] + 1 != c->l_qseq) + cigar[n_cigar++] = (uint32_t)(c->l_qseq - q[1] - 1)<<4 | BAM_CSOFT_CLIP; // copy over the alignment if (4 * (n_cigar - (int)c->n_cigar) + b->data_len > b->m_data) { // enlarge b->data b->m_data = 4 * (n_cigar - (int)c->n_cigar) + b->data_len; @@ -166,22 +257,22 @@ int bam_realn(bam1_t *b, const char *ref) } memcpy(bam1_cigar(b), cigar, n_cigar * 4); c->n_cigar = n_cigar; - free(s_ref); free(s_read); free(cigar); + free(cigar); return 0; } int bam_fillmd(int argc, char *argv[]) { - int c, is_equal = 0, tid = -2, ret, len, is_bam_out, is_sam_in, is_uncompressed, max_nm = 0, is_realn; + int c, is_equal = 0, tid = -2, ret, len, is_bam_out, is_sam_in, is_uncompressed, max_nm = 0, is_realn, is_capQ; samfile_t *fp, *fpout = 0; faidx_t *fai; char *ref = 0, mode_w[8], mode_r[8]; bam1_t *b; - is_bam_out = is_sam_in = is_uncompressed = is_realn = 0; + is_bam_out = is_sam_in = is_uncompressed = is_realn = is_capQ = 0; mode_w[0] = mode_r[0] = 0; strcpy(mode_r, "r"); strcpy(mode_w, "w"); - while ((c = getopt(argc, argv, "reubSn:")) >= 0) { + while ((c = getopt(argc, argv, "reubSCn:")) >= 0) { switch (c) { case 'r': is_realn = 1; break; case 'e': is_equal = 1; break; @@ -189,6 +280,7 @@ int bam_fillmd(int argc, char *argv[]) case 'u': is_uncompressed = is_bam_out = 1; break; case 'S': is_sam_in = 1; break; case 'n': max_nm = atoi(optarg); break; + case 'C': is_capQ = 1; break; default: fprintf(stderr, "[bam_fillmd] unrecognized option '-%c'\n", c); return 1; } } @@ -227,6 +319,10 @@ int bam_fillmd(int argc, char *argv[]) fp->header->target_name[tid]); } if (is_realn) bam_realn(b, ref); + if (is_capQ) { + int q = bam_cap_mapQ(b, ref); + if (b->core.qual > q) b->core.qual = q; + } if (ref) bam_fillmd1_core(b, ref, is_equal, max_nm); } samwrite(fpout, b); diff --git a/bam_plcmd.c b/bam_plcmd.c index cdbf67d..e9b3507 100644 --- a/bam_plcmd.c +++ b/bam_plcmd.c @@ -458,7 +458,7 @@ int bam_pileup(int argc, char *argv[]) #define MPLP_NO_COMP 0x20 #define MPLP_NO_ORPHAN 0x40 #define MPLP_REALN 0x80 -#define MPLP_NO_HALFTRIM 0x100 +#define MPLP_CAPQ 0x100 typedef struct { int max_mq, min_mq, flag, min_baseQ; @@ -471,7 +471,7 @@ typedef struct { typedef struct { bamFile fp; bam_iter_t iter; - int min_mq, flag; + int min_mq, flag, ref_id; char *ref; } mplp_aux_t; @@ -484,17 +484,23 @@ typedef struct { static int mplp_func(void *data, bam1_t *b) { extern int bam_realn(bam1_t *b, const char *ref); + extern int bam_cap_mapQ(bam1_t *b, char *ref); mplp_aux_t *ma = (mplp_aux_t*)data; - int ret, cond = 0; + int ret, skip = 0; do { - cond = 0; + int has_ref = (ma->ref && ma->ref_id == b->core.tid)? 1 : 0; ret = ma->iter? bam_iter_read(ma->fp, ma->iter, b) : bam_read1(ma->fp, b); if (ret < 0) break; - if (b->core.flag&BAM_FUNMAP) cond = 1; - else if (b->core.qual < ma->min_mq) cond = 1; - else if ((ma->flag&MPLP_NO_ORPHAN) && (b->core.flag&1) && !(b->core.flag&2)) cond = 1; - if (ma->ref && !cond && (ma->flag&MPLP_REALN)) bam_realn(b, ma->ref); - } while (cond); + skip = 0; + if (has_ref && (ma->flag&MPLP_REALN)) bam_realn(b, ma->ref); + if ((ma->flag&MPLP_CAPQ) && has_ref) { + int q = bam_cap_mapQ(b, ma->ref); + if (q < 0) skip = 1; + else if (b->core.qual > q) b->core.qual = q; + } else if (b->core.flag&BAM_FUNMAP) skip = 1; + else if (b->core.qual < ma->min_mq) skip = 1; + else if ((ma->flag&MPLP_NO_ORPHAN) && (b->core.flag&1) && !(b->core.flag&2)) skip = 1; + } while (skip); return ret; } @@ -624,9 +630,9 @@ static int mpileup(mplp_conf_t *conf, int n, char **fn) if (k == kh_end(hash)) continue; } if (tid != ref_tid) { - free(ref); + free(ref); ref = 0; if (conf->fai) ref = fai_fetch(conf->fai, h->target_name[tid], &ref_len); - for (i = 0; i < n; ++i) data[i]->ref = ref; + for (i = 0; i < n; ++i) data[i]->ref = ref, data[i]->ref_id = tid; ref_tid = tid; } if (conf->flag & MPLP_GLF) { @@ -693,7 +699,7 @@ int bam_mpileup(int argc, char *argv[]) mplp.max_mq = 60; mplp.theta = 1e-3; mplp.min_baseQ = 13; - while ((c = getopt(argc, argv, "gf:r:l:M:q:t:Q:uaORH")) >= 0) { + while ((c = getopt(argc, argv, "gf:r:l:M:q:t:Q:uaORC")) >= 0) { switch (c) { case 't': mplp.theta = atof(optarg); break; case 'f': @@ -704,9 +710,9 @@ int bam_mpileup(int argc, char *argv[]) case 'l': mplp.fn_pos = strdup(optarg); break; case 'g': mplp.flag |= MPLP_GLF; break; case 'u': mplp.flag |= MPLP_NO_COMP; break; - case 'a': mplp.flag |= MPLP_NO_ORPHAN | MPLP_REALN | MPLP_NO_HALFTRIM; break; + case 'a': mplp.flag |= MPLP_NO_ORPHAN | MPLP_REALN; break; case 'O': mplp.flag |= MPLP_NO_ORPHAN; break; - case 'H': mplp.flag |= MPLP_NO_HALFTRIM; break; + case 'C': mplp.flag |= MPLP_CAPQ; break; case 'R': mplp.flag |= MPLP_REALN; break; case 'M': mplp.max_mq = atoi(optarg); break; case 'q': mplp.min_mq = atoi(optarg); break; diff --git a/bcftools/vcfutils.pl b/bcftools/vcfutils.pl index ee9e998..b01799f 100755 --- a/bcftools/vcfutils.pl +++ b/bcftools/vcfutils.pl @@ -13,7 +13,7 @@ sub main { my $version = '0.1.0'; &usage if (@ARGV < 1); my $command = shift(@ARGV); - my %func = (subsam=>\&subsam, listsam=>\&listsam, fillac=>\&fillac, qstats=>\&qstats); + my %func = (subsam=>\&subsam, listsam=>\&listsam, fillac=>\&fillac, qstats=>\&qstats, varFilter=>\&varFilter); die("Unknown command \"$command\".\n") if (!defined($func{$command})); &{$func{$command}}; } @@ -115,9 +115,9 @@ Note: This command discards indels. Output: QUAL #non-indel #SNPs #transitions # next if (/^#/); my @t = split; next if (length($t[3]) != 1 || uc($t[3]) eq 'N'); + $t[3] = uc($t[3]); $t[4] = uc($t[4]); my @s = split(',', $t[4]); $t[5] = 3 if ($t[5] < 0); - $t[3] = uc($t[3]); $t[4] = uc($t[4]); next if (length($s[0]) != 1); push(@a, [$t[5], ($t[4] eq '.' || $t[4] eq $t[3])? 0 : 1, $ts{$t[3].$s[0]}? 1 : 0, $h{$t[0],$t[1]}? 1 : 0]); } @@ -141,6 +141,146 @@ Note: This command discards indels. Output: QUAL #non-indel #SNPs #transitions # } } +sub varFilter { + my %opts = (d=>1, D=>10000, l=>30, Q=>25, q=>10, G=>25, s=>100, w=>10, W=>10, N=>2, p=>undef, F=>.001); + getopts('pq:d:D:l:Q:w:W:N:G:F:', \%opts); + die(qq/ +Usage: vcfutils.pl varFilter [options] + +Options: -Q INT minimum RMS mapping quality for SNPs [$opts{Q}] + -q INT minimum RMS mapping quality for gaps [$opts{q}] + -d INT minimum read depth [$opts{d}] + -D INT maximum read depth [$opts{D}] + + -G INT min indel score for nearby SNP filtering [$opts{G}] + -w INT SNP within INT bp around a gap to be filtered [$opts{w}] + + -W INT window size for filtering dense SNPs [$opts{W}] + -N INT max number of SNPs in a window [$opts{N}] + + -l INT window size for filtering adjacent gaps [$opts{l}] + + -p print filtered variants +\n/) if (@ARGV == 0 && -t STDIN); + + # calculate the window size + my ($ol, $ow, $oW) = ($opts{l}, $opts{w}, $opts{W}); + my $max_dist = $ol > $ow? $ol : $ow; + $max_dist = $oW if ($max_dist < $oW); + # the core loop + my @staging; # (indel_filtering_score, flt_tag) + while (<>) { + my @t = split; + next if (/^#/); + next if ($t[4] eq '.'); # skip non-var sites + my $is_snp = 1; + if (length($t[3]) > 1) { + $is_snp = 0; + } else { + my @s = split(',', $t[4]); + for (@s) { + $is_snp = 0 if (length > 1); + } + } + # clear the out-of-range elements + while (@staging) { + # Still on the same chromosome and the first element's window still affects this position? + last if ($staging[0][3] eq $t[0] && $staging[0][4] + $staging[0][2] + $max_dist >= $t[1]); + varFilter_aux(shift(@staging), $opts{p}); # calling a function is a bit slower, not much + } + my ($flt, $score) = (0, -1); + + # collect key annotations + my ($dp, $mq, $af) = (-1, -1, 1); + if ($t[7] =~ /DP=(\d+)/i) { + $dp = $1; + } elsif ($t[7] =~ /DP4=(\d+),(\d+),(\d+),(\d+)/i) { + $dp = $1 + $2 + $3 + $4; + } + if ($t[7] =~ /MQ=(\d+)/i) { + $mq = $1; + } + if ($t[7] =~ /AF=([^\s;=]+)/i) { + $af = $1; + } elsif ($t[7] =~ /AF1=([^\s;=]+)/i) { + $af = $1; + } + # the depth filter + if ($dp >= 0) { + if ($dp < $opts{d}) { + $flt = 2; + } elsif ($dp > $opts{D}) { + $flt = 3; + } + } + + # site dependent filters + my $dlen = 0; + if ($flt == 0) { + if (!$is_snp) { # an indel + # If deletion, remember the length of the deletion + $dlen = length($t[3]) - 1; + $flt = 1 if ($mq < $opts{q}); + # filtering SNPs + if ($t[5] >= $opts{G}) { + for my $x (@staging) { + # Is it a SNP and is it outside the SNP filter window? + next if ($x->[0] >= 0 || $x->[4] + $x->[2] + $ow < $t[1]); + $x->[1] = 5 if ($x->[1] == 0); + } + } + # the indel filtering score + $score = $t[5]; + # check the staging list for indel filtering + for my $x (@staging) { + # Is it a SNP and is it outside the gap filter window + next if ($x->[0] < 0 || $x->[4] + $x->[2] + $ol < $t[1]); + if ($x->[0] < $score) { + $x->[1] = 6; + } else { + $flt = 6; last; + } + } + } else { # a SNP + $flt = 1 if ($mq < $opts{Q}); + # check adjacent SNPs + my $k = 1; + for my $x (@staging) { + ++$k if ($x->[0] < 0 && -($x->[0] + 1) > $opts{F} && $x->[4] + $x->[2] + $oW >= $t[1] && ($x->[1] == 0 || $x->[1] == 4 || $x->[1] == 5)); + } + # filtering is necessary + if ($k > $opts{N}) { + $flt = 4; + for my $x (@staging) { + $x->[1] = 4 if ($x->[0] < 0 && $x->[4] + $x->[2] + $oW >= $t[1] && $x->[1] == 0); + } + } else { # then check gap filter + for my $x (@staging) { + next if ($x->[0] < 0 || $x->[4] + $x->[2] + $ow < $t[1]); + if ($x->[0] >= $opts{G}) { + $flt = 5; last; + } + } + } + } + } + push(@staging, [$score < 0? -$af-1 : $score, $flt, $dlen, @t]); + } + # output the last few elements in the staging list + while (@staging) { + varFilter_aux(shift @staging, $opts{p}); + } +} + +sub varFilter_aux { + my ($first, $is_print) = @_; + if ($first->[1] == 0) { + print join("\t", @$first[3 .. @$first-1]), "\n"; + } elsif ($is_print) { + print STDERR join("\t", substr("UQdDWGgsiX", $first->[1], 1), @$first[3 .. @$first-1]), "\n"; + } +} + sub usage { die(qq/ Usage: vcfutils.pl []\n @@ -148,5 +288,6 @@ Command: subsam get a subset of samples listsam list the samples fillac fill the allele count field qstats SNP stats stratified by QUAL + varFilter filtering short variants \n/); } diff --git a/misc/samtools.pl b/misc/samtools.pl index 9f48b8f..ce8449d 100755 --- a/misc/samtools.pl +++ b/misc/samtools.pl @@ -104,7 +104,6 @@ Options: -Q INT minimum RMS mapping quality for SNPs [$opts{Q}] my $len=0; if ($flt == 0) { if ($t[2] eq '*') { # an indel - # If deletion, remember the length of the deletion my ($a,$b) = split(m{/},$t[3]); my $alen = length($a) - 1; -- 2.39.2