X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=bam2bcf_indel.c;h=ab9e83ca8f712ae7d7975d67b94b10c702e64908;hb=9f118264ea012adc21a46d7c03eaad4f9ce7d4d4;hp=4f747b156f439bf5d6e516cea3856bf5054e831c;hpb=ea1bf85308be7146d11a0153391d99fcfb3f83ad;p=samtools.git diff --git a/bam2bcf_indel.c b/bam2bcf_indel.c index 4f747b1..ab9e83c 100644 --- a/bam2bcf_indel.c +++ b/bam2bcf_indel.c @@ -3,15 +3,16 @@ #include #include "bam.h" #include "bam2bcf.h" -#include "ksort.h" #include "kaln.h" #include "kprobaln.h" #include "khash.h" KHASH_SET_INIT_STR(rg) +#include "ksort.h" +KSORT_INIT_GENERIC(uint32_t) + #define MINUS_CONST 0x10000000 #define INDEL_WINDOW_SIZE 50 -#define MIN_SUPPORT_COEF 500 void *bcf_call_add_rg(void *_hash, const char *hdtext, const char *list) { @@ -65,7 +66,7 @@ static int tpos2qpos(const bam1_core_t *c, const uint32_t *cigar, int32_t tpos, for (k = 0; k < c->n_cigar; ++k) { int op = cigar[k] & BAM_CIGAR_MASK; int l = cigar[k] >> BAM_CIGAR_SHIFT; - if (op == BAM_CMATCH) { + if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) { if (c->pos > tpos) return y; if (x + l > tpos) { *_tpos = tpos; @@ -111,10 +112,9 @@ static inline int est_indelreg(int pos, const char *ref, int l, char *ins4) int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_callaux_t *bca, const char *ref, const void *rghash) { - extern void ks_introsort_uint32_t(int, uint32_t*); - int i, s, j, k, t, n_types, *types, max_rd_len, left, right, max_ins, *score1, *score2; + int i, s, j, k, t, n_types, *types, max_rd_len, left, right, max_ins, *score1, *score2, max_ref2; int N, K, l_run, ref_type, n_alt; - char *inscns = 0, *ref2, *query; + char *inscns = 0, *ref2, *query, **ref_sample; khash_t(rg) *hash = (khash_t(rg)*)rghash; if (ref == 0 || bca == 0) return -1; // mark filtered reads @@ -161,13 +161,24 @@ int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_calla if (j > max_rd_len) max_rd_len = j; } } + // To prevent long stretches of N's to be mistaken for indels (sometimes thousands of bases), + // check the number of N's in the sequence and skip places where half or more reference bases are Ns. + int nN=0; for (i=pos; i-posi ) return -1; + ks_introsort(uint32_t, m, aux); // squeeze out identical types for (i = 1, n_types = 1; i < m; ++i) if (aux[i] != aux[i-1]) ++n_types; - if (n_types == 1 || n_alt * MIN_SUPPORT_COEF < n_tot) { // no indels or too few supporting reads + if (n_types == 1 || (double)n_alt / n_tot < bca->min_frac || n_alt < bca->min_support) { // then skip free(aux); return -1; } + if (n_types >= 64) { + free(aux); + if (bam_verbose >= 2) + fprintf(stderr, "[%s] excessive INDEL alleles at position %d. Skip the position.\n", __func__, pos + 1); + return -1; + } types = (int*)calloc(n_types, sizeof(int)); t = 0; types[t++] = aux[0] - MINUS_CONST; @@ -178,7 +189,6 @@ int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_calla for (t = 0; t < n_types; ++t) if (types[t] == 0) break; ref_type = t; // the index of the reference type (0) - assert(n_types < 64); } { // calculate left and right boundary left = pos > INDEL_WINDOW_SIZE? pos - INDEL_WINDOW_SIZE : 0; @@ -189,6 +199,58 @@ int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_calla if (ref[i] == 0) break; right = i; } + /* The following block fixes a long-existing flaw in the INDEL + * calling model: the interference of nearby SNPs. However, it also + * reduces the power because sometimes, substitutions caused by + * indels are not distinguishable from true mutations. Multiple + * sequence realignment helps to increase the power. + */ + { // construct per-sample consensus + int L = right - left + 1, max_i, max2_i; + uint32_t *cns, max, max2; + char *ref0, *r; + ref_sample = calloc(n, sizeof(void*)); + cns = calloc(L, 4); + ref0 = calloc(L, 1); + for (i = 0; i < right - left; ++i) + ref0[i] = bam_nt16_table[(int)ref[i+left]]; + for (s = 0; s < n; ++s) { + r = ref_sample[s] = calloc(L, 1); + memset(cns, 0, sizeof(int) * L); + // collect ref and non-ref counts + for (i = 0; i < n_plp[s]; ++i) { + bam_pileup1_t *p = plp[s] + i; + bam1_t *b = p->b; + uint32_t *cigar = bam1_cigar(b); + uint8_t *seq = bam1_seq(b); + int x = b->core.pos, y = 0; + for (k = 0; k < b->core.n_cigar; ++k) { + int op = cigar[k]&0xf; + int j, l = cigar[k]>>4; + if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) { + for (j = 0; j < l; ++j) + if (x + j >= left && x + j < right) + cns[x+j-left] += (bam1_seqi(seq, y+j) == ref0[x+j-left])? 1 : 0x10000; + x += l; y += l; + } else if (op == BAM_CDEL || op == BAM_CREF_SKIP) x += l; + else if (op == BAM_CINS || op == BAM_CSOFT_CLIP) y += l; + } + } + // determine the consensus + for (i = 0; i < right - left; ++i) r[i] = ref0[i]; + max = max2 = 0; max_i = max2_i = -1; + for (i = 0; i < right - left; ++i) { + if (cns[i]>>16 >= max>>16) max2 = max, max2_i = max_i, max = cns[i], max_i = i; + else if (cns[i]>>16 >= max2>>16) max2 = cns[i], max2_i = i; + } + if ((double)(max&0xffff) / ((max&0xffff) + (max>>16)) >= 0.7) max_i = -1; + if ((double)(max2&0xffff) / ((max2&0xffff) + (max2>>16)) >= 0.7) max2_i = -1; + if (max_i >= 0) r[max_i] = 15; + if (max2_i >= 0) r[max2_i] = 15; +// for (i = 0; i < right - left; ++i) fputc("=ACMGRSVTWYHKDBN"[(int)r[i]], stderr); fputc('\n', stderr); + } + free(ref0); free(cns); + } { // the length of the homopolymer run around the current position int c = bam_nt16_table[(int)ref[pos + 1]]; if (c == 15) l_run = 1; @@ -236,7 +298,8 @@ int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_calla free(inscns_aux); } // compute the likelihood given each type of indel for each read - ref2 = calloc(right - left + max_ins + 2, 1); + max_ref2 = right - left + 2 + 2 * (max_ins > -types[0]? max_ins : -types[0]); + ref2 = calloc(max_ref2, 1); query = calloc(right - left + max_rd_len + max_ins + 2, 1); score1 = calloc(N * n_types, sizeof(int)); score2 = calloc(N * n_types, sizeof(int)); @@ -251,26 +314,29 @@ int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_calla else ir = est_indelreg(pos, ref, -types[t], 0); if (ir > bca->indelreg) bca->indelreg = ir; // fprintf(stderr, "%d, %d, %d\n", pos, types[t], ir); - // write ref2 - for (k = 0, j = left; j <= pos; ++j) - ref2[k++] = bam_nt16_nt4_table[bam_nt16_table[(int)ref[j]]]; - if (types[t] <= 0) j += -types[t]; - else for (l = 0; l < types[t]; ++l) - ref2[k++] = inscns[t*max_ins + l]; - if (types[0] < 0) { // mask deleted sequences to avoid a particular error in the model. - int jj, tmp = types[t] >= 0? -types[0] : -types[0] + types[t]; - for (jj = 0; jj < tmp && j < right && ref[j]; ++jj, ++j) - ref2[k++] = 4; - } - for (; j < right && ref[j]; ++j) - ref2[k++] = bam_nt16_nt4_table[bam_nt16_table[(int)ref[j]]]; - if (j < right) right = j; - // align each read to ref2 + // realignment for (s = K = 0; s < n; ++s) { + // write ref2 + for (k = 0, j = left; j <= pos; ++j) + ref2[k++] = bam_nt16_nt4_table[(int)ref_sample[s][j-left]]; + if (types[t] <= 0) j += -types[t]; + else for (l = 0; l < types[t]; ++l) + ref2[k++] = inscns[t*max_ins + l]; + for (; j < right && ref[j]; ++j) + ref2[k++] = bam_nt16_nt4_table[(int)ref_sample[s][j-left]]; + for (; k < max_ref2; ++k) ref2[k] = 4; + if (j < right) right = j; + // align each read to ref2 for (i = 0; i < n_plp[s]; ++i, ++K) { bam_pileup1_t *p = plp[s] + i; - int qbeg, qend, tbeg, tend, sc; + int qbeg, qend, tbeg, tend, sc, kk; uint8_t *seq = bam1_seq(p->b); + uint32_t *cigar = bam1_cigar(p->b); + if (p->b->core.flag&4) continue; // unmapped reads + // FIXME: the following loop should be better moved outside; nonetheless, realignment should be much slower anyway. + for (kk = 0; kk < p->b->core.n_cigar; ++kk) + if ((cigar[kk]&BAM_CIGAR_MASK) == BAM_CREF_SKIP) break; + if (kk < p->b->core.n_cigar) continue; // FIXME: the following skips soft clips, but using them may be more sensitive. // determine the start and end of sequences for alignment qbeg = tpos2qpos(&p->b->core, bam1_cigar(p->b), left, 0, &tbeg); @@ -365,9 +431,11 @@ int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_calla indelQ2 = tmp > 111? 0 : (int)((1. - tmp/111.) * indelQ2 + .499); // pick the smaller between indelQ1 and indelQ2 indelQ = indelQ1 < indelQ2? indelQ1 : indelQ2; - p->aux = (sc[0]&0x3f)<<16 | seqQ<<8 | indelQ; + if (indelQ > 255) indelQ = 255; + if (seqQ > 255) seqQ = 255; + p->aux = (sc[0]&0x3f)<<16 | seqQ<<8 | indelQ; // use 22 bits in total sumq[sc[0]&0x3f] += indelQ < seqQ? indelQ : seqQ; -// fprintf(stderr, "pos=%d read=%d:%d name=%s call=%d q=%d\n", pos, s, i, bam1_qname(p->b), types[sc[0]&0x3f], indelQ); +// fprintf(stderr, "pos=%d read=%d:%d name=%s call=%d indelQ=%d seqQ=%d\n", pos, s, i, bam1_qname(p->b), types[sc[0]&0x3f], indelQ, seqQ); } } // determine bca->indel_types[] and bca->inscns @@ -405,156 +473,8 @@ int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_calla } free(score1); free(score2); // free + for (i = 0; i < n; ++i) free(ref_sample[i]); + free(ref_sample); free(types); free(inscns); return n_alt > 0? 0 : -1; } - -#define END_SKIP 4 // must be larger than B2B_MAX_MNP -#define MIN_MNP_FLANK_BAQ 20 - -int bcf_call_mnp_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_callaux_t *bca, const char *ref) -{ - extern void ks_introsort_uint32_t(int, uint32_t*); - int i, s, j, k, t, n_types, *types; - int N, ref_seq, ref_type; - if (ref == 0 || bca == 0) return -1; - if (pos < bca->last_mnp_pos + B2B_MNP_WIN) return -2; // to avoid calling a TNP multiple times - if (pos < END_SKIP || ref[pos] == 0 || ref[pos+1] == 0) return -2; // end of the reference - { // determine if there is an MNP - int r[2]; - uint8_t *tt; - r[0] = bam_nt16_table[(int)ref[pos]]; - r[1] = bam_nt16_table[(int)ref[pos+1]]; - for (s = 0; s < n; ++s) { - for (i = 0; i < n_plp[s]; ++i) { - bam_pileup1_t *p = plp[s] + i; - int left, rght; - if (p->qpos < END_SKIP || p->qpos >= p->b->core.l_qseq - END_SKIP) continue; - tt = bam1_seq(p->b); - if (bam1_seqi(tt, p->qpos) == r[0] || bam1_seqi(tt, p->qpos+1) == r[1]) continue; // no MNP - tt = bam1_qual(p->b); - for (left = 0, k = p->qpos - END_SKIP; k <= p->qpos; ++k) - left = left > tt[k]? left : tt[k]; - for (rght = 0, k = p->qpos; k < p->b->core.l_qseq - END_SKIP; ++k) - rght = rght > tt[k]? left : tt[k]; - if (left >= MIN_MNP_FLANK_BAQ && rght >= MIN_MNP_FLANK_BAQ) break; // bracketed by good bases - } - if (i != n_plp[s]) break; - } - if (s == n) return -1; // there is no MNP at this position. - } - for (s = N = 0; s < n; ++s) N += n_plp[s]; // N is the total number of reads - { // construct the MNP consensus - uint8_t *tt; - uint32_t *aux, x, *cnt; - int m = 0; - aux = calloc(N + 1, 4); - for (i = 0, x = 0; i < B2B_MAX_MNP; ++i) - x |= bam_nt16_nt4_table[bam_nt16_table[(int)ref[pos+i]]] << 2*i; - ref_seq = x; - bca->indelreg = 0; - for (s = 0; s < n; ++s) { - for (i = 0; i < n_plp[s]; ++i) { - bam_pileup1_t *p = plp[s] + i; - int stop; - if (p->qpos < END_SKIP || p->qpos >= p->b->core.l_qseq - END_SKIP) continue; - tt = bam1_seq(p->b); - for (k = j = stop = 0, x = 0; k < B2B_MAX_MNP; ++k) { - int c = bam_nt16_nt4_table[bam1_seqi(tt, p->qpos + k)]; - if (c > 3) break; - if (c != (ref_seq>>k*2&3) && !stop) ++j; - else stop = 1; - x |= c << 2*k; - } - if (k == B2B_MAX_MNP && j >= 2) aux[m++] = x; - } - } - if (m == 0) { - free(aux); return -1; - } - ks_introsort(uint32_t, m, aux); - // squeeze out indentical types - for (i = 1, n_types = 1; i < m; ++i) - if (aux[i] != aux[i-1]) ++n_types; - // count reads for each type - cnt = alloca(n_types * 4); - cnt[0] = 1<<8 | aux[0]; - for (i = 1, t = 0; i < m; ++i) { - if (aux[i] != aux[i-1]) { - ++t; - cnt[t] = 1<<8 | aux[i]; - } else cnt[t] += 1<<8; - } - free(aux); - // collect types; NOTE: we only collect ONE alternative type - ks_introsort(uint32_t, n_types, cnt); - if ((cnt[n_types-1]>>8) * MIN_SUPPORT_COEF < N) // no MNPs or too few supporting reads - return -1; - types = (int*)calloc(2, sizeof(int)); - bca->indel_types[0] = types[0] = ref_seq; - bca->indel_types[1] = types[1] = cnt[n_types-1]&0xff; - bca->indel_types[2] = bca->indel_types[3] = B2B_INDEL_NULL; - ref_type = 0; n_types = 2; - // calculate MNP length - for (i = 0; i < B2B_MAX_MNP; ++i) { - int c = types[0] >> 2*i & 3; - for (t = 1; t < n_types; ++t) - if ((types[t] >> 2*i & 3) != c) break; - if (t == n_types) break; - } - bca->indelreg = i; - if (bca->indelreg < 2) return -1; // should not happen in principle - x = (1<<2*bca->indelreg) - 1; - for (t = 0; t < n_types; ++t) types[t] &= x; - ref_seq &= x; - } - { // calling - for (s = 0; s < n; ++s) { - for (i = 0; i < n_plp[s]; ++i) { - bam_pileup1_t *p = plp[s] + i; - uint8_t *qq, *bq, *seq = bam1_seq(p->b); - uint32_t x, *cigar = bam1_cigar(p->b); - int y, skip = 0, baseQ = 0, baq = 0; - // try to get the original base quality - qq = bam1_qual(p->b); - bq = (uint8_t*)bam_aux_get(p->b, "ZQ"); - if (bq) ++bq; - // get the max original base quality - for (j = p->qpos, x = 0; j < p->b->core.l_qseq && j < p->qpos + bca->indelreg; ++j) { - int q = bq? qq[j] + (bq[j] - 64) : qq[j]; - int c = bam_nt16_nt4_table[bam1_seqi(seq, j)]; - baseQ = baseQ > q? baseQ : q; - if (c > 3) skip = 1; - x |= c << 2 * (j - p->qpos); - } - for (t = 0; t < n_types; ++t) - if (types[t] == x) break; - if (t == n_types) skip = 1; - else { // reapply BAQ - for (k = y = 0; k < p->b->core.n_cigar; ++k) { - int op = cigar[k]&0xf; - int len = cigar[k]>>4; - if (op == BAM_CMATCH) { - if (p->qpos >= y && p->qpos + bca->indelreg <= y + len) { - int left, rght; - for (j = y, left = 0; j <= p->qpos; ++j) - left = left > qq[j]? left : qq[j]; - for (j = p->qpos + bca->indelreg - 1, rght = 0; j < y + len; ++j) - rght = rght > qq[j]? rght : qq[j]; - baq = left < rght? left : rght; - break; - } - y += len; - } else if (op == BAM_CSOFT_CLIP || op == BAM_CINS) y += len; - } - } -// fprintf(stderr, "pos=%d read=%d:%d name=%s skip=%d reg=%d call=%d baseQ=%d baq=%d\n", pos+1, s, i, bam1_qname(p->b), skip, bca->indelreg, t, baseQ, baq); - baseQ = baseQ < baq? baseQ : baq; - p->aux = skip? 0 : (t<<16|baseQ<<8|baseQ); - } - } - } - // free - free(types); - return 0; -}