* prepare to replace kaln with kprobaln in realignment

[samtools.git] / bam2bcf_indel.c

diff --git a/bam2bcf_indel.c b/bam2bcf_indel.c
index 2d0b5c6f8a66a4c5c006dbec560718ab1f7a0377..7f6e288014cf1e0b627f3e7f2150e6dfbef5a03d 100644 (file)
--- a/bam2bcf_indel.c
+++ b/bam2bcf_indel.c
@@ -5,11 +5,13 @@
 #include "bam2bcf.h"
 #include "ksort.h"
 #include "kaln.h"
+#include "kprobaln.h"
 #include "khash.h"
 KHASH_SET_INIT_STR(rg)
 
 #define MINUS_CONST 0x10000000
 #define INDEL_WINDOW_SIZE 50
+#define MAX_SCORE 90
 
 void *bcf_call_add_rg(void *_hash, const char *hdtext, const char *list)
 {
@@ -110,7 +112,7 @@ int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_calla
                                          const void *rghash)
 {
        extern void ks_introsort_uint32_t(int, uint32_t*);
-       int i, s, j, k, t, n_types, *types, max_rd_len, left, right, max_ins, *score, N, K, l_run, ref_type;
+       int i, s, j, k, t, n_types, *types, max_rd_len, left, right, max_ins, *score, N, K, l_run, ref_type, n_alt;
        char *inscns = 0, *ref2, *query;
        khash_t(rg) *hash = (khash_t(rg)*)rghash;
        if (ref == 0 || bca == 0) return -1;
@@ -273,9 +275,17 @@ int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_calla
                                for (l = qbeg; l < qend; ++l)
                                        query[l - qbeg] = bam_nt16_nt4_table[bam1_seqi(seq, l)];
                                // do alignment; this takes most of computing time for indel calling
-                               sc = ka_global_score((uint8_t*)ref2 + tbeg - left, tend - tbeg + abs(types[t]),
-                                                                        (uint8_t*)query, qend - qbeg, &ap);
-                               score[K*n_types + t] = -sc;
+                               if (0) {
+                                       uint8_t *qq = calloc(qend - qbeg, 1);
+                                       for (l = 0; l < qend - qbeg; ++l) qq[l] = 23;
+                                       sc = kpa_glocal((uint8_t*)ref2 + tbeg - left, tend - tbeg + abs(types[t]),
+                                                                       (uint8_t*)query, qend - qbeg, qq, &kpa_par_alt, 0, 0);
+                                       score[K*n_types + t] = sc;
+                               } else {
+                                       sc = ka_global_score((uint8_t*)ref2 + tbeg - left, tend - tbeg + abs(types[t]),
+                                                                                (uint8_t*)query, qend - qbeg, &ap);
+                                       score[K*n_types + t] = -sc;
+                               }
 /*
                                for (l = 0; l < tend - tbeg + abs(types[t]); ++l)
                                        fputc("ACGTN"[(int)ref2[tbeg-left+l]], stderr);
@@ -316,6 +326,7 @@ int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_calla
                                        indelQ = (sc[t]>>6) - (sc[0]>>6);
                                        seqQ = est_seqQ(bca, types[sc[0]&0x3f], l_run);
                                }
+                               if (sc[0]>>6 > MAX_SCORE) indelQ = 0; // too many mismatches; something bad possibly happened
                                p->aux = (sc[0]&0x3f)<<16 | seqQ<<8 | indelQ;
                                sumq[sc[0]&0x3f] += indelQ < seqQ? indelQ : seqQ;
 //                             fprintf(stderr, "pos=%d read=%d:%d name=%s call=%d q=%d\n", pos, s, i, bam1_qname(p->b), types[sc[0]&0x3f], indelQ);
@@ -327,7 +338,7 @@ int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_calla
                for (t = 0; t < n_types; ++t)
                        sumq[t] = sumq[t]<<6 | t;
                for (t = 1; t < n_types; ++t) // insertion sort
-                       for (j = t; j > 0 && sumq[j] < sumq[j-1]; --j)
+                       for (j = t; j > 0 && sumq[j] > sumq[j-1]; --j)
                                tmp = sumq[j], sumq[j] = sumq[j-1], sumq[j-1] = tmp;
                for (t = 0; t < n_types; ++t) // look for the reference type
                        if ((sumq[t]&0x3f) == ref_type) break;
@@ -342,19 +353,20 @@ int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_calla
                        memcpy(&bca->inscns[t * bca->maxins], &inscns[(sumq[t]&0x3f) * max_ins], bca->maxins);
                }
                // update p->aux
-               for (s = K = 0; s < n; ++s) {
-                       for (i = 0; i < n_plp[s]; ++i, ++K) {
+               for (s = n_alt = 0; s < n; ++s) {
+                       for (i = 0; i < n_plp[s]; ++i) {
                                bam_pileup1_t *p = plp[s] + i;
                                int x = types[p->aux>>16&0x3f];
                                for (j = 0; j < 4; ++j)
                                        if (x == bca->indel_types[j]) break;
                                p->aux = j<<16 | (j == 4? 0 : (p->aux&0xffff));
-//                             fprintf(stderr, "pos=%d read=%d:%d name=%s call=%d q=%d\n", pos, s, i, bam1_qname(p->b), p->aux>>16&63, p->aux&0xff);
+                               if ((p->aux>>16&0x3f) > 0) ++n_alt;
+//                             fprintf(stderr, "X pos=%d read=%d:%d name=%s call=%d type=%d q=%d seqQ=%d\n", pos, s, i, bam1_qname(p->b), p->aux>>16&63, bca->indel_types[p->aux>>16&63], p->aux&0xff, p->aux>>8&0xff);
                        }
                }               
        }
        free(score);
        // free
        free(types); free(inscns);
-       return 0;
+       return n_alt > 0? 0 : -1;
 }