X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=bam2bcf_indel.c;h=2fdee9f75a2261266912b7fe355e9ea51892ff6d;hb=d195bcfb6632c68032a89c091c20dc24dd6ded60;hp=c157bc65b9663d62f3ede6863d55d0fe26e7e152;hpb=d1643d6e5e6ec3a7cb1ef8b62b9a6f9b21ff2527;p=samtools.git

diff --git a/bam2bcf_indel.c b/bam2bcf_indel.c
index c157bc6..2fdee9f 100644
--- a/bam2bcf_indel.c
+++ b/bam2bcf_indel.c
@@ -3,12 +3,14 @@
 #include <string.h>
 #include "bam.h"
 #include "bam2bcf.h"
-#include "ksort.h"
 #include "kaln.h"
 #include "kprobaln.h"
 #include "khash.h"
 KHASH_SET_INIT_STR(rg)
 
+#include "ksort.h"
+KSORT_INIT_GENERIC(uint32_t)
+
 #define MINUS_CONST 0x10000000
 #define INDEL_WINDOW_SIZE 50
 
@@ -110,7 +112,6 @@ static inline int est_indelreg(int pos, const char *ref, int l, char *ins4)
 int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_callaux_t *bca, const char *ref,
 					  const void *rghash)
 {
-	extern void ks_introsort_uint32_t(int, uint32_t*);
 	int i, s, j, k, t, n_types, *types, max_rd_len, left, right, max_ins, *score1, *score2, max_ref2;
 	int N, K, l_run, ref_type, n_alt;
 	char *inscns = 0, *ref2, *query, **ref_sample;
@@ -167,6 +168,12 @@ int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_calla
 		if (n_types == 1 || (double)n_alt / n_tot < bca->min_frac || n_alt < bca->min_support) { // then skip
 			free(aux); return -1;
 		}
+		if (n_types >= 64) {
+			free(aux);
+			if (bam_verbose >= 2) 
+				fprintf(stderr, "[%s] excessive INDEL alleles at position %d. Skip the position.\n", __func__, pos + 1);
+			return -1;
+		}
 		types = (int*)calloc(n_types, sizeof(int));
 		t = 0;
 		types[t++] = aux[0] - MINUS_CONST; 
@@ -177,7 +184,6 @@ int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_calla
 		for (t = 0; t < n_types; ++t)
 			if (types[t] == 0) break;
 		ref_type = t; // the index of the reference type (0)
-		assert(n_types < 64);
 	}
 	{ // calculate left and right boundary
 		left = pos > INDEL_WINDOW_SIZE? pos - INDEL_WINDOW_SIZE : 0;
@@ -420,9 +426,11 @@ int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_calla
 				indelQ2 = tmp > 111? 0 : (int)((1. - tmp/111.) * indelQ2 + .499);
 				// pick the smaller between indelQ1 and indelQ2
 				indelQ = indelQ1 < indelQ2? indelQ1 : indelQ2;
+				if (indelQ > 255) indelQ = 255;
+				if (seqQ > 255) seqQ = 255;
 				p->aux = (sc[0]&0x3f)<<16 | seqQ<<8 | indelQ; // use 22 bits in total
 				sumq[sc[0]&0x3f] += indelQ < seqQ? indelQ : seqQ;
-//				fprintf(stderr, "pos=%d read=%d:%d name=%s call=%d q=%d\n", pos, s, i, bam1_qname(p->b), types[sc[0]&0x3f], indelQ);
+//				fprintf(stderr, "pos=%d read=%d:%d name=%s call=%d indelQ=%d seqQ=%d\n", pos, s, i, bam1_qname(p->b), types[sc[0]&0x3f], indelQ, seqQ);
 			}
 		}
 		// determine bca->indel_types[] and bca->inscns