X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=bam_import.c;h=da2bf942ed90573cb3c964b4ced593ba7cbcf3d8;hb=0b3418cf166ce4a58cedf0d9a2df5ec3dd4cc5fa;hp=6b3b4bc69352a92d0c9e74379fe8ddd2e4564c43;hpb=f93dae0d03856955f9424e8b2aaf261304ca647e;p=samtools.git diff --git a/bam_import.c b/bam_import.c index 6b3b4bc..da2bf94 100644 --- a/bam_import.c +++ b/bam_import.c @@ -5,11 +5,16 @@ #include #include #include +#ifdef _WIN32 +#include +#endif +#include "kstring.h" #include "bam.h" +#include "sam_header.h" #include "kseq.h" #include "khash.h" -KSTREAM_INIT(gzFile, gzread, 8192) +KSTREAM_INIT(gzFile, gzread, 16384) KHASH_MAP_INIT_STR(ref, uint64_t) void bam_init_header_hash(bam_header_t *header); @@ -35,6 +40,25 @@ unsigned char bam_nt16_table[256] = { 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15 }; +unsigned short bam_char2flag_table[256] = { + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + 0,BAM_FREAD1,BAM_FREAD2,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + BAM_FPROPER_PAIR,0,BAM_FMREVERSE,0, 0,BAM_FMUNMAP,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, BAM_FDUP,0,BAM_FQCFAIL,0, 0,0,0,0, 0,0,0,0, + BAM_FPAIRED,0,BAM_FREVERSE,BAM_FSECONDARY, 0,BAM_FUNMAP,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, + 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 +}; + char *bam_nt16_rev_table = "=ACMGRSVTWYHKDBN"; struct __tamFile_t { @@ -42,31 +66,30 @@ struct __tamFile_t { kstream_t *ks; kstring_t *str; uint64_t n_lines; + int is_first; }; -char **bam_load_pos(const char *fn, int *_n) +char **__bam_get_lines(const char *fn, int *_n) // for bam_plcmd.c only { char **list = 0, *s; - int n = 0, dret, m = 0, c; + int n = 0, dret, m = 0; gzFile fp = (strcmp(fn, "-") == 0)? gzdopen(fileno(stdin), "r") : gzopen(fn, "r"); kstream_t *ks; kstring_t *str; str = (kstring_t*)calloc(1, sizeof(kstring_t)); ks = ks_init(fp); - while (ks_getuntil(ks, 0, str, &dret) > 0) { + while (ks_getuntil(ks, '\n', str, &dret) > 0) { if (n == m) { m = m? m << 1 : 16; list = (char**)realloc(list, m * sizeof(char*)); } - s = list[n++] = (char*)calloc(str->l + 5, 1); + if (str->s[str->l-1] == '\r') + str->s[--str->l] = '\0'; + s = list[n++] = (char*)calloc(str->l + 1, 1); strcpy(s, str->s); - s += str->l + 1; - ks_getuntil(ks, 0, str, &dret); - *((uint32_t*)s) = atoi(str->s); - if (dret != '\n') - while ((c = ks_getc(fp)) >= 0 && c != '\n'); } ks_destroy(ks); + gzclose(fp); free(str->s); free(str); *_n = n; return list; @@ -93,24 +116,29 @@ static bam_header_t *hash2header(const kh_ref_t *hash) bam_header_t *sam_header_read2(const char *fn) { bam_header_t *header; - int c, dret, ret; + int c, dret, ret, error = 0; gzFile fp; kstream_t *ks; kstring_t *str; kh_ref_t *hash; khiter_t k; - hash = kh_init(ref); + if (fn == 0) return 0; fp = (strcmp(fn, "-") == 0)? gzdopen(fileno(stdin), "r") : gzopen(fn, "r"); - assert(fp); + if (fp == 0) return 0; + hash = kh_init(ref); ks = ks_init(fp); str = (kstring_t*)calloc(1, sizeof(kstring_t)); - while (ks_getuntil(ks, 0, str, &dret) >= 0) { + while (ks_getuntil(ks, 0, str, &dret) > 0) { char *s = strdup(str->s); int len, i; i = kh_size(hash); ks_getuntil(ks, 0, str, &dret); len = atoi(str->s); k = kh_put(ref, hash, s, &ret); + if (ret == 0) { + fprintf(stderr, "[sam_header_read2] duplicated sequence name: %s\n", s); + error = 1; + } kh_value(hash, k) = (uint64_t)len<<32 | i; if (dret != '\n') while ((c = ks_getc(ks)) != '\n' && c != -1); @@ -119,6 +147,7 @@ bam_header_t *sam_header_read2(const char *fn) gzclose(fp); free(str->s); free(str); fprintf(stderr, "[sam_header_read2] %d sequences loaded.\n", kh_size(hash)); + if (error) return 0; header = hash2header(hash); kh_destroy(ref, hash); return header; @@ -139,21 +168,57 @@ static inline void parse_error(int64_t n_lines, const char * __restrict msg) } static inline void append_text(bam_header_t *header, kstring_t *str) { - int x = header->l_text, y = header->l_text + str->l + 2; // 2 = 1 byte dret + 1 byte null + size_t x = header->l_text, y = header->l_text + str->l + 2; // 2 = 1 byte dret + 1 byte null kroundup32(x); kroundup32(y); - if (x < y) header->text = (char*)realloc(header->text, y); + if (x < y) + { + header->n_text = y; + header->text = (char*)realloc(header->text, y); + if ( !header->text ) + { + fprintf(stderr,"realloc failed to alloc %ld bytes\n", y); + abort(); + } + } + // Sanity check + if ( header->l_text+str->l+1 >= header->n_text ) + { + fprintf(stderr,"append_text FIXME: %ld>=%ld, x=%ld,y=%ld\n", header->l_text+str->l+1,(long)header->n_text,x,y); + abort(); + } strncpy(header->text + header->l_text, str->s, str->l+1); // we cannot use strcpy() here. header->l_text += str->l + 1; header->text[header->l_text] = 0; } -int sam_read1(tamFile fp, bam_header_t *header, bam1_t *b) + +int sam_header_parse(bam_header_t *h) { - int ret, doff, doff0, dret; - bam1_core_t *c = &b->core; - kstring_t *str = fp->str; - kstream_t *ks = fp->ks; + char **tmp; + int i; + free(h->target_len); free(h->target_name); + h->n_targets = 0; h->target_len = 0; h->target_name = 0; + if (h->l_text < 3) return 0; + if (h->dict == 0) h->dict = sam_header_parse2(h->text); + tmp = sam_header2list(h->dict, "SQ", "SN", &h->n_targets); + if (h->n_targets == 0) return 0; + h->target_name = calloc(h->n_targets, sizeof(void*)); + for (i = 0; i < h->n_targets; ++i) + h->target_name[i] = strdup(tmp[i]); + free(tmp); + tmp = sam_header2list(h->dict, "SQ", "LN", &h->n_targets); + h->target_len = calloc(h->n_targets, 4); + for (i = 0; i < h->n_targets; ++i) + h->target_len[i] = atoi(tmp[i]); + free(tmp); + return h->n_targets; +} - while ((ret = ks_getuntil(fp->ks, 0, str, &dret)) >= 0 && str->s[0] == '@') { // skip header +bam_header_t *sam_header_read(tamFile fp) +{ + int ret, dret; + bam_header_t *header = bam_header_init(); + kstring_t *str = fp->str; + while ((ret = ks_getuntil(fp->ks, KS_SEP_TAB, str, &dret)) >= 0 && str->s[0] == '@') { // skip header str->s[str->l] = dret; // note that str->s is NOT null terminated!! append_text(header, str); if (dret != '\n') { @@ -163,7 +228,28 @@ int sam_read1(tamFile fp, bam_header_t *header, bam1_t *b) } ++fp->n_lines; } - while (ret == 0) ret = ks_getuntil(fp->ks, 0, str, &dret); // special consideration for "\r\n" + sam_header_parse(header); + bam_init_header_hash(header); + fp->is_first = 1; + return header; +} + +int sam_read1(tamFile fp, bam_header_t *header, bam1_t *b) +{ + int ret, doff, doff0, dret, z = 0; + bam1_core_t *c = &b->core; + kstring_t *str = fp->str; + kstream_t *ks = fp->ks; + + if (fp->is_first) { + fp->is_first = 0; + ret = str->l; + } else { + do { // special consideration for empty lines + ret = ks_getuntil(fp->ks, KS_SEP_TAB, str, &dret); + if (ret >= 0) z += str->l + 1; + } while (ret == 0); + } if (ret < 0) return -1; ++fp->n_lines; doff = 0; @@ -173,11 +259,28 @@ int sam_read1(tamFile fp, bam_header_t *header, bam1_t *b) memcpy(alloc_data(b, doff + c->l_qname) + doff, str->s, c->l_qname); doff += c->l_qname; } - { // flag, tid, pos, qual - ret = ks_getuntil(ks, 0, str, &dret); c->flag = atoi(str->s); - ret = ks_getuntil(ks, 0, str, &dret); c->tid = bam_get_tid(header, str->s); - ret = ks_getuntil(ks, 0, str, &dret); c->pos = isdigit(str->s[0])? atoi(str->s) - 1 : -1; - ret = ks_getuntil(ks, 0, str, &dret); c->qual = isdigit(str->s[0])? atoi(str->s) : 0; + { // flag + long flag; + char *s; + ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; + flag = strtol((char*)str->s, &s, 0); + if (*s) { // not the end of the string + flag = 0; + for (s = str->s; *s; ++s) + flag |= bam_char2flag_table[(int)*s]; + } + c->flag = flag; + } + { // tid, pos, qual + ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; c->tid = bam_get_tid(header, str->s); + if (c->tid < 0 && strcmp(str->s, "*")) { + if (header->n_targets == 0) { + fprintf(stderr, "[sam_read1] missing header? Abort!\n"); + exit(1); + } else fprintf(stderr, "[sam_read1] reference '%s' is recognized as '*'.\n", str->s); + } + ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; c->pos = isdigit(str->s[0])? atoi(str->s) - 1 : -1; + ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; c->qual = isdigit(str->s[0])? atoi(str->s) : 0; if (ret < 0) return -2; } { // cigar @@ -185,13 +288,16 @@ int sam_read1(tamFile fp, bam_header_t *header, bam1_t *b) int i, op; long x; c->n_cigar = 0; - if (ks_getuntil(ks, 0, str, &dret) < 0) return -3; + if (ks_getuntil(ks, KS_SEP_TAB, str, &dret) < 0) return -3; + z += str->l + 1; if (str->s[0] != '*') { + uint32_t *cigar; for (s = str->s; *s; ++s) { - if (isalpha(*s)) ++c->n_cigar; + if ((isalpha(*s)) || (*s=='=')) ++c->n_cigar; else if (!isdigit(*s)) parse_error(fp->n_lines, "invalid CIGAR character"); } b->data = alloc_data(b, doff + c->n_cigar * 4); + cigar = bam1_cigar(b); for (i = 0, s = str->s; i != c->n_cigar; ++i) { x = strtol(s, &t, 10); op = toupper(*t); @@ -202,52 +308,73 @@ int sam_read1(tamFile fp, bam_header_t *header, bam1_t *b) else if (op == 'S') op = BAM_CSOFT_CLIP; else if (op == 'H') op = BAM_CHARD_CLIP; else if (op == 'P') op = BAM_CPAD; + else if (op == '=') op = BAM_CEQUAL; + else if (op == 'X') op = BAM_CDIFF; + else if (op == 'B') op = BAM_CBACK; else parse_error(fp->n_lines, "invalid CIGAR operation"); s = t + 1; - bam1_cigar(b)[i] = x << BAM_CIGAR_SHIFT | op; + cigar[i] = bam_cigar_gen(x, op); } if (*s) parse_error(fp->n_lines, "unmatched CIGAR operation"); - c->bin = bam_reg2bin(c->pos, bam_calend(c, bam1_cigar(b))); + c->bin = bam_reg2bin(c->pos, bam_calend(c, cigar)); doff += c->n_cigar * 4; + } else { + if (!(c->flag&BAM_FUNMAP)) { + fprintf(stderr, "Parse warning at line %lld: mapped sequence without CIGAR\n", (long long)fp->n_lines); + c->flag |= BAM_FUNMAP; + } + c->bin = bam_reg2bin(c->pos, c->pos + 1); } } { // mtid, mpos, isize - ret = ks_getuntil(ks, 0, str, &dret); c->mtid = strcmp(str->s, "=")? bam_get_tid(header, str->s) : c->tid; - ret = ks_getuntil(ks, 0, str, &dret); c->mpos = isdigit(str->s[0])? atoi(str->s) - 1 : -1; - ret = ks_getuntil(ks, 0, str, &dret); c->isize = (str->s[0] == '-' || isdigit(str->s[0]))? atoi(str->s) : 0; + ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; + c->mtid = strcmp(str->s, "=")? bam_get_tid(header, str->s) : c->tid; + ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; + c->mpos = isdigit(str->s[0])? atoi(str->s) - 1 : -1; + ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; + c->isize = (str->s[0] == '-' || isdigit(str->s[0]))? atoi(str->s) : 0; if (ret < 0) return -4; } { // seq and qual int i; - uint8_t *p; - if (ks_getuntil(ks, 0, str, &dret) < 0) return -5; // seq - c->l_qseq = strlen(str->s); - if (c->n_cigar && c->l_qseq != (int32_t)bam_cigar2qlen(c, bam1_cigar(b))) - parse_error(fp->n_lines, "CIGAR and sequence length are inconsistent"); - p = (uint8_t*)alloc_data(b, doff + c->l_qseq + (c->l_qseq+1)/2) + doff; - bzero(p, (c->l_qseq+1)/2); - for (i = 0; i < c->l_qseq; ++i) - p[i/2] |= bam_nt16_table[(int)str->s[i]] << 4*(1-i%2); - if (ks_getuntil(ks, 0, str, &dret) < 0) return -6; // qual - if (c->l_qseq != strlen(str->s)) + uint8_t *p = 0; + if (ks_getuntil(ks, KS_SEP_TAB, str, &dret) < 0) return -5; // seq + z += str->l + 1; + if (strcmp(str->s, "*")) { + c->l_qseq = strlen(str->s); + if (c->n_cigar && c->l_qseq != (int32_t)bam_cigar2qlen(c, bam1_cigar(b))) { + fprintf(stderr, "Line %ld, sequence length %i vs %i from CIGAR\n", + (long)fp->n_lines, c->l_qseq, (int32_t)bam_cigar2qlen(c, bam1_cigar(b))); + parse_error(fp->n_lines, "CIGAR and sequence length are inconsistent"); + } + p = (uint8_t*)alloc_data(b, doff + c->l_qseq + (c->l_qseq+1)/2) + doff; + memset(p, 0, (c->l_qseq+1)/2); + for (i = 0; i < c->l_qseq; ++i) + p[i/2] |= bam_nt16_table[(int)str->s[i]] << 4*(1-i%2); + } else c->l_qseq = 0; + if (ks_getuntil(ks, KS_SEP_TAB, str, &dret) < 0) return -6; // qual + z += str->l + 1; + if (strcmp(str->s, "*") && c->l_qseq != strlen(str->s)) parse_error(fp->n_lines, "sequence and quality are inconsistent"); p += (c->l_qseq+1)/2; - for (i = 0; i < c->l_qseq; ++i) p[i] = str->s[i] - 33; + if (strcmp(str->s, "*") == 0) for (i = 0; i < c->l_qseq; ++i) p[i] = 0xff; + else for (i = 0; i < c->l_qseq; ++i) p[i] = str->s[i] - 33; doff += c->l_qseq + (c->l_qseq+1)/2; } doff0 = doff; if (dret != '\n' && dret != '\r') { // aux - while (ks_getuntil(ks, 0, str, &dret) >= 0) { + while (ks_getuntil(ks, KS_SEP_TAB, str, &dret) >= 0) { uint8_t *s, type, key[2]; + z += str->l + 1; if (str->l < 6 || str->s[2] != ':' || str->s[4] != ':') parse_error(fp->n_lines, "missing colon in auxiliary data"); key[0] = str->s[0]; key[1] = str->s[1]; type = str->s[3]; s = alloc_data(b, doff + 3) + doff; s[0] = key[0]; s[1] = key[1]; s += 2; doff += 2; - if (type == 'A' || type == 'a') { + if (type == 'A' || type == 'a' || type == 'c' || type == 'C') { // c and C for backward compatibility s = alloc_data(b, doff + 2) + doff; - *s++ = type; *s = str->s[5]; + *s++ = 'A'; *s = str->s[5]; doff += 2; } else if (type == 'I' || type == 'i') { long long x; @@ -287,6 +414,11 @@ int sam_read1(tamFile fp, bam_header_t *header, bam1_t *b) *s++ = 'f'; *(float*)s = (float)atof(str->s + 5); s += 4; doff += 5; + } else if (type == 'd') { + s = alloc_data(b, doff + 9) + doff; + *s++ = 'd'; + *(float*)s = (float)atof(str->s + 9); + s += 8; doff += 9; } else if (type == 'Z' || type == 'H') { int size = 1 + (str->l - 5) + 1; if (type == 'H') { // check whether the hex string is valid @@ -303,23 +435,46 @@ int sam_read1(tamFile fp, bam_header_t *header, bam1_t *b) memcpy(s, str->s + 5, str->l - 5); s[str->l - 5] = 0; doff += size; + } else if (type == 'B') { + int32_t n = 0, Bsize, k = 0, size; + char *p; + if (str->l < 8) parse_error(fp->n_lines, "too few values in aux type B"); + Bsize = bam_aux_type2size(str->s[5]); // the size of each element + for (p = (char*)str->s + 6; *p; ++p) // count the number of elements in the array + if (*p == ',') ++n; + p = str->s + 7; // now p points to the first number in the array + size = 6 + Bsize * n; // total number of bytes allocated to this tag + s = alloc_data(b, doff + 6 * Bsize * n) + doff; // allocate memory + *s++ = 'B'; *s++ = str->s[5]; + memcpy(s, &n, 4); s += 4; // write the number of elements + if (str->s[5] == 'c') while (p < str->s + str->l) ((int8_t*)s)[k++] = (int8_t)strtol(p, &p, 0), ++p; + else if (str->s[5] == 'C') while (p < str->s + str->l) ((uint8_t*)s)[k++] = (uint8_t)strtol(p, &p, 0), ++p; + else if (str->s[5] == 's') while (p < str->s + str->l) ((int16_t*)s)[k++] = (int16_t)strtol(p, &p, 0), ++p; // FIXME: avoid unaligned memory + else if (str->s[5] == 'S') while (p < str->s + str->l) ((uint16_t*)s)[k++] = (uint16_t)strtol(p, &p, 0), ++p; + else if (str->s[5] == 'i') while (p < str->s + str->l) ((int32_t*)s)[k++] = (int32_t)strtol(p, &p, 0), ++p; + else if (str->s[5] == 'I') while (p < str->s + str->l) ((uint32_t*)s)[k++] = (uint32_t)strtol(p, &p, 0), ++p; + else if (str->s[5] == 'f') while (p < str->s + str->l) ((float*)s)[k++] = (float)strtod(p, &p), ++p; + else parse_error(fp->n_lines, "unrecognized array type"); + s += Bsize * n; doff += size; } else parse_error(fp->n_lines, "unrecognized type"); if (dret == '\n' || dret == '\r') break; } } b->l_aux = doff - doff0; b->data_len = doff; - return 0; + if (bam_no_B) bam_remove_B(b); + return z; } tamFile sam_open(const char *fn) { tamFile fp; + gzFile gzfp = (strcmp(fn, "-") == 0)? gzdopen(fileno(stdin), "rb") : gzopen(fn, "rb"); + if (gzfp == 0) return 0; fp = (tamFile)calloc(1, sizeof(struct __tamFile_t)); fp->str = (kstring_t*)calloc(1, sizeof(kstring_t)); - fp->fp = (strcmp(fn, "-") == 0)? gzdopen(fileno(stdin), "r") : gzopen(fn, "r"); + fp->fp = gzfp; fp->ks = ks_init(fp->fp); - fp->n_lines = 0; return fp; } @@ -332,41 +487,3 @@ void sam_close(tamFile fp) free(fp); } } - -static void taf2baf_core(const char *fntaf, const char *fnbaf, bam_header_t *header) -{ - bamFile fpbaf; - bam1_t *b; - tamFile fp; - int ret; - - b = (bam1_t*)calloc(1, sizeof(bam1_t)); - fpbaf = bam_open(fnbaf, "w"); - fp = sam_open(fntaf); - ret = sam_read1(fp, header, b); - bam_header_write(fpbaf, header); - if (ret >= 0) { - bam_write1(fpbaf, b); - while (sam_read1(fp, header, b) >= 0) bam_write1(fpbaf, b); - } - bam_close(fpbaf); - free(b->data); free(b); - sam_close(fp); -} - -int bam_taf2baf(int argc, char *argv[]) -{ - int c; - bam_header_t *header; - - while ((c = getopt(argc, argv, "")) >= 0) { - } - if (optind + 3 > argc) { - fprintf(stderr, "Usage: bamtk import \n"); - return 1; - } - header = sam_header_read2(argv[optind]); - taf2baf_core(argv[optind+1], argv[optind+2], header); - bam_header_destroy(header); - return 0; -}