* or functionality.
*/
-/* 2009-06-12 by lh3: support a mode string like "wu" where 'u' for uncompressed output */
+/*
+ 2009-06-29 by lh3: cache recent uncompressed blocks.
+ 2009-06-25 by lh3: optionally use my knetfile library to access file on a FTP.
+ 2009-06-12 by lh3: support a mode string like "wu" where 'u' for uncompressed output */
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include "bgzf.h"
+#include "khash.h"
+typedef struct {
+ int size;
+ uint8_t *block;
+ int64_t end_offset;
+} cache_t;
+KHASH_MAP_INIT_INT64(cache, cache_t)
+
extern off_t ftello(FILE *stream);
extern int fseeko(FILE *stream, off_t offset, int whence);
fp->error = message;
}
+static BGZF *bgzf_read_init()
+{
+ BGZF *fp;
+ fp = calloc(1, sizeof(BGZF));
+ fp->uncompressed_block_size = MAX_BLOCK_SIZE;
+ fp->uncompressed_block = malloc(MAX_BLOCK_SIZE);
+ fp->compressed_block_size = MAX_BLOCK_SIZE;
+ fp->compressed_block = malloc(MAX_BLOCK_SIZE);
+ fp->cache_size = 0;
+ fp->cache = kh_init(cache);
+ return fp;
+}
+
static
BGZF*
open_read(int fd)
{
+#ifdef _USE_KNETFILE
+ knetFile *file = knet_dopen(fd, "r");
+#else
FILE* file = fdopen(fd, "r");
+#endif
BGZF* fp;
if (file == 0) return 0;
- fp = malloc(sizeof(BGZF));
+ fp = bgzf_read_init();
fp->file_descriptor = fd;
fp->open_mode = 'r';
- fp->owned_file = 0; fp->is_uncompressed = 0;
+#ifdef _USE_KNETFILE
+ fp->x.fpr = file;
+#else
fp->file = file;
- fp->uncompressed_block_size = MAX_BLOCK_SIZE;
- fp->uncompressed_block = malloc(MAX_BLOCK_SIZE);
- fp->compressed_block_size = MAX_BLOCK_SIZE;
- fp->compressed_block = malloc(MAX_BLOCK_SIZE);
- fp->block_address = 0;
- fp->block_offset = 0;
- fp->block_length = 0;
- fp->error = NULL;
+#endif
return fp;
}
fp->file_descriptor = fd;
fp->open_mode = 'w';
fp->owned_file = 0; fp->is_uncompressed = is_uncompressed;
+#ifdef _USE_KNETFILE
+ fp->x.fpw = file;
+#else
fp->file = file;
+#endif
fp->uncompressed_block_size = DEFAULT_BLOCK_SIZE;
fp->uncompressed_block = NULL;
fp->compressed_block_size = MAX_BLOCK_SIZE;
{
BGZF* fp = NULL;
if (mode[0] == 'r' || mode[0] == 'R') { /* The reading mode is preferred. */
+#ifdef _USE_KNETFILE
+ knetFile *file = knet_open(path, mode);
+ if (file == 0) return 0;
+ fp = bgzf_read_init();
+ fp->file_descriptor = -1;
+ fp->open_mode = 'r';
+ fp->x.fpr = file;
+#else
int oflag = O_RDONLY;
int fd = open(path, oflag);
if (fd == -1) return 0;
fp = open_read(fd);
+#endif
} else if (mode[0] == 'w' || mode[0] == 'W') {
int oflag = O_WRONLY | O_CREAT | O_TRUNC;
int fd = open(path, oflag, 0644);
unpackInt16((uint8_t*)&header[14]) == BGZF_LEN);
}
+static void free_cache(BGZF *fp)
+{
+ khint_t k;
+ khash_t(cache) *h = (khash_t(cache)*)fp->cache;
+ if (fp->open_mode != 'r') return;
+ for (k = kh_begin(h); k < kh_end(h); ++k)
+ if (kh_exist(h, k)) free(kh_val(h, k).block);
+ kh_destroy(cache, h);
+}
+
+static int load_block_from_cache(BGZF *fp, int64_t block_address)
+{
+ khint_t k;
+ cache_t *p;
+ khash_t(cache) *h = (khash_t(cache)*)fp->cache;
+ k = kh_get(cache, h, block_address);
+ if (k == kh_end(h)) return 0;
+ p = &kh_val(h, k);
+ if (fp->block_length != 0) fp->block_offset = 0;
+ fp->block_address = block_address;
+ fp->block_length = p->size;
+ memcpy(fp->uncompressed_block, p->block, MAX_BLOCK_SIZE);
+#ifdef _USE_KNETFILE
+ knet_seek(fp->x.fpr, p->end_offset, SEEK_SET);
+#else
+ fseeko(fp->file, p->end_offset, SEEK_SET);
+#endif
+ return p->size;
+}
+
+static void cache_block(BGZF *fp, int size)
+{
+ int ret;
+ khint_t k;
+ cache_t *p;
+ khash_t(cache) *h = (khash_t(cache)*)fp->cache;
+ if (MAX_BLOCK_SIZE >= fp->cache_size) return;
+ if ((kh_size(h) + 1) * MAX_BLOCK_SIZE > fp->cache_size) {
+ /* A better way would be to remove the oldest block in the
+ * cache, but here we remove a random one for simplicity. This
+ * should not have a big impact on performance. */
+ for (k = kh_begin(h); k < kh_end(h); ++k)
+ if (kh_exist(h, k)) break;
+ if (k < kh_end(h)) {
+ free(kh_val(h, k).block);
+ kh_del(cache, h, k);
+ }
+ }
+ k = kh_put(cache, h, fp->block_address, &ret);
+ if (ret == 0) return; // if this happens, a bug!
+ p = &kh_val(h, k);
+ p->size = fp->block_length;
+ p->end_offset = fp->block_address + size;
+ p->block = malloc(MAX_BLOCK_SIZE);
+ memcpy(kh_val(h, k).block, fp->uncompressed_block, MAX_BLOCK_SIZE);
+}
+
static
int
read_block(BGZF* fp)
{
byte header[BLOCK_HEADER_LENGTH];
+ int size = 0;
+#ifdef _USE_KNETFILE
+ int64_t block_address = knet_tell(fp->x.fpr);
+ if (load_block_from_cache(fp, block_address)) return 0;
+ int count = knet_read(fp->x.fpr, header, sizeof(header));
+#else
int64_t block_address = ftello(fp->file);
+ if (load_block_from_cache(fp, block_address)) return 0;
int count = fread(header, 1, sizeof(header), fp->file);
+#endif
if (count == 0) {
fp->block_length = 0;
return 0;
}
+ size = count;
if (count != sizeof(header)) {
report_error(fp, "read failed");
return -1;
byte* compressed_block = (byte*) fp->compressed_block;
memcpy(compressed_block, header, BLOCK_HEADER_LENGTH);
int remaining = block_length - BLOCK_HEADER_LENGTH;
+#ifdef _USE_KNETFILE
+ count = knet_read(fp->x.fpr, &compressed_block[BLOCK_HEADER_LENGTH], remaining);
+#else
count = fread(&compressed_block[BLOCK_HEADER_LENGTH], 1, remaining, fp->file);
+#endif
if (count != remaining) {
report_error(fp, "read failed");
return -1;
}
+ size += count;
count = inflate_block(fp, block_length);
if (count < 0) {
return -1;
}
fp->block_address = block_address;
fp->block_length = count;
+ cache_block(fp, size);
return 0;
}
bytes_read += copy_length;
}
if (fp->block_offset == fp->block_length) {
+#ifdef _USE_KNETFILE
+ fp->block_address = knet_tell(fp->x.fpr);
+#else
fp->block_address = ftello(fp->file);
+#endif
fp->block_offset = 0;
fp->block_length = 0;
}
if (block_length < 0) {
return -1;
}
+#ifdef _USE_KNETFILE
+ int count = fwrite(fp->compressed_block, 1, block_length, fp->x.fpw);
+#else
int count = fwrite(fp->compressed_block, 1, block_length, fp->file);
+#endif
if (count != block_length) {
report_error(fp, "write failed");
return -1;
if (flush_block(fp) != 0) {
return -1;
}
+#ifdef _USE_KNETFILE
+ if (fflush(fp->x.fpw) != 0) {
+#else
if (fflush(fp->file) != 0) {
+#endif
report_error(fp, "flush failed");
return -1;
}
}
if (fp->owned_file) {
+#ifdef _USE_KNETFILE
+ int ret;
+ if (fp->open_mode == 'w') ret = fclose(fp->x.fpw);
+ else ret = knet_close(fp->x.fpr);
+ if (ret != 0) return -1;
+#else
if (fclose(fp->file) != 0) {
return -1;
}
+#endif
}
free(fp->uncompressed_block);
free(fp->compressed_block);
+ free_cache(fp);
free(fp);
return 0;
}
return ((fp->block_address << 16) | (fp->block_offset & 0xFFFF));
}
+void bgzf_set_cache_size(BGZF *fp, int cache_size)
+{
+ if (fp) fp->cache_size = cache_size;
+}
+
int64_t
bgzf_seek(BGZF* fp, int64_t pos, int where)
{
}
int block_offset = pos & 0xFFFF;
int64_t block_address = (pos >> 16) & 0xFFFFFFFFFFFFLL;
+#ifdef _USE_KNETFILE
+ if (knet_seek(fp->x.fpr, block_address, SEEK_SET) != 0) {
+#else
if (fseeko(fp->file, block_address, SEEK_SET) != 0) {
+#endif
report_error(fp, "seek failed");
return -1;
}
projects / samtools.git / blobdiff