2 * RAZF : Random Access compressed(Z) File
4 * Release Date: 2008-10-27
6 * Copyright 2008, Jue Ruan <ruanjue@gmail.com>, Heng Li <lh3@sanger.ac.uk>
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * To compile razf.c, zlib-1.2.3(or greater) is required.
40 static inline uint32_t byte_swap_4(uint32_t v){
41 v = ((v & 0x0000FFFFU) << 16) | (v >> 16);
42 return ((v & 0x00FF00FFU) << 8) | ((v & 0xFF00FF00U) >> 8);
45 static inline uint64_t byte_swap_8(uint64_t v){
46 v = ((v & 0x00000000FFFFFFFFLLU) << 32) | (v >> 32);
47 v = ((v & 0x0000FFFF0000FFFFLLU) << 16) | ((v & 0xFFFF0000FFFF0000LLU) >> 16);
48 return ((v & 0x00FF00FF00FF00FFLLU) << 8) | ((v & 0xFF00FF00FF00FF00LLU) >> 8);
51 static inline int is_big_endian(){
54 return (c[0] != 0x01);
57 static void add_zindex(RAZF *rz, int64_t in, int64_t out){
58 if(rz->index->size == rz->index->cap){
59 rz->index->cap = rz->index->cap * 1.5 + 2;
60 rz->index->cell_offsets = realloc(rz->index->cell_offsets, sizeof(int) * rz->index->cap);
61 rz->index->bin_offsets = realloc(rz->index->bin_offsets, sizeof(int64_t) * (rz->index->cap/RZ_BIN_SIZE + 1));
63 if(rz->index->size % RZ_BIN_SIZE == 0) rz->index->bin_offsets[rz->index->size / RZ_BIN_SIZE] = out;
64 rz->index->cell_offsets[rz->index->size] = out - rz->index->bin_offsets[rz->index->size / RZ_BIN_SIZE];
68 static void save_zindex(RAZF *rz, int fd){
71 is_be = is_big_endian();
72 if(is_be) write(fd, &rz->index->size, sizeof(int));
74 v32 = byte_swap_4((uint32_t)rz->index->size);
75 write(fd, &v32, sizeof(uint32_t));
77 v32 = rz->index->size / RZ_BIN_SIZE + 1;
79 for(i=0;i<v32;i++) rz->index->bin_offsets[i] = byte_swap_8((uint64_t)rz->index->bin_offsets[i]);
80 for(i=0;i<rz->index->size;i++) rz->index->cell_offsets[i] = byte_swap_4((uint32_t)rz->index->cell_offsets[i]);
82 write(fd, rz->index->bin_offsets, sizeof(int64_t) * v32);
83 write(fd, rz->index->cell_offsets, sizeof(int32_t) * rz->index->size);
86 static void load_zindex(RAZF *rz, int fd){
89 if(!rz->load_index) return;
90 if(rz->index == NULL) rz->index = malloc(sizeof(ZBlockIndex));
91 is_be = is_big_endian();
92 read(fd, &rz->index->size, sizeof(int));
93 if(!is_be) rz->index->size = byte_swap_4((uint32_t)rz->index->size);
94 rz->index->cap = rz->index->size;
95 v32 = rz->index->size / RZ_BIN_SIZE + 1;
96 rz->index->bin_offsets = malloc(sizeof(int64_t) * v32);
97 read(fd, rz->index->bin_offsets, sizeof(int64_t) * v32);
98 rz->index->cell_offsets = malloc(sizeof(int) * rz->index->size);
99 read(fd, rz->index->cell_offsets, sizeof(int) * rz->index->size);
101 for(i=0;i<v32;i++) rz->index->bin_offsets[i] = byte_swap_8((uint64_t)rz->index->bin_offsets[i]);
102 for(i=0;i<rz->index->size;i++) rz->index->cell_offsets[i] = byte_swap_4((uint32_t)rz->index->cell_offsets[i]);
106 static RAZF* razf_open_w(int fd){
108 rz = calloc(1, sizeof(RAZF));
111 rz->stream = calloc(sizeof(z_stream), 1);
112 rz->inbuf = malloc(RZ_BUFFER_SIZE);
113 rz->outbuf = malloc(RZ_BUFFER_SIZE);
114 rz->index = calloc(sizeof(ZBlockIndex), 1);
115 deflateInit2(rz->stream, RZ_COMPRESS_LEVEL, Z_DEFLATED, WINDOW_BITS + 16, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY);
116 rz->stream->avail_out = RZ_BUFFER_SIZE;
117 rz->stream->next_out = rz->outbuf;
118 rz->header = calloc(sizeof(gz_header), 1);
119 rz->header->os = 0x03; //Unix
120 rz->header->text = 0;
121 rz->header->time = 0;
122 rz->header->extra = malloc(7);
123 strncpy((char*)rz->header->extra, "RAZF", 4);
124 rz->header->extra[4] = 1; // obsolete field
125 // block size = RZ_BLOCK_SIZE, Big-Endian
126 rz->header->extra[5] = RZ_BLOCK_SIZE >> 8;
127 rz->header->extra[6] = RZ_BLOCK_SIZE & 0xFF;
128 rz->header->extra_len = 7;
129 rz->header->name = rz->header->comment = 0;
130 rz->header->hcrc = 0;
131 deflateSetHeader(rz->stream, rz->header);
132 rz->block_pos = rz->block_off = 0;
136 static void _razf_write(RAZF* rz, const void *data, int size){
138 rz->stream->avail_in = size;
139 rz->stream->next_in = (void*)data;
141 tout = rz->stream->avail_out;
142 deflate(rz->stream, Z_NO_FLUSH);
143 rz->out += tout - rz->stream->avail_out;
144 if(rz->stream->avail_out) break;
145 write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
146 rz->stream->avail_out = RZ_BUFFER_SIZE;
147 rz->stream->next_out = rz->outbuf;
148 if(rz->stream->avail_in == 0) break;
150 rz->in += size - rz->stream->avail_in;
151 rz->block_off += size - rz->stream->avail_in;
154 static void razf_flush(RAZF *rz){
157 _razf_write(rz, rz->inbuf, rz->buf_len);
158 rz->buf_off = rz->buf_len = 0;
160 if(rz->stream->avail_out){
161 write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
162 rz->stream->avail_out = RZ_BUFFER_SIZE;
163 rz->stream->next_out = rz->outbuf;
166 tout = rz->stream->avail_out;
167 deflate(rz->stream, Z_FULL_FLUSH);
168 rz->out += tout - rz->stream->avail_out;
169 if(rz->stream->avail_out == 0){
170 write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
171 rz->stream->avail_out = RZ_BUFFER_SIZE;
172 rz->stream->next_out = rz->outbuf;
175 rz->block_pos = rz->out;
179 static void razf_end_flush(RAZF *rz){
182 _razf_write(rz, rz->inbuf, rz->buf_len);
183 rz->buf_off = rz->buf_len = 0;
186 tout = rz->stream->avail_out;
187 deflate(rz->stream, Z_FINISH);
188 rz->out += tout - rz->stream->avail_out;
189 if(rz->stream->avail_out < RZ_BUFFER_SIZE){
190 write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
191 rz->stream->avail_out = RZ_BUFFER_SIZE;
192 rz->stream->next_out = rz->outbuf;
197 static void _razf_buffered_write(RAZF *rz, const void *data, int size){
200 if(rz->buf_len == RZ_BUFFER_SIZE){
201 _razf_write(rz, rz->inbuf, rz->buf_len);
204 if(size + rz->buf_len < RZ_BUFFER_SIZE){
205 for(i=0;i<size;i++) ((char*)rz->inbuf + rz->buf_len)[i] = ((char*)data)[i];
209 n = RZ_BUFFER_SIZE - rz->buf_len;
210 for(i=0;i<n;i++) ((char*)rz->inbuf + rz->buf_len)[i] = ((char*)data)[i];
218 int razf_write(RAZF* rz, const void *data, int size){
222 next_block = ((rz->in / RZ_BLOCK_SIZE) + 1) * RZ_BLOCK_SIZE;
223 while(rz->in + rz->buf_len + size >= next_block){
224 n = next_block - rz->in - rz->buf_len;
225 _razf_buffered_write(rz, data, n);
229 add_zindex(rz, rz->in, rz->out);
230 next_block = ((rz->in / RZ_BLOCK_SIZE) + 1) * RZ_BLOCK_SIZE;
232 _razf_buffered_write(rz, data, size);
237 #define ASCII_FLAG 0x01 /* bit 0 set: file probably ascii text */
238 #define HEAD_CRC 0x02 /* bit 1 set: header CRC present */
239 #define EXTRA_FIELD 0x04 /* bit 2 set: extra field present */
240 #define ORIG_NAME 0x08 /* bit 3 set: original file name present */
241 #define COMMENT 0x10 /* bit 4 set: file comment present */
242 #define RESERVED 0xE0 /* bits 5..7: reserved */
244 static int _read_gz_header(unsigned char *data, int size, int *extra_off, int *extra_len){
245 int method, flags, n, len;
246 if(size < 2) return 0;
247 if(data[0] != 0x1f || data[1] != 0x8b) return 0;
248 if(size < 4) return 0;
251 if(method != Z_DEFLATED || (flags & RESERVED)) return 0;
252 n = 4 + 6; // Skip 6 bytes
255 if(flags & EXTRA_FIELD){
256 if(size < n + 2) return 0;
257 len = ((int)data[n + 1] << 8) | data[n];
261 if(n >= size) return 0;
265 *extra_len = n - (*extra_off);
267 if(flags & ORIG_NAME) while(n < size && data[n++]);
268 if(flags & COMMENT) while(n < size && data[n++]);
269 if(flags & HEAD_CRC){
270 if(n + 2 > size) return 0;
276 static RAZF* razf_open_r(int fd, int _load_index){
278 int ext_off, ext_len;
281 unsigned char c[] = "RAZF";
282 rz = calloc(1, sizeof(RAZF));
285 rz->stream = calloc(sizeof(z_stream), 1);
286 rz->inbuf = malloc(RZ_BUFFER_SIZE);
287 rz->outbuf = malloc(RZ_BUFFER_SIZE);
288 rz->end = rz->src_end = 0x7FFFFFFFFFFFFFFFLL;
289 n = read(rz->filedes, rz->inbuf, RZ_BUFFER_SIZE);
290 ret = _read_gz_header(rz->inbuf, n, &ext_off, &ext_len);
294 rz->file_type = FILE_TYPE_PLAIN;
295 memcpy(rz->outbuf, rz->inbuf, n);
301 rz->header_size = ret;
302 ret = inflateInit2(rz->stream, -WINDOW_BITS);
303 if(ret != Z_OK){ inflateEnd(rz->stream); goto PLAIN_FILE;}
304 rz->stream->avail_in = n - rz->header_size;
305 rz->stream->next_in = rz->inbuf + rz->header_size;
306 rz->stream->avail_out = RZ_BUFFER_SIZE;
307 rz->stream->next_out = rz->outbuf;
308 rz->file_type = FILE_TYPE_GZ;
309 rz->in = rz->header_size;
310 rz->block_pos = rz->header_size;
311 rz->next_block_pos = rz->header_size;
313 if(ext_len < 7 || memcmp(rz->inbuf + ext_off, c, 4) != 0) return rz;
314 if(((((unsigned char*)rz->inbuf)[ext_off + 5] << 8) | ((unsigned char*)rz->inbuf)[ext_off + 6]) != RZ_BLOCK_SIZE){
315 fprintf(stderr, " -- WARNING: RZ_BLOCK_SIZE is not %d, treat source as gz file. in %s -- %s:%d --\n", RZ_BLOCK_SIZE, __FUNCTION__, __FILE__, __LINE__);
318 rz->load_index = _load_index;
319 rz->file_type = FILE_TYPE_RZ;
320 if(lseek(fd, -16, SEEK_END) == -1){
324 rz->src_end = rz->end = 0x7FFFFFFFFFFFFFFFLL;
326 is_be = is_big_endian();
328 read(fd, &end, sizeof(int64_t));
329 if(!is_be) rz->src_end = (int64_t)byte_swap_8((uint64_t)end);
330 else rz->src_end = end;
331 read(fd, &end, sizeof(int64_t));
332 if(!is_be) rz->end = (int64_t)byte_swap_8((uint64_t)end);
335 rz->stream->avail_in -= n - rz->end;
338 if(rz->end > rz->src_end){
339 lseek(fd, rz->in, SEEK_SET);
342 if(lseek(fd, rz->end, SEEK_SET) != rz->end){
343 lseek(fd, rz->in, SEEK_SET);
347 lseek(fd, n, SEEK_SET);
352 RAZF* razf_dopen(int fd, const char *mode){
353 if(strcasecmp(mode, "r") == 0) return razf_open_r(fd, 1);
354 else if(strcasecmp(mode, "w") == 0) return razf_open_w(fd);
358 RAZF* razf_dopen2(int fd, const char *mode)
360 if(strcasecmp(mode, "r") == 0) return razf_open_r(fd, 0);
361 else if(strcasecmp(mode, "w") == 0) return razf_open_w(fd);
365 static inline RAZF* _razf_open(const char *filename, const char *mode, int _load_index){
368 if(strcasecmp(mode, "r") == 0){
369 fd = open(filename, O_RDONLY);
370 rz = razf_open_r(fd, _load_index);
371 } else if(strcasecmp(mode, "w") == 0){
372 fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0644);
373 rz = razf_open_w(fd);
378 RAZF* razf_open(const char *filename, const char *mode){
379 return _razf_open(filename, mode, 1);
382 RAZF* razf_open2(const char *filename, const char *mode){
383 return _razf_open(filename, mode, 0);
386 int razf_get_data_size(RAZF *rz, int64_t *u_size, int64_t *c_size){
388 if(rz->mode != 'r' && rz->mode != 'R') return 0;
389 switch(rz->file_type){
390 case FILE_TYPE_PLAIN:
391 if(rz->end == 0x7fffffffffffffffLL){
392 if((n = lseek(rz->filedes, 0, SEEK_CUR)) == -1) return 0;
393 rz->end = lseek(rz->filedes, 0, SEEK_END);
394 lseek(rz->filedes, n, SEEK_SET);
396 *u_size = *c_size = rz->end;
401 if(rz->src_end == rz->end) return 0;
402 *u_size = rz->src_end;
410 static int _razf_read(RAZF* rz, void *data, int size){
412 if(rz->z_eof || rz->z_err) return 0;
413 if (rz->file_type == FILE_TYPE_PLAIN) {
414 ret = read(rz->filedes, data, size);
415 if (ret == 0) rz->z_eof = 1;
418 rz->stream->avail_out = size;
419 rz->stream->next_out = data;
420 while(rz->stream->avail_out){
421 if(rz->stream->avail_in == 0){
422 if(rz->in >= rz->end){ rz->z_eof = 1; break; }
423 if(rz->end - rz->in < RZ_BUFFER_SIZE){
424 rz->stream->avail_in = read(rz->filedes, rz->inbuf, rz->end -rz->in);
426 rz->stream->avail_in = read(rz->filedes, rz->inbuf, RZ_BUFFER_SIZE);
428 if(rz->stream->avail_in == 0){
432 rz->stream->next_in = rz->inbuf;
434 tin = rz->stream->avail_in;
435 ret = inflate(rz->stream, Z_BLOCK);
436 rz->in += tin - rz->stream->avail_in;
437 if(ret == Z_NEED_DICT || ret == Z_MEM_ERROR || ret == Z_DATA_ERROR){
438 fprintf(stderr, "[_razf_read] inflate error: %d (at %s:%d)\n", ret, __FILE__, __LINE__);
442 if(ret == Z_STREAM_END){
446 if ((rz->stream->data_type&128) && !(rz->stream->data_type&64)){
448 rz->next_block_pos = rz->in;
452 return size - rz->stream->avail_out;
455 int razf_read(RAZF *rz, void *data, int size){
460 if(size < rz->buf_len){
461 for(i=0;i<size;i++) ((char*)data)[i] = ((char*)rz->outbuf + rz->buf_off)[i];
465 rz->block_off += size;
469 for(i=0;i<rz->buf_len;i++) ((char*)data)[i] = ((char*)rz->outbuf + rz->buf_off)[i];
472 rz->block_off += rz->buf_len;
476 rz->block_pos = rz->next_block_pos;
481 } else if(rz->buf_flush){
482 rz->block_pos = rz->next_block_pos;
486 if(rz->buf_flush) continue;
487 rz->buf_len = _razf_read(rz, rz->outbuf, RZ_BUFFER_SIZE);
488 if(rz->z_eof && rz->buf_len == 0) break;
490 rz->out += ori_size - size;
491 return ori_size - size;
494 int razf_skip(RAZF* rz, int size){
499 if(size < rz->buf_len){
502 rz->block_off += size;
509 rz->block_off += rz->buf_len;
511 rz->block_pos = rz->next_block_pos;
516 } else if(rz->buf_flush){
517 rz->block_pos = rz->next_block_pos;
521 if(rz->buf_flush) continue;
522 rz->buf_len = _razf_read(rz, rz->outbuf, RZ_BUFFER_SIZE);
525 rz->out += ori_size - size;
526 return ori_size - size;
529 static void _razf_reset_read(RAZF *rz, int64_t in, int64_t out){
530 lseek(rz->filedes, in, SEEK_SET);
534 rz->next_block_pos = in;
537 rz->z_eof = rz->z_err = 0;
538 inflateReset(rz->stream);
539 rz->stream->avail_in = 0;
540 rz->buf_off = rz->buf_len = 0;
543 int64_t razf_jump(RAZF *rz, int64_t block_start, int block_offset){
546 if(rz->file_type == FILE_TYPE_PLAIN){
547 rz->buf_off = rz->buf_len = 0;
548 pos = block_start + block_offset;
549 pos = lseek(rz->filedes, pos, SEEK_SET);
550 rz->out = rz->in = pos;
553 if(block_start == rz->block_pos && block_offset >= rz->block_off) {
554 block_offset -= rz->block_off;
555 goto SKIP; // Needn't reset inflate
557 if(block_start == 0) block_start = rz->header_size; // Automaticly revist wrong block_start
558 _razf_reset_read(rz, block_start, 0);
560 if(block_offset) razf_skip(rz, block_offset);
561 return rz->block_off;
564 int64_t razf_seek(RAZF* rz, int64_t pos, int where){
566 int64_t seek_pos, new_out;
568 if (where == SEEK_CUR) pos += rz->out;
569 else if (where == SEEK_END) pos += rz->src_end;
570 if(rz->file_type == FILE_TYPE_PLAIN){
571 seek_pos = lseek(rz->filedes, pos, SEEK_SET);
572 rz->buf_off = rz->buf_len = 0;
573 rz->out = rz->in = seek_pos;
575 } else if(rz->file_type == FILE_TYPE_GZ){
576 if(pos >= rz->out) goto SKIP;
579 if(pos == rz->out) return pos;
580 if(pos > rz->src_end) return rz->out;
581 if(!rz->seekable || !rz->load_index){
582 if(pos >= rz->out) goto SKIP;
584 idx = pos / RZ_BLOCK_SIZE - 1;
585 seek_pos = (idx < 0)? rz->header_size:(rz->index->cell_offsets[idx] + rz->index->bin_offsets[idx / RZ_BIN_SIZE]);
586 new_out = (idx + 1) * RZ_BLOCK_SIZE;
587 if(pos > rz->out && new_out <= rz->out) goto SKIP;
588 _razf_reset_read(rz, seek_pos, new_out);
590 razf_skip(rz, (int)(pos - rz->out));
594 uint64_t razf_tell2(RAZF *rz)
597 if (rz->load_index) {
598 int64_t idx, seek_pos;
599 idx = rz->out / RZ_BLOCK_SIZE - 1;
600 seek_pos = (idx < 0)? rz->header_size:(rz->index->cell_offsets[idx] + rz->index->bin_offsets[idx / RZ_BIN_SIZE]);
601 if (seek_pos != rz->block_pos || rz->out%RZ_BLOCK_SIZE != rz->block_off)
602 fprintf(stderr, "[razf_tell2] inconsistent block offset: (%lld, %lld) != (%lld, %lld)\n",
603 (long long)seek_pos, (long long)rz->out%RZ_BLOCK_SIZE, (long long)rz->block_pos, (long long) rz->block_off);
606 return (uint64_t)rz->block_pos<<16 | (rz->block_off&0xffff);
609 int64_t razf_seek2(RAZF *rz, uint64_t voffset, int where)
611 if (where != SEEK_SET) return -1;
612 return razf_jump(rz, voffset>>16, voffset&0xffff);
615 void razf_close(RAZF *rz){
619 deflateEnd(rz->stream);
620 save_zindex(rz, rz->filedes);
622 write(rz->filedes, &rz->in, sizeof(int64_t));
623 write(rz->filedes, &rz->out, sizeof(int64_t));
625 v64 = byte_swap_8((uint64_t)rz->in);
626 write(rz->filedes, &v64, sizeof(int64_t));
627 v64 = byte_swap_8((uint64_t)rz->out);
628 write(rz->filedes, &v64, sizeof(int64_t));
630 } else if(rz->mode == 'r'){
631 if(rz->stream) inflateEnd(rz->stream);
633 if(rz->inbuf) free(rz->inbuf);
634 if(rz->outbuf) free(rz->outbuf);
636 free(rz->header->extra);
637 free(rz->header->name);
638 free(rz->header->comment);
642 free(rz->index->bin_offsets);
643 free(rz->index->cell_offsets);