From d619466092e2d1b2c89c346f69d863875fd8ff8d Mon Sep 17 00:00:00 2001 From: derek Date: Wed, 9 Nov 2011 23:58:20 -0500 Subject: [PATCH] Added generic I/O device to BamIndex side of things * Remote BAM access (now w/ random access) seems to be working with the simple test cases so far * Major TODO: not yet implemented for Windows --- src/api/IBamIODevice.h | 5 +- src/api/internal/index/BamIndexFactory_p.cpp | 12 +- src/api/internal/index/BamStandardIndex_p.cpp | 123 ++++++++------- src/api/internal/index/BamStandardIndex_p.h | 12 +- src/api/internal/index/BamToolsIndex_p.cpp | 149 +++++++++++++----- src/api/internal/index/BamToolsIndex_p.h | 10 +- src/api/internal/io/BamFile_p.cpp | 6 +- src/api/internal/io/BamFile_p.h | 2 +- src/api/internal/io/BamFtp_p.cpp | 11 +- src/api/internal/io/BamFtp_p.h | 2 +- src/api/internal/io/BamHttp_p.cpp | 14 +- src/api/internal/io/BamHttp_p.h | 2 +- src/api/internal/io/BamPipe_p.cpp | 2 +- src/api/internal/io/BamPipe_p.h | 2 +- src/api/internal/io/HostInfo_p.cpp | 8 - 15 files changed, 225 insertions(+), 135 deletions(-) diff --git a/src/api/IBamIODevice.h b/src/api/IBamIODevice.h index 8e14827..8a74696 100644 --- a/src/api/IBamIODevice.h +++ b/src/api/IBamIODevice.h @@ -19,6 +19,7 @@ #define IBAMIODEVICE_H #include "api/api_global.h" +#include #include namespace BamTools { @@ -39,12 +40,14 @@ class API_EXPORT IBamIODevice { // IBamIODevice interface public: + // TODO: add seek(pos, *from*) + // pure virtuals virtual void Close(void) =0; virtual bool IsRandomAccess(void) const =0; virtual bool Open(const OpenMode mode) =0; virtual int64_t Read(char* data, const unsigned int numBytes) =0; - virtual bool Seek(const int64_t& position) =0; + virtual bool Seek(const int64_t& position, const int origin = SEEK_SET) =0; virtual int64_t Tell(void) const =0; virtual int64_t Write(const char* data, const unsigned int numBytes) =0; diff --git a/src/api/internal/index/BamIndexFactory_p.cpp b/src/api/internal/index/BamIndexFactory_p.cpp index 3afcbb9..c5399b2 100644 --- a/src/api/internal/index/BamIndexFactory_p.cpp +++ b/src/api/internal/index/BamIndexFactory_p.cpp @@ -31,9 +31,9 @@ const string BamIndexFactory::CreateIndexFilename(const string& bamFilename, // creates a new BamIndex object, depending on extension of @indexFilename BamIndex* BamIndexFactory::CreateIndexFromFilename(const string& indexFilename, BamReaderPrivate* reader) { - // if file doesn't exist, return null index - if ( !BamTools::FileExists(indexFilename) ) - return 0; +// // if file doesn't exist, return null index +// if ( !BamTools::FileExists(indexFilename) ) +// return 0; // get file extension from index filename, including dot (".EXT") // if can't get file extension, return null index @@ -91,19 +91,19 @@ const string BamIndexFactory::FindIndexFilename(const string& bamFilename, // try to find index of preferred type first // return index filename if found string indexFilename = CreateIndexFilename(bamFilename, preferredType); - if ( !indexFilename.empty() && BamTools::FileExists(indexFilename) ) + if ( !indexFilename.empty() /*&& BamTools::FileExists(indexFilename)*/ ) return indexFilename; // couldn't find preferred type, try the other supported types // return index filename if found if ( preferredType != BamIndex::STANDARD ) { indexFilename = CreateIndexFilename(bamFilename, BamIndex::STANDARD); - if ( !indexFilename.empty() && BamTools::FileExists(indexFilename) ) + if ( !indexFilename.empty() /*&& BamTools::FileExists(indexFilename)*/ ) return indexFilename; } if ( preferredType != BamIndex::BAMTOOLS ) { indexFilename = CreateIndexFilename(bamFilename, BamIndex::BAMTOOLS); - if ( !indexFilename.empty() && BamTools::FileExists(indexFilename) ) + if ( !indexFilename.empty()/* && BamTools::FileExists(indexFilename) */) return indexFilename; } diff --git a/src/api/internal/index/BamStandardIndex_p.cpp b/src/api/internal/index/BamStandardIndex_p.cpp index 706c7c1..56c92f9 100644 --- a/src/api/internal/index/BamStandardIndex_p.cpp +++ b/src/api/internal/index/BamStandardIndex_p.cpp @@ -10,6 +10,7 @@ #include "api/BamAlignment.h" #include "api/internal/bam/BamReader_p.h" #include "api/internal/index/BamStandardIndex_p.h" +#include "api/internal/io/BamDeviceFactory_p.h" #include "api/internal/utils/BamException_p.h" using namespace BamTools; using namespace BamTools::Internal; @@ -38,15 +39,16 @@ const int BamStandardIndex::SIZEOF_LINEAROFFSET = sizeof(uint64_t); // ---------------------------- BamStandardIndex::RaiiWrapper::RaiiWrapper(void) - : IndexStream(0) + : Device(0) , Buffer(0) { } BamStandardIndex::RaiiWrapper::~RaiiWrapper(void) { - if ( IndexStream ) { - fclose(IndexStream); - IndexStream = 0; + if ( Device ) { + Device->Close(); + delete Device; + Device = 0; } if ( Buffer ) { @@ -145,9 +147,9 @@ void BamStandardIndex::CalculateCandidateOffsets(const BaiReferenceSummary& refS for ( int j = 0; j < numAlignmentChunks; ++j ) { // read chunk start & stop from buffer - memcpy((char*)&chunkStart, Resources.Buffer+offset, sizeof(uint64_t)); + memcpy((char*)&chunkStart, m_resources.Buffer+offset, sizeof(uint64_t)); offset += sizeof(uint64_t); - memcpy((char*)&chunkStop, Resources.Buffer+offset, sizeof(uint64_t)); + memcpy((char*)&chunkStop, m_resources.Buffer+offset, sizeof(uint64_t)); offset += sizeof(uint64_t); // swap endian-ness if necessary @@ -226,8 +228,8 @@ void BamStandardIndex::CheckMagicNumber(void) { // check 'magic number' to see if file is BAI index char magic[4]; - const size_t elementsRead = fread(magic, sizeof(char), 4, Resources.IndexStream); - if ( elementsRead != 4 ) + const int64_t numBytesRead = m_resources.Device->Read(magic, sizeof(magic)); + if ( numBytesRead != 4 ) throw BamException("BamStandardIndex::CheckMagicNumber", "could not read BAI magic number"); // compare to expected value @@ -244,17 +246,18 @@ void BamStandardIndex::ClearReferenceEntry(BaiReferenceEntry& refEntry) { void BamStandardIndex::CloseFile(void) { // close file stream - if ( IsFileOpen() ) { - fclose(Resources.IndexStream); - Resources.IndexStream = 0; + if ( IsDeviceOpen() ) { + m_resources.Device->Close(); + delete m_resources.Device; + m_resources.Device = 0; } // clear index file summary data m_indexFileSummary.clear(); // clean up I/O buffer - delete[] Resources.Buffer; - Resources.Buffer = 0; + delete[] m_resources.Buffer; + m_resources.Buffer = 0; m_bufferLength = 0; } @@ -279,7 +282,7 @@ bool BamStandardIndex::Create(void) { // open new index file (read & write) string indexFilename = m_reader->Filename() + Extension(); - OpenFile(indexFilename, "w+b"); + OpenFile(indexFilename, IBamIODevice::ReadWrite); // initialize BaiFileSummary with number of references const int& numReferences = m_reader->GetReferenceCount(); @@ -498,8 +501,10 @@ bool BamStandardIndex::HasAlignments(const int& referenceID) const { return ( refSummary.NumBins > 0 ); } -bool BamStandardIndex::IsFileOpen(void) const { - return ( Resources.IndexStream != 0 ); +bool BamStandardIndex::IsDeviceOpen(void) const { + if ( m_resources.Device == 0 ) + return false; + return m_resources.Device->IsOpen(); } // attempts to use index data to jump to @region, returns success/fail @@ -541,7 +546,7 @@ bool BamStandardIndex::Load(const std::string& filename) { try { // attempt to open file (read-only) - OpenFile(filename, "rb"); + OpenFile(filename, IBamIODevice::ReadOnly); // validate format CheckMagicNumber(); @@ -612,23 +617,29 @@ void BamStandardIndex::MergeAlignmentChunks(BaiAlignmentChunkVector& chunks) { chunks = mergedChunks; } -void BamStandardIndex::OpenFile(const std::string& filename, const char* mode) { +void BamStandardIndex::OpenFile(const std::string& filename, IBamIODevice::OpenMode mode) { // make sure any previous index file is closed CloseFile(); + m_resources.Device = BamDeviceFactory::CreateDevice(filename); + if ( m_resources.Device == 0 ) { + const string message = string("could not open file: ") + filename; + throw BamException("BamStandardIndex::OpenFile", message); + } + // attempt to open file - Resources.IndexStream = fopen(filename.c_str(), mode); - if ( !IsFileOpen() ) { + m_resources.Device->Open(mode); + if ( !IsDeviceOpen() ) { const string message = string("could not open file: ") + filename; throw BamException("BamStandardIndex::OpenFile", message); } } void BamStandardIndex::ReadBinID(uint32_t& binId) { - const size_t elementsRead = fread(&binId, sizeof(binId), 1, Resources.IndexStream); + const int64_t numBytesRead = m_resources.Device->Read((char*)&binId, sizeof(binId)); if ( m_isBigEndian ) SwapEndian_32(binId); - if ( elementsRead != 1 ) + if ( numBytesRead != sizeof(binId) ) throw BamException("BamStandardIndex::ReadBinID", "could not read BAI bin ID"); } @@ -646,11 +657,11 @@ void BamStandardIndex::ReadBinIntoBuffer(uint32_t& binId, int32_t& numAlignmentC void BamStandardIndex::ReadIntoBuffer(const unsigned int& bytesRequested) { // ensure that our buffer is big enough for request - BamStandardIndex::CheckBufferSize(Resources.Buffer, m_bufferLength, bytesRequested); + BamStandardIndex::CheckBufferSize(m_resources.Buffer, m_bufferLength, bytesRequested); // read from BAI file stream - const size_t bytesRead = fread( Resources.Buffer, sizeof(char), bytesRequested, Resources.IndexStream ); - if ( bytesRead != (size_t)bytesRequested ) { + const int64_t bytesRead = m_resources.Device->Read(m_resources.Buffer, bytesRequested); + if ( bytesRead != (int64_t)bytesRequested ) { stringstream s(""); s << "expected to read: " << bytesRequested << " bytes, " << "but instead read: " << bytesRead; @@ -659,37 +670,37 @@ void BamStandardIndex::ReadIntoBuffer(const unsigned int& bytesRequested) { } void BamStandardIndex::ReadLinearOffset(uint64_t& linearOffset) { - const size_t elementsRead = fread(&linearOffset, sizeof(linearOffset), 1, Resources.IndexStream); + const int64_t numBytesRead = m_resources.Device->Read((char*)&linearOffset, sizeof(linearOffset)); if ( m_isBigEndian ) SwapEndian_64(linearOffset); - if ( elementsRead != 1 ) + if ( numBytesRead != sizeof(linearOffset) ) throw BamException("BamStandardIndex::ReadLinearOffset", "could not read BAI linear offset"); } void BamStandardIndex::ReadNumAlignmentChunks(int& numAlignmentChunks) { - const size_t elementsRead = fread(&numAlignmentChunks, sizeof(numAlignmentChunks), 1, Resources.IndexStream); + const int64_t numBytesRead = m_resources.Device->Read((char*)&numAlignmentChunks, sizeof(numAlignmentChunks)); if ( m_isBigEndian ) SwapEndian_32(numAlignmentChunks); - if ( elementsRead != 1 ) + if ( numBytesRead != sizeof(numAlignmentChunks) ) throw BamException("BamStandardIndex::ReadNumAlignmentChunks", "could not read BAI chunk count"); } void BamStandardIndex::ReadNumBins(int& numBins) { - const size_t elementsRead = fread(&numBins, sizeof(numBins), 1, Resources.IndexStream); + const int64_t numBytesRead = m_resources.Device->Read((char*)&numBins, sizeof(numBins)); if ( m_isBigEndian ) SwapEndian_32(numBins); - if ( elementsRead != 1 ) + if ( numBytesRead != sizeof(numBins) ) throw BamException("BamStandardIndex::ReadNumBins", "could not read BAI bin count"); } void BamStandardIndex::ReadNumLinearOffsets(int& numLinearOffsets) { - const size_t elementsRead = fread(&numLinearOffsets, sizeof(numLinearOffsets), 1, Resources.IndexStream); + const int64_t numBytesRead = m_resources.Device->Read((char*)&numLinearOffsets, sizeof(numLinearOffsets)); if ( m_isBigEndian ) SwapEndian_32(numLinearOffsets); - if ( elementsRead != 1 ) + if ( numBytesRead != sizeof(numLinearOffsets) ) throw BamException("BamStandardIndex::ReadNumAlignmentChunks", "could not read BAI linear offset count"); } void BamStandardIndex::ReadNumReferences(int& numReferences) { - const size_t elementsRead = fread(&numReferences, sizeof(numReferences), 1, Resources.IndexStream); + const int64_t numBytesRead = m_resources.Device->Read((char*)&numReferences, sizeof(numReferences)); if ( m_isBigEndian ) SwapEndian_32(numReferences); - if ( elementsRead != 1 ) + if ( numBytesRead != sizeof(numReferences) ) throw BamException("BamStandardIndex::ReadNumReferences", "could not read reference count"); } @@ -756,8 +767,8 @@ void BamStandardIndex::SaveLinearOffsetsSummary(const int& refId, const int& num } // seek to position in index file stream -void BamStandardIndex::Seek(const int64_t& position, const int& origin) { - if ( fseek64(Resources.IndexStream, position, origin) != 0 ) +void BamStandardIndex::Seek(const int64_t& position, const int origin) { + if ( !m_resources.Device->Seek(position, origin) ) throw BamException("BamStandardIndex::Seek", "could not seek in BAI file"); } @@ -828,7 +839,7 @@ void BamStandardIndex::SummarizeReference(BaiReferenceSummary& refSummary) { // return position of file pointer in index file stream int64_t BamStandardIndex::Tell(void) const { - return ftell64(Resources.IndexStream); + return m_resources.Device->Tell(); } void BamStandardIndex::WriteAlignmentChunk(const BaiAlignmentChunk& chunk) { @@ -844,10 +855,10 @@ void BamStandardIndex::WriteAlignmentChunk(const BaiAlignmentChunk& chunk) { } // write to index file - size_t elementsWritten = 0; - elementsWritten += fwrite(&start, sizeof(start), 1, Resources.IndexStream); - elementsWritten += fwrite(&stop, sizeof(stop), 1, Resources.IndexStream); - if ( elementsWritten != 2 ) + int64_t numBytesWritten = 0; + numBytesWritten += m_resources.Device->Write((const char*)&start, sizeof(start)); + numBytesWritten += m_resources.Device->Write((const char*)&stop, sizeof(stop)); + if ( numBytesWritten != (sizeof(start)+sizeof(stop)) ) throw BamException("BamStandardIndex::WriteAlignmentChunk", "could not write BAI alignment chunk"); } @@ -859,8 +870,8 @@ void BamStandardIndex::WriteAlignmentChunks(BaiAlignmentChunkVector& chunks) { // write chunks int32_t chunkCount = chunks.size(); if ( m_isBigEndian ) SwapEndian_32(chunkCount); - const size_t elementsWritten = fwrite(&chunkCount, sizeof(chunkCount), 1, Resources.IndexStream); - if ( elementsWritten != 1 ) + const int64_t numBytesWritten = m_resources.Device->Write((const char*)&chunkCount, sizeof(chunkCount)); + if ( numBytesWritten != sizeof(chunkCount) ) throw BamException("BamStandardIndex::WriteAlignmentChunks", "could not write BAI chunk count"); // iterate over chunks @@ -875,8 +886,8 @@ void BamStandardIndex::WriteBin(const uint32_t& binId, BaiAlignmentChunkVector& // write BAM bin ID uint32_t binKey = binId; if ( m_isBigEndian ) SwapEndian_32(binKey); - const size_t elementsWritten = fwrite(&binKey, sizeof(binKey), 1, Resources.IndexStream); - if ( elementsWritten != 1 ) + const int64_t numBytesWritten = m_resources.Device->Write((const char*)&binKey, sizeof(binKey)); + if ( numBytesWritten != sizeof(binKey) ) throw BamException("BamStandardIndex::WriteBin", "could not write bin ID"); // write bin's alignment chunks @@ -888,8 +899,8 @@ void BamStandardIndex::WriteBins(const int& refId, BaiBinMap& bins) { // write number of bins int32_t binCount = bins.size(); if ( m_isBigEndian ) SwapEndian_32(binCount); - const size_t elementsWritten = fwrite(&binCount, sizeof(binCount), 1, Resources.IndexStream); - if ( elementsWritten != 1 ) + const int64_t numBytesWritten = m_resources.Device->Write((const char*)&binCount, sizeof(binCount)); + if ( numBytesWritten != sizeof(binCount) ) throw BamException("BamStandardIndex::WriteBins", "could not write bin count"); // save summary for reference's bins @@ -904,17 +915,17 @@ void BamStandardIndex::WriteBins(const int& refId, BaiBinMap& bins) { void BamStandardIndex::WriteHeader(void) { - size_t elementsWritten = 0; + int64_t numBytesWritten = 0; // write magic number - elementsWritten += fwrite(BamStandardIndex::BAI_MAGIC, sizeof(char), 4, Resources.IndexStream); + numBytesWritten += m_resources.Device->Write(BamStandardIndex::BAI_MAGIC, 4); // write number of reference sequences int32_t numReferences = m_indexFileSummary.size(); if ( m_isBigEndian ) SwapEndian_32(numReferences); - elementsWritten += fwrite(&numReferences, sizeof(numReferences), 1, Resources.IndexStream); + numBytesWritten += m_resources.Device->Write((const char*)&numReferences, sizeof(numReferences)); - if ( elementsWritten != 5 ) + if ( numBytesWritten != sizeof(numReferences)+4 ) throw BamException("BamStandardIndex::WriteHeader", "could not write BAI header"); } @@ -923,12 +934,12 @@ void BamStandardIndex::WriteLinearOffsets(const int& refId, BaiLinearOffsetVecto // make sure linear offsets are sorted before writing & saving summary SortLinearOffsets(linearOffsets); - size_t elementsWritten = 0; + int64_t numBytesWritten = 0; // write number of linear offsets int32_t offsetCount = linearOffsets.size(); if ( m_isBigEndian ) SwapEndian_32(offsetCount); - elementsWritten += fwrite(&offsetCount, sizeof(offsetCount), 1, Resources.IndexStream); + numBytesWritten += m_resources.Device->Write((const char*)&offsetCount, sizeof(offsetCount)); // save summary for reference's linear offsets SaveLinearOffsetsSummary(refId, linearOffsets.size()); @@ -941,10 +952,10 @@ void BamStandardIndex::WriteLinearOffsets(const int& refId, BaiLinearOffsetVecto // write linear offset uint64_t linearOffset = (*offsetIter); if ( m_isBigEndian ) SwapEndian_64(linearOffset); - elementsWritten += fwrite(&linearOffset, sizeof(linearOffset), 1, Resources.IndexStream); + numBytesWritten += m_resources.Device->Write((const char*)&linearOffset, sizeof(linearOffset)); } - if ( elementsWritten != (linearOffsets.size() + 1) ) + if ( numBytesWritten != (sizeof(offsetCount) + linearOffsets.size()*sizeof(uint64_t)) ) throw BamException("BamStandardIndex::WriteLinearOffsets", "could not write BAI linear offsets"); } diff --git a/src/api/internal/index/BamStandardIndex_p.h b/src/api/internal/index/BamStandardIndex_p.h index 03e0042..dfa81ee 100644 --- a/src/api/internal/index/BamStandardIndex_p.h +++ b/src/api/internal/index/BamStandardIndex_p.h @@ -22,6 +22,7 @@ #include "api/BamAux.h" #include "api/BamIndex.h" +#include "api/IBamIODevice.h" #include #include #include @@ -132,9 +133,9 @@ class BamStandardIndex : public BamIndex { // index file ops void CheckMagicNumber(void); void CloseFile(void); - bool IsFileOpen(void) const; - void OpenFile(const std::string& filename, const char* mode); - void Seek(const int64_t& position, const int& origin); + bool IsDeviceOpen(void) const; + void OpenFile(const std::string& filename, IBamIODevice::OpenMode mode); + void Seek(const int64_t& position, const int origin); int64_t Tell(void) const; // BAI index building methods @@ -200,14 +201,13 @@ class BamStandardIndex : public BamIndex { // our input buffer unsigned int m_bufferLength; - struct RaiiWrapper { - FILE* IndexStream; + IBamIODevice* Device; char* Buffer; RaiiWrapper(void); ~RaiiWrapper(void); }; - RaiiWrapper Resources; + RaiiWrapper m_resources; // static methods private: diff --git a/src/api/internal/index/BamToolsIndex_p.cpp b/src/api/internal/index/BamToolsIndex_p.cpp index af0a684..2d05eaa 100644 --- a/src/api/internal/index/BamToolsIndex_p.cpp +++ b/src/api/internal/index/BamToolsIndex_p.cpp @@ -10,6 +10,7 @@ #include "api/BamAlignment.h" #include "api/internal/bam/BamReader_p.h" #include "api/internal/index/BamToolsIndex_p.h" +#include "api/internal/io/BamDeviceFactory_p.h" #include "api/internal/io/BgzfStream_p.h" #include "api/internal/utils/BamException_p.h" using namespace BamTools; @@ -39,11 +40,20 @@ const int BamToolsIndex::SIZEOF_BLOCK = sizeof(int32_t)*2 + sizeof(int64 BamToolsIndex::RaiiWrapper::RaiiWrapper(void) : IndexStream(0) + , Device(0) { } BamToolsIndex::RaiiWrapper::~RaiiWrapper(void) { - if ( IndexStream ) + if ( IndexStream ) { fclose(IndexStream); + IndexStream = 0; + } + + if ( Device ) { + Device->Close(); + delete Device; + Device = 0; + } } // ------------------------------ @@ -69,8 +79,8 @@ void BamToolsIndex::CheckMagicNumber(void) { // read magic number char magic[4]; - size_t elementsRead = fread(magic, sizeof(char), 4, Resources.IndexStream); - if ( elementsRead != 4 ) + const int64_t numBytesRead = m_resources.Device->Read(magic, 4); + if ( numBytesRead != 4 ) throw BamException("BamToolsIndex::CheckMagicNumber", "could not read BTI magic number"); // validate expected magic number @@ -82,8 +92,10 @@ void BamToolsIndex::CheckMagicNumber(void) { void BamToolsIndex::CheckVersion(void) { // read version from file - size_t elementsRead = fread(&m_inputVersion, sizeof(m_inputVersion), 1, Resources.IndexStream); - if ( elementsRead != 1 ) + const int64_t numBytesRead = m_resources.Device->Read((char*)&m_inputVersion, sizeof(m_inputVersion)); +// size_t elementsRead = fread(&m_inputVersion, sizeof(m_inputVersion), 1, m_resources.IndexStream); +// if ( elementsRead != 1 ) + if ( numBytesRead != sizeof(m_inputVersion) ) throw BamException("BamToolsIndex::CheckVersion", "could not read format version"); if ( m_isBigEndian ) SwapEndian_32(m_inputVersion); @@ -118,9 +130,13 @@ void BamToolsIndex::ClearReferenceEntry(BtiReferenceEntry& refEntry) { } void BamToolsIndex::CloseFile(void) { - if ( IsFileOpen() ) { - fclose(Resources.IndexStream); - Resources.IndexStream = 0; + if ( IsDeviceOpen() ) { + fclose(m_resources.IndexStream); + m_resources.IndexStream = 0; + + m_resources.Device->Close(); + delete m_resources.Device; + m_resources.Device = 0; } m_indexFileSummary.clear(); } @@ -145,7 +161,7 @@ bool BamToolsIndex::Create(void) { try { // open new index file (read & write) const string indexFilename = m_reader->Filename() + Extension(); - OpenFile(indexFilename, "w+b"); + OpenFile(indexFilename, IBamIODevice::ReadWrite); // initialize BtiFileSummary with number of references const int& numReferences = m_reader->GetReferenceCount(); @@ -355,8 +371,11 @@ void BamToolsIndex::InitializeFileSummary(const int& numReferences) { } // returns true if the index stream is open -bool BamToolsIndex::IsFileOpen(void) const { - return ( Resources.IndexStream != 0 ); +bool BamToolsIndex::IsDeviceOpen(void) const { + if ( m_resources.Device == 0 ) + return false; + return m_resources.Device->IsOpen(); +// return ( m_resources.IndexStream != 0 ); } // attempts to use index data to jump to @region, returns success/fail @@ -400,7 +419,7 @@ bool BamToolsIndex::Load(const std::string& filename) { try { // attempt to open file (read-only) - OpenFile(filename, "rb"); + OpenFile(filename, IBamIODevice::ReadOnly); // load metadata & generate in-memory summary LoadHeader(); @@ -438,23 +457,29 @@ void BamToolsIndex::LoadHeader(void) { CheckVersion(); // use file's BTI block size to set member variable - const size_t elementsRead = fread(&m_blockSize, sizeof(m_blockSize), 1, Resources.IndexStream); + const int64_t numBytesRead = m_resources.Device->Read((char*)&m_blockSize, sizeof(m_blockSize)); +// const size_t elementsRead = fread(&m_blockSize, sizeof(m_blockSize), 1, m_resources.IndexStream); if ( m_isBigEndian ) SwapEndian_32(m_blockSize); - if ( elementsRead != 1 ) +// if ( elementsRead != 1 ) + if ( numBytesRead != sizeof(m_blockSize) ) throw BamException("BamToolsIndex::LoadHeader", "could not read BTI block size"); } void BamToolsIndex::LoadNumBlocks(int& numBlocks) { - const size_t elementsRead = fread(&numBlocks, sizeof(numBlocks), 1, Resources.IndexStream); + const int64_t numBytesRead = m_resources.Device->Read((char*)&numBlocks, sizeof(numBlocks)); +// const size_t elementsRead = fread(&numBlocks, sizeof(numBlocks), 1, m_resources.IndexStream); if ( m_isBigEndian ) SwapEndian_32(numBlocks); - if ( elementsRead != 1 ) +// if ( elementsRead != 1 ) + if ( numBytesRead != sizeof(numBlocks) ) throw BamException("BamToolsIndex::LoadNumBlocks", "could not read number of BTI blocks"); } void BamToolsIndex::LoadNumReferences(int& numReferences) { - const size_t elementsRead = fread(&numReferences, sizeof(numReferences), 1, Resources.IndexStream); + const int64_t numBytesRead = m_resources.Device->Read((char*)&numReferences, sizeof(numReferences)); +// const size_t elementsRead = fread(&numReferences, sizeof(numReferences), 1, m_resources.IndexStream); if ( m_isBigEndian ) SwapEndian_32(numReferences); - if ( elementsRead != 1 ) +// if ( elementsRead != 1 ) + if ( numBytesRead != sizeof(numReferences) ) throw BamException("BamToolsIndex::LoadNumReferences", "could not read number of references"); } @@ -472,14 +497,21 @@ void BamToolsIndex::LoadReferenceSummary(BtiReferenceSummary& refSummary) { SkipBlocks(numBlocks); } -void BamToolsIndex::OpenFile(const std::string& filename, const char* mode) { +void BamToolsIndex::OpenFile(const std::string& filename, IBamIODevice::OpenMode mode) { // make sure any previous index file is closed CloseFile(); + m_resources.Device = BamDeviceFactory::CreateDevice(filename); + if ( m_resources.Device == 0 ) { + const string message = string("could not open file: ") + filename; + throw BamException("BamStandardIndex::OpenFile", message); + } + // attempt to open file - Resources.IndexStream = fopen(filename.c_str(), mode); - if ( !IsFileOpen() ) { +// m_resources.IndexStream = fopen(filename.c_str(), mode); + m_resources.Device->Open(mode); + if ( !IsDeviceOpen() ) { const string message = string("could not open file: ") + filename; throw BamException("BamToolsIndex::OpenFile", message); } @@ -488,10 +520,15 @@ void BamToolsIndex::OpenFile(const std::string& filename, const char* mode) { void BamToolsIndex::ReadBlock(BtiBlock& block) { // read in block data members - size_t elementsRead = 0; - elementsRead += fread(&block.MaxEndPosition, sizeof(block.MaxEndPosition), 1, Resources.IndexStream); - elementsRead += fread(&block.StartOffset, sizeof(block.StartOffset), 1, Resources.IndexStream); - elementsRead += fread(&block.StartPosition, sizeof(block.StartPosition), 1, Resources.IndexStream); +// size_t elementsRead = 0; +// elementsRead += fread(&block.MaxEndPosition, sizeof(block.MaxEndPosition), 1, m_resources.IndexStream); +// elementsRead += fread(&block.StartOffset, sizeof(block.StartOffset), 1, m_resources.IndexStream); +// elementsRead += fread(&block.StartPosition, sizeof(block.StartPosition), 1, m_resources.IndexStream); + + int64_t numBytesRead = 0; + numBytesRead += m_resources.Device->Read((char*)&block.MaxEndPosition, sizeof(block.MaxEndPosition)); + numBytesRead += m_resources.Device->Read((char*)&block.StartOffset, sizeof(block.StartOffset)); + numBytesRead += m_resources.Device->Read((char*)&block.StartPosition, sizeof(block.StartPosition)); // swap endian-ness if necessary if ( m_isBigEndian ) { @@ -500,7 +537,11 @@ void BamToolsIndex::ReadBlock(BtiBlock& block) { SwapEndian_32(block.StartPosition); } - if ( elementsRead != 3 ) +// if ( elementsRead != 3 ) + const int expectedBytes = sizeof(block.MaxEndPosition) + + sizeof(block.StartOffset) + + sizeof(block.StartPosition); + if ( numBytesRead != expectedBytes ) throw BamException("BamToolsIndex::ReadBlock", "could not read block"); } @@ -532,8 +573,9 @@ void BamToolsIndex::ReadReferenceEntry(BtiReferenceEntry& refEntry) { ReadBlocks(refSummary, refEntry.Blocks); } -void BamToolsIndex::Seek(const int64_t& position, const int& origin) { - if ( fseek64(Resources.IndexStream, position, origin) != 0 ) +void BamToolsIndex::Seek(const int64_t& position, const int origin) { + if ( !m_resources.Device->Seek(position, origin) ) +// if ( fseek64(m_resources.IndexStream, position, origin) != 0 ) throw BamException("BamToolsIndex::Seek", "could not seek in BAI file"); } @@ -542,7 +584,8 @@ void BamToolsIndex::SkipBlocks(const int& numBlocks) { } int64_t BamToolsIndex::Tell(void) const { - return ftell64(Resources.IndexStream); + return m_resources.Device->Tell(); +// return ftell64(m_resources.IndexStream); } void BamToolsIndex::WriteBlock(const BtiBlock& block) { @@ -560,11 +603,20 @@ void BamToolsIndex::WriteBlock(const BtiBlock& block) { } // write the reference index entry - size_t elementsWritten = 0; - elementsWritten += fwrite(&maxEndPosition, sizeof(maxEndPosition), 1, Resources.IndexStream); - elementsWritten += fwrite(&startOffset, sizeof(startOffset), 1, Resources.IndexStream); - elementsWritten += fwrite(&startPosition, sizeof(startPosition), 1, Resources.IndexStream); - if ( elementsWritten != 3 ) + int64_t numBytesWritten = 0; + numBytesWritten += m_resources.Device->Write((const char*)&maxEndPosition, sizeof(maxEndPosition)); + numBytesWritten += m_resources.Device->Write((const char*)&startOffset, sizeof(startOffset)); + numBytesWritten += m_resources.Device->Write((const char*)&startPosition, sizeof(startPosition)); + const int expectedBytes = sizeof(maxEndPosition) + + sizeof(startOffset) + + sizeof(startPosition); + +// size_t elementsWritten = 0; +// elementsWritten += fwrite(&maxEndPosition, sizeof(maxEndPosition), 1, m_resources.IndexStream); +// elementsWritten += fwrite(&startOffset, sizeof(startOffset), 1, m_resources.IndexStream); +// elementsWritten += fwrite(&startPosition, sizeof(startPosition), 1, m_resources.IndexStream); +// if ( elementsWritten != 3 ) + if ( numBytesWritten != expectedBytes ) throw BamException("BamToolsIndex::WriteBlock", "could not write BTI block"); } @@ -577,27 +629,38 @@ void BamToolsIndex::WriteBlocks(const BtiBlockVector& blocks) { void BamToolsIndex::WriteHeader(void) { - size_t elementsWritten = 0; + int64_t numBytesWritten = 0 ; +// size_t elementsWritten = 0; // write BTI index format 'magic number' - elementsWritten += fwrite(BamToolsIndex::BTI_MAGIC, 1, 4, Resources.IndexStream); + numBytesWritten += m_resources.Device->Write(BamToolsIndex::BTI_MAGIC, 4); +// elementsWritten += fwrite(BamToolsIndex::BTI_MAGIC, 1, 4, m_resources.IndexStream); // write BTI index format version int32_t currentVersion = (int32_t)m_outputVersion; if ( m_isBigEndian ) SwapEndian_32(currentVersion); - elementsWritten += fwrite(¤tVersion, sizeof(currentVersion), 1, Resources.IndexStream); + numBytesWritten += m_resources.Device->Write((const char*)¤tVersion, sizeof(currentVersion)); +// elementsWritten += fwrite(¤tVersion, sizeof(currentVersion), 1, m_resources.IndexStream); // write block size uint32_t blockSize = m_blockSize; if ( m_isBigEndian ) SwapEndian_32(blockSize); - elementsWritten += fwrite(&blockSize, sizeof(blockSize), 1, Resources.IndexStream); + numBytesWritten += m_resources.Device->Write((const char*)&blockSize, sizeof(blockSize)); +// elementsWritten += fwrite(&blockSize, sizeof(blockSize), 1, m_resources.IndexStream); // write number of references int32_t numReferences = m_indexFileSummary.size(); if ( m_isBigEndian ) SwapEndian_32(numReferences); - elementsWritten += fwrite(&numReferences, sizeof(numReferences), 1, Resources.IndexStream); + numBytesWritten += m_resources.Device->Write((const char*)&numReferences, sizeof(numReferences)); +// elementsWritten += fwrite(&numReferences, sizeof(numReferences), 1, m_resources.IndexStream); + + const int expectedBytes = 4 + + sizeof(currentVersion) + + sizeof(blockSize) + + sizeof(numReferences); - if ( elementsWritten != 7 ) +// if ( elementsWritten != 7 ) + if ( numBytesWritten != expectedBytes ) throw BamException("BamToolsIndex::WriteHeader", "could not write BTI header"); } @@ -606,8 +669,10 @@ void BamToolsIndex::WriteReferenceEntry(const BtiReferenceEntry& refEntry) { // write number of blocks this reference uint32_t numBlocks = refEntry.Blocks.size(); if ( m_isBigEndian ) SwapEndian_32(numBlocks); - const size_t elementsWritten = fwrite(&numBlocks, sizeof(numBlocks), 1, Resources.IndexStream); - if ( elementsWritten != 1 ) + const int64_t numBytesWritten = m_resources.Device->Write((const char*)&numBlocks, sizeof(numBlocks)); +// const size_t elementsWritten = fwrite(&numBlocks, sizeof(numBlocks), 1, m_resources.IndexStream); +// if ( elementsWritten != 1 ) + if ( numBytesWritten != sizeof(numBlocks) ) throw BamException("BamToolsIndex::WriteReferenceEntry", "could not write number of blocks"); // write actual block entries diff --git a/src/api/internal/index/BamToolsIndex_p.h b/src/api/internal/index/BamToolsIndex_p.h index 7c1550b..f6ffb72 100644 --- a/src/api/internal/index/BamToolsIndex_p.h +++ b/src/api/internal/index/BamToolsIndex_p.h @@ -22,6 +22,7 @@ #include "api/BamAux.h" #include "api/BamIndex.h" +#include "api/IBamIODevice.h" #include #include #include @@ -128,9 +129,9 @@ class BamToolsIndex : public BamIndex { void CheckMagicNumber(void); void CheckVersion(void); void CloseFile(void); - bool IsFileOpen(void) const; - void OpenFile(const std::string& filename, const char* mode); - void Seek(const int64_t& position, const int& origin); + bool IsDeviceOpen(void) const; + void OpenFile(const std::string& filename, IBamIODevice::OpenMode mode); + void Seek(const int64_t& position, const int origin); int64_t Tell(void) const; // index-creation methods @@ -165,10 +166,11 @@ class BamToolsIndex : public BamIndex { struct RaiiWrapper { FILE* IndexStream; + IBamIODevice* Device; RaiiWrapper(void); ~RaiiWrapper(void); }; - RaiiWrapper Resources; + RaiiWrapper m_resources; // static constants private: diff --git a/src/api/internal/io/BamFile_p.cpp b/src/api/internal/io/BamFile_p.cpp index 94c919e..2bb0715 100644 --- a/src/api/internal/io/BamFile_p.cpp +++ b/src/api/internal/io/BamFile_p.cpp @@ -43,6 +43,8 @@ bool BamFile::Open(const IBamIODevice::OpenMode mode) { m_stream = fopen(m_filename.c_str(), "rb"); else if ( mode == IBamIODevice::WriteOnly ) m_stream = fopen(m_filename.c_str(), "wb"); + else if ( mode == IBamIODevice::ReadWrite ) + m_stream = fopen(m_filename.c_str(), "w+b"); else { SetErrorString("BamFile::Open", "unknown open mode requested"); return false; @@ -61,7 +63,7 @@ bool BamFile::Open(const IBamIODevice::OpenMode mode) { return true; } -bool BamFile::Seek(const int64_t& position) { +bool BamFile::Seek(const int64_t& position, const int origin) { BT_ASSERT_X( m_stream, "BamFile::Seek() - null stream" ); - return ( fseek64(m_stream, position, SEEK_SET) == 0 ); + return ( fseek64(m_stream, position, origin) == 0 ); } diff --git a/src/api/internal/io/BamFile_p.h b/src/api/internal/io/BamFile_p.h index bd7d64b..dd93894 100644 --- a/src/api/internal/io/BamFile_p.h +++ b/src/api/internal/io/BamFile_p.h @@ -38,7 +38,7 @@ class BamFile : public ILocalIODevice { void Close(void); bool IsRandomAccess(void) const; bool Open(const IBamIODevice::OpenMode mode); - bool Seek(const int64_t& position); + bool Seek(const int64_t& position, const int origin = SEEK_SET); // data members private: diff --git a/src/api/internal/io/BamFtp_p.cpp b/src/api/internal/io/BamFtp_p.cpp index 31195bd..c6d3bf8 100644 --- a/src/api/internal/io/BamFtp_p.cpp +++ b/src/api/internal/io/BamFtp_p.cpp @@ -418,7 +418,7 @@ bool BamFtp::ReceiveReply(void) { return true; } -bool BamFtp::Seek(const int64_t& position) { +bool BamFtp::Seek(const int64_t& position, const int origin) { // if FTP device not in a valid state if ( !IsOpen() ) { @@ -434,7 +434,14 @@ bool BamFtp::Seek(const int64_t& position) { m_commandSocket->DisconnectFromHost(); // update file position & return success - m_filePosition = position; + if ( origin == SEEK_CUR ) + m_filePosition += position; + else if ( origin == SEEK_SET) + m_filePosition = position; + else { + // TODO: set error string + return false; + } return true; } diff --git a/src/api/internal/io/BamFtp_p.h b/src/api/internal/io/BamFtp_p.h index d9e02fb..cedf716 100644 --- a/src/api/internal/io/BamFtp_p.h +++ b/src/api/internal/io/BamFtp_p.h @@ -42,7 +42,7 @@ class BamFtp : public IBamIODevice { bool IsRandomAccess(void) const; bool Open(const IBamIODevice::OpenMode mode); int64_t Read(char* data, const unsigned int numBytes); - bool Seek(const int64_t& position); + bool Seek(const int64_t& position, const int origin = SEEK_SET); int64_t Tell(void) const; int64_t Write(const char* data, const unsigned int numBytes); diff --git a/src/api/internal/io/BamHttp_p.cpp b/src/api/internal/io/BamHttp_p.cpp index d889db4..30d2c14 100644 --- a/src/api/internal/io/BamHttp_p.cpp +++ b/src/api/internal/io/BamHttp_p.cpp @@ -333,7 +333,7 @@ bool BamHttp::ReceiveResponse(void) { return false; } -bool BamHttp::Seek(const int64_t& position) { +bool BamHttp::Seek(const int64_t& position, const int origin) { // if HTTP device not in a valid state if ( !IsOpen() ) { @@ -343,8 +343,16 @@ bool BamHttp::Seek(const int64_t& position) { // discard socket's buffer contents, update positions, & return success m_socket->ClearBuffer(); - m_filePosition = position; - m_endRangeFilePosition = position; + + if ( origin == SEEK_CUR ) + m_filePosition += position; + else if ( origin == SEEK_SET ) + m_filePosition = position; + else { + // TODO: set error string + return false; + } + m_endRangeFilePosition = m_filePosition; return true; } diff --git a/src/api/internal/io/BamHttp_p.h b/src/api/internal/io/BamHttp_p.h index c3d9502..fae9302 100644 --- a/src/api/internal/io/BamHttp_p.h +++ b/src/api/internal/io/BamHttp_p.h @@ -44,7 +44,7 @@ class BamHttp : public IBamIODevice { bool IsRandomAccess(void) const; bool Open(const IBamIODevice::OpenMode mode); int64_t Read(char* data, const unsigned int numBytes); - bool Seek(const int64_t& position); + bool Seek(const int64_t& position, const int origin = SEEK_SET); int64_t Tell(void) const; int64_t Write(const char* data, const unsigned int numBytes); diff --git a/src/api/internal/io/BamPipe_p.cpp b/src/api/internal/io/BamPipe_p.cpp index 92cf798..d700189 100644 --- a/src/api/internal/io/BamPipe_p.cpp +++ b/src/api/internal/io/BamPipe_p.cpp @@ -55,7 +55,7 @@ bool BamPipe::Open(const IBamIODevice::OpenMode mode) { return true; } -bool BamPipe::Seek(const int64_t& ) { +bool BamPipe::Seek(const int64_t&, const int) { SetErrorString("BamPipe::Seek", "random access not allowed in FIFO pipe"); return false; } diff --git a/src/api/internal/io/BamPipe_p.h b/src/api/internal/io/BamPipe_p.h index 8e4e4c4..115cab9 100644 --- a/src/api/internal/io/BamPipe_p.h +++ b/src/api/internal/io/BamPipe_p.h @@ -37,7 +37,7 @@ class BamPipe : public ILocalIODevice { public: bool IsRandomAccess(void) const; bool Open(const IBamIODevice::OpenMode mode); - bool Seek(const int64_t& position); + bool Seek(const int64_t& position, const int origin = SEEK_SET); }; } // namespace Internal diff --git a/src/api/internal/io/HostInfo_p.cpp b/src/api/internal/io/HostInfo_p.cpp index 693b2f2..1431fb8 100644 --- a/src/api/internal/io/HostInfo_p.cpp +++ b/src/api/internal/io/HostInfo_p.cpp @@ -9,8 +9,6 @@ using namespace BamTools::Internal; # include "api/internal/io/NetUnix_p.h" #endif -#include // debug - // standard C++ includes #include #include @@ -141,8 +139,6 @@ HostInfo HostInfo::Lookup(const string& hostname, const string& port) { // do 'normal' lookup else { - cout << "HostInfo::Lookup() - looking up addresses for domain name: " << hostname << endl; - // setup address lookup 'hints' addrinfo hints; memset(&hints, 0, sizeof(hints)); @@ -157,8 +153,6 @@ HostInfo HostInfo::Lookup(const string& hostname, const string& port) { // if everything OK if ( status == 0 ) { - cout << "HostInfo::Lookup() - found addresses" << endl; - // iterate over all IP addresses found addrinfo* p = res; for ( ; p != NULL; p = p->ai_next ) { @@ -167,7 +161,6 @@ HostInfo HostInfo::Lookup(const string& hostname, const string& port) { if ( p->ai_family == AF_INET ) { sockaddr_in* ipv4 = (sockaddr_in*)p->ai_addr; HostAddress a( ntohl(ipv4->sin_addr.s_addr) ); - cout << "\t" << a.GetIPString() << endl; uniqueAddresses.insert(a); } @@ -175,7 +168,6 @@ HostInfo HostInfo::Lookup(const string& hostname, const string& port) { else if ( p->ai_family == AF_INET6 ) { sockaddr_in6* ipv6 = (sockaddr_in6*)p->ai_addr; HostAddress a(ipv6->sin6_addr.s6_addr); - cout << "\t" << a.GetIPString() << endl; uniqueAddresses.insert(a); } } -- 2.39.2