From 4d3ecf0d35c014b176345d21383343e7a58d2445 Mon Sep 17 00:00:00 2001 From: Sarah Westcott Date: Wed, 1 Aug 2012 11:08:39 -0400 Subject: [PATCH] paralellized chop.seqs --- binsequencecommand.cpp | 111 +++++++---------- chopseqscommand.cpp | 263 +++++++++++++++++++++++++++++++++++------ chopseqscommand.h | 223 +++++++++++++++++++++++++++++++++- clusterclassic.cpp | 30 ++--- trimseqscommand.cpp | 2 +- 5 files changed, 503 insertions(+), 126 deletions(-) diff --git a/binsequencecommand.cpp b/binsequencecommand.cpp index 7569a4b..0c867e3 100644 --- a/binsequencecommand.cpp +++ b/binsequencecommand.cpp @@ -362,79 +362,48 @@ void BinSeqCommand::readNamesFile() { //return 1 if error, 0 otherwise int BinSeqCommand::process(ListVector* list) { try { - string binnames, name, sequence; - string outputFileName = outputDir + m->getRootName(m->getSimpleName(listfile)) + list->getLabel() + getOutputFileNameTag("fasta"); m->openOutputFile(outputFileName, out); - - //save to output list of output file names - outputNames.push_back(outputFileName); outputTypes["fasta"].push_back(outputFileName); - - m->mothurOut(list->getLabel()); m->mothurOutEndLine(); - - //for each bin in the list vector - for (int i = 0; i < list->size(); i++) { - - if (m->control_pressed) { return 1; } - - binnames = list->get(i); - while (binnames.find_first_of(',') != -1) { - name = binnames.substr(0,binnames.find_first_of(',')); - binnames = binnames.substr(binnames.find_first_of(',')+1, binnames.length()); - - //do work for that name - sequence = fasta->getSequence(name); - if (sequence != "not found") { - //if you don't have groups - if (groupfile == "") { - name = name + "\t" + toString(i+1); - out << ">" << name << endl; - out << sequence << endl; - }else {//if you do have groups - string group = groupMap->getGroup(name); - if (group == "not found") { - m->mothurOut(name + " is missing from your group file. Please correct. "); m->mothurOutEndLine(); - return 1; - }else{ - name = name + "\t" + group + "\t" + toString(i+1); - out << ">" << name << endl; - out << sequence << endl; - } - } - }else { - m->mothurOut(name + " is missing from your fasta or name file. Please correct. "); m->mothurOutEndLine(); - return 1; - } - - } - - //get last name - sequence = fasta->getSequence(binnames); - if (sequence != "not found") { - //if you don't have groups - if (groupfile == "") { - binnames = binnames + "\t" + toString(i+1); - out << ">" << binnames << endl; - out << sequence << endl; - }else {//if you do have groups - string group = groupMap->getGroup(binnames); - if (group == "not found") { - m->mothurOut(binnames + " is missing from your group file. Please correct. "); m->mothurOutEndLine(); - return 1; - }else{ - binnames = binnames + "\t" + group + "\t" + toString(i+1); - out << ">" << binnames << endl; - out << sequence << endl; - } - } - }else { - m->mothurOut(binnames + " is missing from your fasta or name file. Please correct. "); m->mothurOutEndLine(); - return 1; - } - } - - out.close(); - return 0; + outputNames.push_back(outputFileName); outputTypes["fasta"].push_back(outputFileName); + + m->mothurOut(list->getLabel()); m->mothurOutEndLine(); + + //for each bin in the list vector + for (int i = 0; i < list->size(); i++) { + + if (m->control_pressed) { return 1; } + + string binnames = list->get(i); + vector names; + m->splitAtComma(binnames, names); + for (int j = 0; j < names.size(); j++) { + string name = names[j]; + + //do work for that name + string sequence = fasta->getSequence(name); + if (sequence != "not found") { + //if you don't have groups + if (groupfile == "") { + name = name + "\t" + toString(i+1); + out << ">" << name << endl; + out << sequence << endl; + }else {//if you do have groups + string group = groupMap->getGroup(name); + if (group == "not found") { + m->mothurOut(name + " is missing from your group file. Please correct. "); m->mothurOutEndLine(); + return 1; + }else{ + name = name + "\t" + group + "\t" + toString(i+1); + out << ">" << name << endl; + out << sequence << endl; + } + } + }else { m->mothurOut(name + " is missing from your fasta or name file. Please correct. "); m->mothurOutEndLine(); return 1; } + } + } + + out.close(); + return 0; } catch(exception& e) { diff --git a/chopseqscommand.cpp b/chopseqscommand.cpp index 05037f6..4bcd707 100644 --- a/chopseqscommand.cpp +++ b/chopseqscommand.cpp @@ -14,7 +14,8 @@ vector ChopSeqsCommand::setParameters(){ try { CommandParameter pfasta("fasta", "InputTypes", "", "", "none", "none", "none",false,true); parameters.push_back(pfasta); - CommandParameter pnumbases("numbases", "Number", "", "0", "", "", "",false,true); parameters.push_back(pnumbases); + CommandParameter pprocessors("processors", "Number", "", "1", "", "", "",false,false); parameters.push_back(pprocessors); + CommandParameter pnumbases("numbases", "Number", "", "0", "", "", "",false,true); parameters.push_back(pnumbases); CommandParameter pcountgaps("countgaps", "Boolean", "", "F", "", "", "",false,false); parameters.push_back(pcountgaps); CommandParameter pshort("short", "Boolean", "", "F", "", "", "",false,false); parameters.push_back(pshort); CommandParameter pkeep("keep", "Multiple", "front-back", "front", "", "", "",false,false); parameters.push_back(pkeep); @@ -41,7 +42,8 @@ string ChopSeqsCommand::getHelpString(){ helpString += "The keep parameter allows you to specify whether you want to keep the front or the back of your sequence, default=front.\n"; helpString += "The countgaps parameter allows you to specify whether you want to count gaps as bases, default=false.\n"; helpString += "The short parameter allows you to specify you want to keep sequences that are too short to chop, default=false.\n"; - helpString += "For example, if you ran chop.seqs with numbases=200 and short=t, if a sequence had 100 bases mothur would keep the sequence rather than eliminate it.\n"; + helpString += "The processors parameter allows you to specify how many processors you would like to use. The default is 1. \n"; + helpString += "For example, if you ran chop.seqs with numbases=200 and short=t, if a sequence had 100 bases mothur would keep the sequence rather than eliminate it.\n"; helpString += "Example chop.seqs(fasta=amazon.fasta, numbases=200, keep=front).\n"; helpString += "Note: No spaces between parameter labels (i.e. fasta), '=' and parameters (i.e.yourFasta).\n"; return helpString; @@ -143,6 +145,10 @@ ChopSeqsCommand::ChopSeqsCommand(string option) { string temp = validParameter.validFile(parameters, "numbases", false); if (temp == "not found") { temp = "0"; } m->mothurConvert(temp, numbases); + temp = validParameter.validFile(parameters, "processors", false); if (temp == "not found"){ temp = m->getProcessors(); } + m->setProcessors(temp); + m->mothurConvert(temp, processors); + temp = validParameter.validFile(parameters, "countgaps", false); if (temp == "not found") { temp = "f"; } countGaps = m->isTrue(temp); @@ -169,39 +175,32 @@ int ChopSeqsCommand::execute(){ string outputFileName = outputDir + m->getRootName(m->getSimpleName(fastafile)) + getOutputFileNameTag("fasta"); string outputFileNameAccnos = outputDir + m->getRootName(m->getSimpleName(fastafile)) + getOutputFileNameTag("accnos"); + + + vector positions; + vector lines; +#if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix) + positions = m->divideFile(fastafile, processors); + for (int i = 0; i < (positions.size()-1); i++) { lines.push_back(linePair(positions[i], positions[(i+1)])); } +#else + int numSeqs = 0; + positions = m->setFilePosFasta(fastafile, numSeqs); + if (positions.size() < processors) { processors = positions.size(); } - ofstream out; - m->openOutputFile(outputFileName, out); - - ofstream outAcc; - m->openOutputFile(outputFileNameAccnos, outAcc); - - ifstream in; - m->openInputFile(fastafile, in); - - bool wroteAccnos = false; - - while (!in.eof()) { - - Sequence seq(in); - - if (m->control_pressed) { outputTypes.clear(); in.close(); out.close(); outAcc.close(); m->mothurRemove(outputFileName); m->mothurRemove(outputFileNameAccnos); return 0; } - - if (seq.getName() != "") { - string newSeqString = getChopped(seq); - - //output trimmed sequence - if (newSeqString != "") { - out << ">" << seq.getName() << endl << newSeqString << endl; - }else{ - outAcc << seq.getName() << endl; - wroteAccnos = true; - } - } - } - in.close(); - out.close(); - outAcc.close(); + //figure out how many sequences you have to process + int numSeqsPerProcessor = numSeqs / processors; + for (int i = 0; i < processors; i++) { + int startIndex = i * numSeqsPerProcessor; + if(i == (processors - 1)){ numSeqsPerProcessor = numSeqs - i * numSeqsPerProcessor; } + lines.push_back(linePair(positions[startIndex], numSeqsPerProcessor)); + } +#endif + + bool wroteAccnos = false; + if(processors == 1) { wroteAccnos = driver(lines[0], fastafile, outputFileName, outputFileNameAccnos); } + else { wroteAccnos = createProcesses(lines, fastafile, outputFileName, outputFileNameAccnos); } + + if (m->control_pressed) { return 0; } m->mothurOutEndLine(); m->mothurOut("Output File Name: "); m->mothurOutEndLine(); @@ -235,6 +234,202 @@ int ChopSeqsCommand::execute(){ exit(1); } } +/**************************************************************************************************/ +bool ChopSeqsCommand::createProcesses(vector lines, string filename, string outFasta, string outAccnos) { + try { + int process = 1; + bool wroteAccnos = false; + vector processIDS; + vector nonBlankAccnosFiles; + +#if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix) + + //loop through and create all the processes you want + while (process != processors) { + int pid = fork(); + + if (pid > 0) { + processIDS.push_back(pid); //create map from line number to pid so you can append files in correct order later + process++; + }else if (pid == 0){ + wroteAccnos = driver(lines[process], filename, outFasta + toString(getpid()) + ".temp", outAccnos + toString(getpid()) + ".temp"); + + //pass numSeqs to parent + ofstream out; + string tempFile = fastafile + toString(getpid()) + ".bool.temp"; + m->openOutputFile(tempFile, out); + out << wroteAccnos << endl; + out.close(); + + exit(0); + }else { + m->mothurOut("[ERROR]: unable to spawn the necessary processes."); m->mothurOutEndLine(); + for (int i = 0; i < processIDS.size(); i++) { kill (processIDS[i], SIGINT); } + exit(0); + } + } + + //do your part + wroteAccnos = driver(lines[0], filename, outFasta, outAccnos); + + //force parent to wait until all the processes are done + for (int i=0;imothurRemove(outAccnos); } //remove so other files can be renamed to it + + //parent reads in and combine Filter info + for (int i = 0; i < processIDS.size(); i++) { + string tempFilename = fastafile + toString(processIDS[i]) + ".bool.temp"; + ifstream in; + m->openInputFile(tempFilename, in); + + bool temp; + in >> temp; m->gobble(in); + if (temp) { wroteAccnos = temp; nonBlankAccnosFiles.push_back(outAccnos + toString(processIDS[i]) + ".temp"); } + else { m->mothurRemove((outAccnos + toString(processIDS[i]) + ".temp")); } + + in.close(); + m->mothurRemove(tempFilename); + } +#else + ////////////////////////////////////////////////////////////////////////////////////////////////////// + //Windows version shared memory, so be careful when passing variables through the seqSumData struct. + //Above fork() will clone, so memory is separate, but that's not the case with windows, + //Taking advantage of shared memory to allow both threads to add info to vectors. + ////////////////////////////////////////////////////////////////////////////////////////////////////// + + vector pDataArray; + DWORD dwThreadIdArray[processors-1]; + HANDLE hThreadArray[processors-1]; + + //Create processor worker threads. + for( int i=0; imothurRemove(outAccnos); } //remove so other files can be renamed to it + + //Close all thread handles and free memory allocations. + for(int i=0; i < pDataArray.size(); i++){ + if (pDataArray[i]->wroteAccnos) { wroteAccnos = pDataArray[i]->wroteAccnos; nonBlankAccnosFiles.push_back(outAccnos + toString(processIDS[i]) + ".temp"); } + else { m->mothurRemove((outAccnos + toString(processIDS[i]) + ".temp")); } + CloseHandle(hThreadArray[i]); + delete pDataArray[i]; + } +#endif + + for (int i = 0; i < processIDS.size(); i++) { + m->appendFiles((outFasta + toString(processIDS[i]) + ".temp"), outFasta); + m->mothurRemove((outFasta + toString(processIDS[i]) + ".temp")); + } + + if (nonBlankAccnosFiles.size() != 0) { + m->renameFile(nonBlankAccnosFiles[0], outAccnos); + + for (int h=1; h < nonBlankAccnosFiles.size(); h++) { + m->appendFiles(nonBlankAccnosFiles[h], outAccnos); + m->mothurRemove(nonBlankAccnosFiles[h]); + } + }else { //recreate the accnosfile if needed + ofstream out; + m->openOutputFile(outAccnos, out); + out.close(); + } + + return wroteAccnos; + } + catch(exception& e) { + m->errorOut(e, "ChopSeqsCommand", "createProcesses"); + exit(1); + } +} +/**************************************************************************************/ +bool ChopSeqsCommand::driver(linePair filePos, string filename, string outFasta, string outAccnos) { + try { + + ofstream out; + m->openOutputFile(outFasta, out); + + ofstream outAcc; + m->openOutputFile(outAccnos, outAcc); + + ifstream in; + m->openInputFile(filename, in); + + in.seekg(filePos.start); + + bool done = false; + bool wroteAccnos = false; + int count = 0; + + while (!done) { + + if (m->control_pressed) { in.close(); out.close(); return 1; } + + Sequence seq(in); m->gobble(in); + + if (m->control_pressed) { in.close(); out.close(); outAcc.close(); m->mothurRemove(outFasta); m->mothurRemove(outAccnos); return 0; } + + if (seq.getName() != "") { + string newSeqString = getChopped(seq); + + //output trimmed sequence + if (newSeqString != "") { + out << ">" << seq.getName() << endl << newSeqString << endl; + }else{ + outAcc << seq.getName() << endl; + wroteAccnos = true; + } + count++; + } + +#if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix) + unsigned long long pos = in.tellg(); + if ((pos == -1) || (pos >= filePos.end)) { break; } +#else + if (in.eof()) { break; } +#endif + //report progress + if((count) % 1000 == 0){ m->mothurOut(toString(count)); m->mothurOutEndLine(); } + + } + //report progress + if((count) % 1000 != 0){ m->mothurOut(toString(count)); m->mothurOutEndLine(); } + + + in.close(); + out.close(); + outAcc.close(); + + return wroteAccnos; + } + catch(exception& e) { + m->errorOut(e, "ChopSeqsCommand", "driver"); + exit(1); + } +} //********************************************************************************************************************** string ChopSeqsCommand::getChopped(Sequence seq) { try { diff --git a/chopseqscommand.h b/chopseqscommand.h index cc22c75..fa3f559 100644 --- a/chopseqscommand.h +++ b/chopseqscommand.h @@ -34,14 +34,235 @@ class ChopSeqsCommand : public Command { void help() { m->mothurOut(getHelpString()); } private: + struct linePair { + unsigned long long start; + unsigned long long end; + linePair(unsigned long long i, unsigned long long j) : start(i), end(j) {} + }; + string fastafile, outputDir, keep; bool abort, countGaps, Short; - int numbases; + int numbases, processors; vector outputNames; string getChopped(Sequence); + bool driver (linePair, string, string, string); + bool createProcesses(vector, string, string, string); }; +/**************************************************************************************************/ +//custom data structure for threads to use. +// This is passed by void pointer so it can be any data type +// that can be passed using a single void pointer (LPVOID). +struct chopData { + string filename; + string outFasta, outAccnos, keep; + unsigned long long start; + unsigned long long end; + int numbases; + bool countGaps, Short, wroteAccnos; + MothurOut* m; + string namefile; + map nameMap; + + + chopData(){} + chopData(string f, string ff, string a, MothurOut* mout, unsigned long long st, unsigned long long en, string k, bool cGaps, int nbases, bool S) { + filename = f; + outFasta = ff; + outAccnos = a; + m = mout; + start = st; + end = en; + keep = k; + countGaps = cGaps; + numbases = nbases; + Short = S; + wroteAccnos = false; + } +}; + +/**************************************************************************************************/ +#if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix) +#else +static DWORD WINAPI MyChopThreadFunction(LPVOID lpParam){ + chopData* pDataArray; + pDataArray = (chopData*)lpParam; + + try { + ofstream out; + pDataArray->m->openOutputFile(pDataArray->outFasta, out); + + ofstream outAcc; + pDataArray->m->openOutputFile(pDataArray->outAccnos, outAcc); + + ifstream in; + pDataArray->m->openInputFile(pDataArray->filename, in); + + if ((pDataArray->start == 0) || (pDataArray->start == 1)) { + in.seekg(0); + }else { //this accounts for the difference in line endings. + in.seekg(pDataArray->start-1); pDataArray->m->gobble(in); + } + + bool done = false; + bool wroteAccnos = false; + int count = 0; + + for(int i = 0; i < pDataArray->end; i++){ //end is the number of sequences to process + + if (pDataArray->m->control_pressed) { in.close(); out.close(); outAcc.close(); pDataArray->m->mothurRemove(pDataArray->outFasta); pDataArray->m->mothurRemove(pDataArray->outAccnos); return 0; } + + Sequence seq(in); pDataArray->m->gobble(in); + + if (seq.getName() != "") { + //string newSeqString = getChopped(seq); + /////////////////////////////////////////////////////////////////////// + string temp = seq.getAligned(); + string tempUnaligned = seq.getUnaligned(); + + if (pDataArray->countGaps) { + //if needed trim sequence + if (pDataArray->keep == "front") {//you want to keep the beginning + int tempLength = temp.length(); + + if (tempLength > pDataArray->numbases) { //you have enough bases to remove some + + int stopSpot = 0; + int numBasesCounted = 0; + + for (int i = 0; i < temp.length(); i++) { + //eliminate N's + if (toupper(temp[i]) == 'N') { temp[i] = '.'; } + + numBasesCounted++; + + if (numBasesCounted >= pDataArray->numbases) { stopSpot = i; break; } + } + + if (stopSpot == 0) { temp = ""; } + else { temp = temp.substr(0, stopSpot+1); } + + }else { + if (!pDataArray->Short) { temp = ""; } //sequence too short + } + }else { //you are keeping the back + int tempLength = temp.length(); + if (tempLength > pDataArray->numbases) { //you have enough bases to remove some + + int stopSpot = 0; + int numBasesCounted = 0; + + for (int i = (temp.length()-1); i >= 0; i--) { + //eliminate N's + if (toupper(temp[i]) == 'N') { temp[i] = '.'; } + + numBasesCounted++; + + if (numBasesCounted >= pDataArray->numbases) { stopSpot = i; break; } + } + + if (stopSpot == 0) { temp = ""; } + else { temp = temp.substr(stopSpot+1); } + }else { + if (!pDataArray->Short) { temp = ""; } //sequence too short + } + } + + }else{ + + //if needed trim sequence + if (pDataArray->keep == "front") {//you want to keep the beginning + int tempLength = tempUnaligned.length(); + + if (tempLength > pDataArray->numbases) { //you have enough bases to remove some + + int stopSpot = 0; + int numBasesCounted = 0; + + for (int i = 0; i < temp.length(); i++) { + //eliminate N's + if (toupper(temp[i]) == 'N') { + temp[i] = '.'; + tempLength--; + if (tempLength < pDataArray->numbases) { stopSpot = 0; break; } + } + + if(isalpha(temp[i])) { numBasesCounted++; } + + if (numBasesCounted >= pDataArray->numbases) { stopSpot = i; break; } + } + + if (stopSpot == 0) { temp = ""; } + else { temp = temp.substr(0, stopSpot+1); } + + }else { + if (!pDataArray->Short) { temp = ""; } //sequence too short + } + }else { //you are keeping the back + int tempLength = tempUnaligned.length(); + if (tempLength > pDataArray->numbases) { //you have enough bases to remove some + + int stopSpot = 0; + int numBasesCounted = 0; + + for (int i = (temp.length()-1); i >= 0; i--) { + //eliminate N's + if (toupper(temp[i]) == 'N') { + temp[i] = '.'; + tempLength--; + if (tempLength < pDataArray->numbases) { stopSpot = 0; break; } + } + + if(isalpha(temp[i])) { numBasesCounted++; } + + if (numBasesCounted >= pDataArray->numbases) { stopSpot = i; break; } + } + + if (stopSpot == 0) { temp = ""; } + else { temp = temp.substr(stopSpot); } + }else { + if (!pDataArray->Short) { temp = ""; } //sequence too short + } + } + } + + string newSeqString = temp; + /////////////////////////////////////////////////////////////////////// + + //output trimmed sequence + if (newSeqString != "") { + out << ">" << seq.getName() << endl << newSeqString << endl; + }else{ + outAcc << seq.getName() << endl; + pDataArray->wroteAccnos = true; + } + count++; + } + //report progress + if((count) % 1000 == 0){ pDataArray->m->mothurOut(toString(count)); pDataArray->m->mothurOutEndLine(); } + + } + //report progress + if((count) % 1000 != 0){ pDataArray->m->mothurOut(toString(count)); pDataArray->m->mothurOutEndLine(); } + + + in.close(); + out.close(); + outAcc.close(); + + return 0; + + } + catch(exception& e) { + pDataArray->m->errorOut(e, "ChopsSeqsCommand", "MyChopThreadFunction"); + exit(1); + } +} +#endif + + + #endif diff --git a/clusterclassic.cpp b/clusterclassic.cpp index ba1f7f6..32a9341 100644 --- a/clusterclassic.cpp +++ b/clusterclassic.cpp @@ -597,16 +597,12 @@ void ClusterClassic::setMapWanted(bool f) { //parse bin string names = list->get(i); - while (names.find_first_of(',') != -1) { - //get name from bin - string name = names.substr(0,names.find_first_of(',')); + vector binnames; + m->splitAtComma(names, binnames); + for (int j = 0; j < binnames.size(); j++) { //save name and bin number - seq2Bin[name] = i; - names = names.substr(names.find_first_of(',')+1, names.length()); + seq2Bin[binnames[j]] = i; } - - //get last name - seq2Bin[names] = i; } } @@ -619,17 +615,13 @@ void ClusterClassic::setMapWanted(bool f) { void ClusterClassic::updateMap() { try { //update location of seqs in smallRow since they move to smallCol now - string names = list->get(smallRow); - while (names.find_first_of(',') != -1) { - //get name from bin - string name = names.substr(0,names.find_first_of(',')); - //save name and bin number - seq2Bin[name] = smallCol; - names = names.substr(names.find_first_of(',')+1, names.length()); - } - - //get last name - seq2Bin[names] = smallCol; + string names = list->get(smallRow); + vector binnames; + m->splitAtComma(names, binnames); + for (int j = 0; j < binnames.size(); j++) { + //save name and bin number + seq2Bin[binnames[j]] = smallCol; + } } catch(exception& e) { diff --git a/trimseqscommand.cpp b/trimseqscommand.cpp index 0036769..637a720 100644 --- a/trimseqscommand.cpp +++ b/trimseqscommand.cpp @@ -1332,7 +1332,7 @@ bool TrimSeqsCommand::getOligos(vector >& fastaFileNames, vector< }else if(type == "SPACER"){ spacer.push_back(oligo); } - else{ m->mothurOut(type + " is not recognized as a valid type. Choices are forward, reverse, and barcode. Ignoring " + oligo + "."); m->mothurOutEndLine(); } + else{ m->mothurOut("[WARNING]: " + type + " is not recognized as a valid type. Choices are forward, reverse, and barcode. Ignoring " + oligo + "."); m->mothurOutEndLine(); } } m->gobble(inOligos); } -- 2.39.2