X-Git-Url: https://git.donarmstrong.com/?p=mothur.git;a=blobdiff_plain;f=chopseqscommand.cpp;h=96d7ae641b8b956694fe5ad12b9671c0279b302c;hp=4e06201cd352104b5ee9c9d1c41764d16473003e;hb=a8e2df1b96a57f5f29576b08361b86a96a8eff4f;hpb=d6c0a11d1cecfac18b323285e7ffadb7f58e848f diff --git a/chopseqscommand.cpp b/chopseqscommand.cpp index 4e06201..96d7ae6 100644 --- a/chopseqscommand.cpp +++ b/chopseqscommand.cpp @@ -9,17 +9,22 @@ #include "chopseqscommand.h" #include "sequence.hpp" +#include "removeseqscommand.h" //********************************************************************************************************************** vector ChopSeqsCommand::setParameters(){ try { - CommandParameter pfasta("fasta", "InputTypes", "", "", "none", "none", "none",false,true); parameters.push_back(pfasta); - CommandParameter pnumbases("numbases", "Number", "", "0", "", "", "",false,true); parameters.push_back(pnumbases); - CommandParameter pcountgaps("countgaps", "Boolean", "", "F", "", "", "",false,false); parameters.push_back(pcountgaps); - CommandParameter pshort("short", "Boolean", "", "F", "", "", "",false,false); parameters.push_back(pshort); - CommandParameter pkeep("keep", "Multiple", "front-back", "front", "", "", "",false,false); parameters.push_back(pkeep); - CommandParameter pinputdir("inputdir", "String", "", "", "", "", "",false,false); parameters.push_back(pinputdir); - CommandParameter poutputdir("outputdir", "String", "", "", "", "", "",false,false); parameters.push_back(poutputdir); + CommandParameter pfasta("fasta", "InputTypes", "", "", "none", "none", "none","fasta",false,true,true); parameters.push_back(pfasta); + CommandParameter pname("name", "InputTypes", "", "", "NameCount", "none", "none","name",false,false,true); parameters.push_back(pname); + CommandParameter pcount("count", "InputTypes", "", "", "NameCount-CountGroup", "none", "none","count",false,false,true); parameters.push_back(pcount); + CommandParameter pgroup("group", "InputTypes", "", "", "CountGroup", "none", "none","group",false,false,true); parameters.push_back(pgroup); + CommandParameter pprocessors("processors", "Number", "", "1", "", "", "","",false,false,true); parameters.push_back(pprocessors); + CommandParameter pnumbases("numbases", "Number", "", "0", "", "", "","",false,true,true); parameters.push_back(pnumbases); + CommandParameter pcountgaps("countgaps", "Boolean", "", "F", "", "", "","",false,false); parameters.push_back(pcountgaps); + CommandParameter pshort("short", "Boolean", "", "F", "", "", "","",false,false); parameters.push_back(pshort); + CommandParameter pkeep("keep", "Multiple", "front-back", "front", "", "", "","",false,false); parameters.push_back(pkeep); + CommandParameter pinputdir("inputdir", "String", "", "", "", "", "","",false,false); parameters.push_back(pinputdir); + CommandParameter poutputdir("outputdir", "String", "", "", "", "", "","",false,false); parameters.push_back(poutputdir); vector myArray; for (int i = 0; i < parameters.size(); i++) { myArray.push_back(parameters[i].name); } @@ -35,13 +40,15 @@ string ChopSeqsCommand::getHelpString(){ try { string helpString = ""; helpString += "The chop.seqs command reads a fasta file and outputs a .chop.fasta containing the trimmed sequences. Note: If a sequence is completely 'chopped', an accnos file will be created with the names of the sequences removed. \n"; - helpString += "The chop.seqs command parameters are fasta, numbases, countgaps and keep. fasta is required unless you have a valid current fasta file. numbases is required.\n"; + helpString += "The chop.seqs command parameters are fasta, name, group, count, numbases, countgaps and keep. fasta is required unless you have a valid current fasta file. numbases is required.\n"; helpString += "The chop.seqs command should be in the following format: chop.seqs(fasta=yourFasta, numbases=yourNum, keep=yourKeep).\n"; + helpString += "If you provide a name, group or count file any sequences removed from the fasta file will also be removed from those files.\n"; helpString += "The numbases parameter allows you to specify the number of bases you want to keep.\n"; helpString += "The keep parameter allows you to specify whether you want to keep the front or the back of your sequence, default=front.\n"; helpString += "The countgaps parameter allows you to specify whether you want to count gaps as bases, default=false.\n"; helpString += "The short parameter allows you to specify you want to keep sequences that are too short to chop, default=false.\n"; - helpString += "For example, if you ran chop.seqs with numbases=200 and short=t, if a sequence had 100 bases mothur would keep the sequence rather than eliminate it.\n"; + helpString += "The processors parameter allows you to specify how many processors you would like to use. The default is 1. \n"; + helpString += "For example, if you ran chop.seqs with numbases=200 and short=t, if a sequence had 100 bases mothur would keep the sequence rather than eliminate it.\n"; helpString += "Example chop.seqs(fasta=amazon.fasta, numbases=200, keep=front).\n"; helpString += "Note: No spaces between parameter labels (i.e. fasta), '=' and parameters (i.e.yourFasta).\n"; return helpString; @@ -51,7 +58,25 @@ string ChopSeqsCommand::getHelpString(){ exit(1); } } - +//********************************************************************************************************************** +string ChopSeqsCommand::getOutputPattern(string type) { + try { + string pattern = ""; + + if (type == "fasta") { pattern = "[filename],chop.fasta"; } + else if (type == "name") { pattern = "[filename],chop.names"; } + else if (type == "group") { pattern = "[filename],chop.groups"; } + else if (type == "count") { pattern = "[filename],chop.count_table"; } + else if (type == "accnos") { pattern = "[filename],chop.accnos"; } + else { m->mothurOut("[ERROR]: No definition for type " + type + " output pattern.\n"); m->control_pressed = true; } + + return pattern; + } + catch(exception& e) { + m->errorOut(e, "ChopSeqsCommand", "getOutputPattern"); + exit(1); + } +} //********************************************************************************************************************** ChopSeqsCommand::ChopSeqsCommand(){ try { @@ -60,6 +85,9 @@ ChopSeqsCommand::ChopSeqsCommand(){ vector tempOutNames; outputTypes["fasta"] = tempOutNames; outputTypes["accnos"] = tempOutNames; + outputTypes["name"] = tempOutNames; + outputTypes["group"] = tempOutNames; + outputTypes["count"] = tempOutNames; } catch(exception& e) { m->errorOut(e, "ChopSeqsCommand", "ChopSeqsCommand"); @@ -93,6 +121,9 @@ ChopSeqsCommand::ChopSeqsCommand(string option) { vector tempOutNames; outputTypes["fasta"] = tempOutNames; outputTypes["accnos"] = tempOutNames; + outputTypes["name"] = tempOutNames; + outputTypes["group"] = tempOutNames; + outputTypes["count"] = tempOutNames; //if the user changes the input directory command factory will send this info to us in the output parameter string inputDir = validParameter.validFile(parameters, "inputdir", false); @@ -106,6 +137,30 @@ ChopSeqsCommand::ChopSeqsCommand(string option) { //if the user has not given a path then, add inputdir. else leave path alone. if (path == "") { parameters["fasta"] = inputDir + it->second; } } + + it = parameters.find("name"); + //user has given a template file + if(it != parameters.end()){ + path = m->hasPath(it->second); + //if the user has not given a path then, add inputdir. else leave path alone. + if (path == "") { parameters["name"] = inputDir + it->second; } + } + + it = parameters.find("group"); + //user has given a template file + if(it != parameters.end()){ + path = m->hasPath(it->second); + //if the user has not given a path then, add inputdir. else leave path alone. + if (path == "") { parameters["group"] = inputDir + it->second; } + } + + it = parameters.find("count"); + //user has given a template file + if(it != parameters.end()){ + path = m->hasPath(it->second); + //if the user has not given a path then, add inputdir. else leave path alone. + if (path == "") { parameters["count"] = inputDir + it->second; } + } } //check for required parameters @@ -117,12 +172,39 @@ ChopSeqsCommand::ChopSeqsCommand(string option) { else { m->mothurOut("You have no current fastafile and the fasta parameter is required."); m->mothurOutEndLine(); abort = true; } }else { m->setFastaFile(fastafile); } + namefile = validParameter.validFile(parameters, "name", true); + if (namefile == "not open") { namefile = ""; abort = true; } + else if (namefile == "not found") { namefile = ""; } + else { m->setNameFile(namefile); } + + groupfile = validParameter.validFile(parameters, "group", true); + if (groupfile == "not open") { groupfile = ""; abort = true; } + else if (groupfile == "not found") { groupfile = ""; } + else { m->setGroupFile(groupfile); } + + countfile = validParameter.validFile(parameters, "count", true); + if (countfile == "not open") { countfile = ""; abort = true; } + else if (countfile == "not found") { countfile = ""; } + else { m->setCountTableFile(countfile); } + + if ((namefile != "") && (countfile != "")) { + m->mothurOut("[ERROR]: you may only use one of the following: name or count."); m->mothurOutEndLine(); abort = true; + } + + if ((groupfile != "") && (countfile != "")) { + m->mothurOut("[ERROR]: you may only use one of the following: group or count."); m->mothurOutEndLine(); abort=true; + } + //if the user changes the output directory command factory will send this info to us in the output parameter - outputDir = validParameter.validFile(parameters, "outputdir", false); if (outputDir == "not found"){ outputDir = m->hasPath(fastafile); } + outputDir = validParameter.validFile(parameters, "outputdir", false); if (outputDir == "not found"){ outputDir = ""; } string temp = validParameter.validFile(parameters, "numbases", false); if (temp == "not found") { temp = "0"; } m->mothurConvert(temp, numbases); + temp = validParameter.validFile(parameters, "processors", false); if (temp == "not found"){ temp = m->getProcessors(); } + m->setProcessors(temp); + m->mothurConvert(temp, processors); + temp = validParameter.validFile(parameters, "countgaps", false); if (temp == "not found") { temp = "f"; } countGaps = m->isTrue(temp); @@ -147,25 +229,298 @@ int ChopSeqsCommand::execute(){ if (abort == true) { if (calledHelp) { return 0; } return 2; } - string outputFileName = outputDir + m->getRootName(m->getSimpleName(fastafile)) + "chop.fasta"; - string outputFileNameAccnos = outputDir + m->getRootName(m->getSimpleName(fastafile)) + "chop.accnos"; + map variables; + string thisOutputDir = outputDir; + if (outputDir == "") { thisOutputDir += m->hasPath(fastafile); } + variables["[filename]"] = thisOutputDir + m->getRootName(m->getSimpleName(fastafile)); + string outputFileName = getOutputFileName("fasta", variables); + outputNames.push_back(outputFileName); outputTypes["fasta"].push_back(outputFileName); + string outputFileNameAccnos = getOutputFileName("accnos", variables); + + vector positions; + vector lines; +#if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix) + positions = m->divideFile(fastafile, processors); + for (int i = 0; i < (positions.size()-1); i++) { lines.push_back(linePair(positions[i], positions[(i+1)])); } +#else + int numSeqs = 0; + positions = m->setFilePosFasta(fastafile, numSeqs); + if (positions.size() < processors) { processors = positions.size(); } - ofstream out; - m->openOutputFile(outputFileName, out); + //figure out how many sequences you have to process + int numSeqsPerProcessor = numSeqs / processors; + for (int i = 0; i < processors; i++) { + int startIndex = i * numSeqsPerProcessor; + if(i == (processors - 1)){ numSeqsPerProcessor = numSeqs - i * numSeqsPerProcessor; } + lines.push_back(linePair(positions[startIndex], numSeqsPerProcessor)); + } +#endif + + bool wroteAccnos = false; + if(processors == 1) { wroteAccnos = driver(lines[0], fastafile, outputFileName, outputFileNameAccnos); } + else { wroteAccnos = createProcesses(lines, fastafile, outputFileName, outputFileNameAccnos); } + + if (m->control_pressed) { return 0; } - ofstream outAcc; - m->openOutputFile(outputFileNameAccnos, outAcc); + if (wroteAccnos) { + outputNames.push_back(outputFileNameAccnos); outputTypes["accnos"].push_back(outputFileNameAccnos); + + //use remove.seqs to create new name, group and count file + if ((countfile != "") || (namefile != "") || (groupfile != "")) { + string inputString = "accnos=" + outputFileNameAccnos; + + if (countfile != "") { inputString += ", count=" + countfile; } + else{ + if (namefile != "") { inputString += ", name=" + namefile; } + if (groupfile != "") { inputString += ", group=" + groupfile; } + } + + m->mothurOut("/******************************************/"); m->mothurOutEndLine(); + m->mothurOut("Running command: remove.seqs(" + inputString + ")"); m->mothurOutEndLine(); + m->mothurCalling = true; + + Command* removeCommand = new RemoveSeqsCommand(inputString); + removeCommand->execute(); + + map > filenames = removeCommand->getOutputFiles(); + + delete removeCommand; + m->mothurCalling = false; + m->mothurOut("/******************************************/"); m->mothurOutEndLine(); + + if (groupfile != "") { + thisOutputDir = outputDir; + if (outputDir == "") { thisOutputDir += m->hasPath(groupfile); } + variables["[filename]"] = thisOutputDir + m->getRootName(m->getSimpleName(groupfile)); + string outGroup = getOutputFileName("group", variables); + m->renameFile(filenames["group"][0], outGroup); + outputNames.push_back(outGroup); outputTypes["group"].push_back(outGroup); + } + + if (namefile != "") { + thisOutputDir = outputDir; + if (outputDir == "") { thisOutputDir += m->hasPath(namefile); } + variables["[filename]"] = thisOutputDir + m->getRootName(m->getSimpleName(namefile)); + string outName = getOutputFileName("name", variables); + m->renameFile(filenames["name"][0], outName); + outputNames.push_back(outName); outputTypes["name"].push_back(outName); + } + + if (countfile != "") { + thisOutputDir = outputDir; + if (outputDir == "") { thisOutputDir += m->hasPath(countfile); } + variables["[filename]"] = thisOutputDir + m->getRootName(m->getSimpleName(countfile)); + string outCount = getOutputFileName("count", variables); + m->renameFile(filenames["count"][0], outCount); + outputNames.push_back(outCount); outputTypes["count"].push_back(outCount); + } + } + } + else { m->mothurRemove(outputFileNameAccnos); } - ifstream in; - m->openInputFile(fastafile, in); + //set fasta file as new current fastafile + string current = ""; + itTypes = outputTypes.find("fasta"); + if (itTypes != outputTypes.end()) { + if ((itTypes->second).size() != 0) { current = (itTypes->second)[0]; m->setFastaFile(current); } + } + + if (wroteAccnos) { //set accnos file as new current accnosfile + itTypes = outputTypes.find("accnos"); + if (itTypes != outputTypes.end()) { + if ((itTypes->second).size() != 0) { current = (itTypes->second)[0]; m->setAccnosFile(current); } + } + + itTypes = outputTypes.find("name"); + if (itTypes != outputTypes.end()) { + if ((itTypes->second).size() != 0) { current = (itTypes->second)[0]; m->setNameFile(current); } + } + + itTypes = outputTypes.find("group"); + if (itTypes != outputTypes.end()) { + if ((itTypes->second).size() != 0) { current = (itTypes->second)[0]; m->setGroupFile(current); } + } + + itTypes = outputTypes.find("count"); + if (itTypes != outputTypes.end()) { + if ((itTypes->second).size() != 0) { current = (itTypes->second)[0]; m->setCountTableFile(current); } + } + } + m->mothurOutEndLine(); + m->mothurOut("Output File Names: "); m->mothurOutEndLine(); + for (int i = 0; i < outputNames.size(); i++) { m->mothurOut(outputNames[i]); m->mothurOutEndLine(); } + m->mothurOutEndLine(); + + return 0; + } + + catch(exception& e) { + m->errorOut(e, "ChopSeqsCommand", "execute"); + exit(1); + } +} +/**************************************************************************************************/ +bool ChopSeqsCommand::createProcesses(vector lines, string filename, string outFasta, string outAccnos) { + try { + int process = 1; bool wroteAccnos = false; + vector processIDS; + vector nonBlankAccnosFiles; - while (!in.eof()) { +#if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix) + + //loop through and create all the processes you want + while (process != processors) { + int pid = fork(); - Sequence seq(in); + if (pid > 0) { + processIDS.push_back(pid); //create map from line number to pid so you can append files in correct order later + process++; + }else if (pid == 0){ + wroteAccnos = driver(lines[process], filename, outFasta + toString(getpid()) + ".temp", outAccnos + toString(getpid()) + ".temp"); + + //pass numSeqs to parent + ofstream out; + string tempFile = fastafile + toString(getpid()) + ".bool.temp"; + m->openOutputFile(tempFile, out); + out << wroteAccnos << endl; + out.close(); + + exit(0); + }else { + m->mothurOut("[ERROR]: unable to spawn the necessary processes."); m->mothurOutEndLine(); + for (int i = 0; i < processIDS.size(); i++) { kill (processIDS[i], SIGINT); } + exit(0); + } + } + + //do your part + wroteAccnos = driver(lines[0], filename, outFasta, outAccnos); + + //force parent to wait until all the processes are done + for (int i=0;imothurRemove(outAccnos); } //remove so other files can be renamed to it + + //parent reads in and combine Filter info + for (int i = 0; i < processIDS.size(); i++) { + string tempFilename = fastafile + toString(processIDS[i]) + ".bool.temp"; + ifstream in; + m->openInputFile(tempFilename, in); - if (m->control_pressed) { outputTypes.clear(); in.close(); out.close(); outAcc.close(); m->mothurRemove(outputFileName); m->mothurRemove(outputFileNameAccnos); return 0; } + bool temp; + in >> temp; m->gobble(in); + if (temp) { wroteAccnos = temp; nonBlankAccnosFiles.push_back(outAccnos + toString(processIDS[i]) + ".temp"); } + else { m->mothurRemove((outAccnos + toString(processIDS[i]) + ".temp")); } + + in.close(); + m->mothurRemove(tempFilename); + } +#else + ////////////////////////////////////////////////////////////////////////////////////////////////////// + //Windows version shared memory, so be careful when passing variables through the seqSumData struct. + //Above fork() will clone, so memory is separate, but that's not the case with windows, + //Taking advantage of shared memory to allow both threads to add info to vectors. + ////////////////////////////////////////////////////////////////////////////////////////////////////// + + vector pDataArray; + DWORD dwThreadIdArray[processors-1]; + HANDLE hThreadArray[processors-1]; + + //Create processor worker threads. + for( int i=0; imothurRemove(outAccnos); } //remove so other files can be renamed to it + + //Close all thread handles and free memory allocations. + for(int i=0; i < pDataArray.size(); i++){ + if (pDataArray[i]->wroteAccnos) { wroteAccnos = pDataArray[i]->wroteAccnos; nonBlankAccnosFiles.push_back(outAccnos + toString(processIDS[i]) + ".temp"); } + else { m->mothurRemove((outAccnos + toString(processIDS[i]) + ".temp")); } + //check to make sure the process finished + if (pDataArray[i]->count != pDataArray[i]->end) { + m->mothurOut("[ERROR]: process " + toString(i) + " only processed " + toString(pDataArray[i]->count) + " of " + toString(pDataArray[i]->end) + " sequences assigned to it, quitting. \n"); m->control_pressed = true; + } + CloseHandle(hThreadArray[i]); + delete pDataArray[i]; + } +#endif + + for (int i = 0; i < processIDS.size(); i++) { + m->appendFiles((outFasta + toString(processIDS[i]) + ".temp"), outFasta); + m->mothurRemove((outFasta + toString(processIDS[i]) + ".temp")); + } + + if (nonBlankAccnosFiles.size() != 0) { + m->renameFile(nonBlankAccnosFiles[0], outAccnos); + + for (int h=1; h < nonBlankAccnosFiles.size(); h++) { + m->appendFiles(nonBlankAccnosFiles[h], outAccnos); + m->mothurRemove(nonBlankAccnosFiles[h]); + } + }else { //recreate the accnosfile if needed + ofstream out; + m->openOutputFile(outAccnos, out); + out.close(); + } + + return wroteAccnos; + } + catch(exception& e) { + m->errorOut(e, "ChopSeqsCommand", "createProcesses"); + exit(1); + } +} +/**************************************************************************************/ +bool ChopSeqsCommand::driver(linePair filePos, string filename, string outFasta, string outAccnos) { + try { + + ofstream out; + m->openOutputFile(outFasta, out); + + ofstream outAcc; + m->openOutputFile(outAccnos, outAcc); + + ifstream in; + m->openInputFile(filename, in); + + in.seekg(filePos.start); + + bool done = false; + bool wroteAccnos = false; + int count = 0; + + while (!done) { + + if (m->control_pressed) { in.close(); out.close(); return 1; } + + Sequence seq(in); m->gobble(in); + + if (m->control_pressed) { in.close(); out.close(); outAcc.close(); m->mothurRemove(outFasta); m->mothurRemove(outAccnos); return 0; } if (seq.getName() != "") { string newSeqString = getChopped(seq); @@ -177,41 +532,31 @@ int ChopSeqsCommand::execute(){ outAcc << seq.getName() << endl; wroteAccnos = true; } + count++; } + +#if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix) + unsigned long long pos = in.tellg(); + if ((pos == -1) || (pos >= filePos.end)) { break; } +#else + if (in.eof()) { break; } +#endif + //report progress + if((count) % 1000 == 0){ m->mothurOut(toString(count)); m->mothurOutEndLine(); } + } - in.close(); - out.close(); - outAcc.close(); - - m->mothurOutEndLine(); - m->mothurOut("Output File Name: "); m->mothurOutEndLine(); - m->mothurOut(outputFileName); m->mothurOutEndLine(); outputNames.push_back(outputFileName); outputTypes["fasta"].push_back(outputFileName); - - if (wroteAccnos) { m->mothurOut(outputFileNameAccnos); m->mothurOutEndLine(); outputNames.push_back(outputFileNameAccnos); outputTypes["accnos"].push_back(outputFileNameAccnos); } - else { m->mothurRemove(outputFileNameAccnos); } - - m->mothurOutEndLine(); - - //set fasta file as new current fastafile - string current = ""; - itTypes = outputTypes.find("fasta"); - if (itTypes != outputTypes.end()) { - if ((itTypes->second).size() != 0) { current = (itTypes->second)[0]; m->setFastaFile(current); } - } - - if (wroteAccnos) { //set accnos file as new current accnosfile - itTypes = outputTypes.find("accnos"); - if (itTypes != outputTypes.end()) { - if ((itTypes->second).size() != 0) { current = (itTypes->second)[0]; m->setAccnosFile(current); } - } - } + //report progress + if((count) % 1000 != 0){ m->mothurOut(toString(count)); m->mothurOutEndLine(); } + + in.close(); + out.close(); + outAcc.close(); - return 0; + return wroteAccnos; } - catch(exception& e) { - m->errorOut(e, "ChopSeqsCommand", "execute"); + m->errorOut(e, "ChopSeqsCommand", "driver"); exit(1); } }