From b866e1519a60681527244036428104ad1cb90c93 Mon Sep 17 00:00:00 2001 From: Sarah Westcott Date: Tue, 21 Aug 2012 13:39:22 -0400 Subject: [PATCH] added oligos, pdiffs, bdiffs, ldiffs, sdiffs, tiffs parameters to sffinfo to allow for spitting the sff file by barcode. working on sff.multiple command. --- commandfactory.cpp | 5 + sffinfocommand.cpp | 802 ++++++++++++++++++++++++++++++++++++++--- sffinfocommand.h | 24 +- sffmultiplecommand.cpp | 371 +++++++++++++++++-- sffmultiplecommand.h | 18 +- trimseqscommand.cpp | 1 + 6 files changed, 1127 insertions(+), 94 deletions(-) diff --git a/commandfactory.cpp b/commandfactory.cpp index 02af676..8653643 100644 --- a/commandfactory.cpp +++ b/commandfactory.cpp @@ -134,6 +134,7 @@ #include "removeotulabelscommand.h" #include "makecontigscommand.h" #include "loadlogfilecommand.h" +#include "sffmultiplecommand.h" /*******************************************************/ @@ -290,6 +291,7 @@ CommandFactory::CommandFactory(){ commands["make.contigs"] = "make.contigs"; commands["load.logfile"] = "load.logfile"; commands["make.table"] = "make.table"; + commands["sff.multiple"] = "sff.multiple"; commands["quit"] = "MPIEnabled"; } @@ -503,6 +505,7 @@ Command* CommandFactory::getCommand(string commandName, string optionString){ else if(commandName == "remove.otulabels") { command = new RemoveOtuLabelsCommand(optionString); } else if(commandName == "make.contigs") { command = new MakeContigsCommand(optionString); } else if(commandName == "load.logfile") { command = new LoadLogfileCommand(optionString); } + else if(commandName == "sff.multiple") { command = new SffMultipleCommand(optionString); } else { command = new NoCommand(optionString); } return command; @@ -657,6 +660,7 @@ Command* CommandFactory::getCommand(string commandName, string optionString, str else if(commandName == "remove.otulabels") { pipecommand = new RemoveOtuLabelsCommand(optionString); } else if(commandName == "make.contigs") { pipecommand = new MakeContigsCommand(optionString); } else if(commandName == "load.logfile") { pipecommand = new LoadLogfileCommand(optionString); } + else if(commandName == "sff.multiple") { pipecommand = new SffMultipleCommand(optionString); } else { pipecommand = new NoCommand(optionString); } return pipecommand; @@ -797,6 +801,7 @@ Command* CommandFactory::getCommand(string commandName){ else if(commandName == "remove.otulabels") { shellcommand = new RemoveOtuLabelsCommand(); } else if(commandName == "make.contigs") { shellcommand = new MakeContigsCommand(); } else if(commandName == "load.logfile") { shellcommand = new LoadLogfileCommand(); } + else if(commandName == "sff.multiple") { shellcommand = new SffMultipleCommand(); } else { shellcommand = new NoCommand(); } return shellcommand; diff --git a/sffinfocommand.cpp b/sffinfocommand.cpp index 08cf21e..2d77d28 100644 --- a/sffinfocommand.cpp +++ b/sffinfocommand.cpp @@ -9,17 +9,26 @@ #include "sffinfocommand.h" #include "endiannessmacros.h" +#include "trimoligos.h" +#include "sequence.hpp" +#include "qualityscores.h" //********************************************************************************************************************** vector SffInfoCommand::setParameters(){ try { CommandParameter psff("sff", "InputTypes", "", "", "none", "none", "none",false,false); parameters.push_back(psff); + CommandParameter poligos("oligos", "InputTypes", "", "", "none", "none", "none",false,false); parameters.push_back(poligos); CommandParameter paccnos("accnos", "InputTypes", "", "", "none", "none", "none",false,false); parameters.push_back(paccnos); CommandParameter psfftxt("sfftxt", "String", "", "", "", "", "",false,false); parameters.push_back(psfftxt); CommandParameter pflow("flow", "Boolean", "", "T", "", "", "",false,false); parameters.push_back(pflow); CommandParameter ptrim("trim", "Boolean", "", "T", "", "", "",false,false); parameters.push_back(ptrim); CommandParameter pfasta("fasta", "Boolean", "", "T", "", "", "",false,false); parameters.push_back(pfasta); CommandParameter pqfile("name", "Boolean", "", "T", "", "", "",false,false); parameters.push_back(pqfile); + CommandParameter ppdiffs("pdiffs", "Number", "", "0", "", "", "",false,false); parameters.push_back(ppdiffs); + CommandParameter pbdiffs("bdiffs", "Number", "", "0", "", "", "",false,false); parameters.push_back(pbdiffs); + CommandParameter pldiffs("ldiffs", "Number", "", "0", "", "", "",false,false); parameters.push_back(pldiffs); + CommandParameter psdiffs("sdiffs", "Number", "", "0", "", "", "",false,false); parameters.push_back(psdiffs); + CommandParameter ptdiffs("tdiffs", "Number", "", "0", "", "", "",false,false); parameters.push_back(ptdiffs); CommandParameter pinputdir("inputdir", "String", "", "", "", "", "",false,false); parameters.push_back(pinputdir); CommandParameter poutputdir("outputdir", "String", "", "", "", "", "",false,false); parameters.push_back(poutputdir); @@ -37,10 +46,16 @@ string SffInfoCommand::getHelpString(){ try { string helpString = ""; helpString += "The sffinfo command reads a sff file and extracts the sequence data, or you can use it to parse a sfftxt file.\n"; - helpString += "The sffinfo command parameters are sff, fasta, qfile, accnos, flow, sfftxt, and trim. sff is required. \n"; + helpString += "The sffinfo command parameters are sff, fasta, qfile, accnos, flow, sfftxt, oligos, bdiffs, tdiffs, ldiffs, sdiffs, pdiffs and trim. sff is required. \n"; helpString += "The sff parameter allows you to enter the sff file you would like to extract data from. You may enter multiple files by separating them by -'s.\n"; helpString += "The fasta parameter allows you to indicate if you would like a fasta formatted file generated. Default=True. \n"; helpString += "The qfile parameter allows you to indicate if you would like a quality file generated. Default=True. \n"; + helpString += "The oligos parameter allows you to provide an oligos file to split your sff file into separate sff files by barcode. \n"; + helpString += "The tdiffs parameter is used to specify the total number of differences allowed in the sequence. The default is pdiffs + bdiffs + sdiffs + ldiffs.\n"; + helpString += "The bdiffs parameter is used to specify the number of differences allowed in the barcode. The default is 0.\n"; + helpString += "The pdiffs parameter is used to specify the number of differences allowed in the primer. The default is 0.\n"; + helpString += "The ldiffs parameter is used to specify the number of differences allowed in the linker. The default is 0.\n"; + helpString += "The sdiffs parameter is used to specify the number of differences allowed in the spacer. The default is 0.\n"; helpString += "The flow parameter allows you to indicate if you would like a flowgram file generated. Default=True. \n"; helpString += "The sfftxt parameter allows you to indicate if you would like a sff.txt file generated. Default=False. \n"; helpString += "If you want to parse an existing sfftxt file into flow, fasta and quality file, enter the file name using the sfftxt parameter. \n"; @@ -68,6 +83,7 @@ string SffInfoCommand::getOutputFileNameTag(string type, string inputName=""){ if (type == "fasta") { outputFileName = "fasta"; } else if (type == "flow") { outputFileName = "flow"; } else if (type == "sfftxt") { outputFileName = "sff.txt"; } + else if (type == "sff") { outputFileName = "sff"; } else if (type == "qfile") { outputFileName = "qual"; } else { m->mothurOut("[ERROR]: No definition for type " + type + " output file tag.\n"); m->control_pressed = true; } } @@ -90,6 +106,7 @@ SffInfoCommand::SffInfoCommand(){ outputTypes["flow"] = tempOutNames; outputTypes["sfftxt"] = tempOutNames; outputTypes["qfile"] = tempOutNames; + outputTypes["sff"] = tempOutNames; } catch(exception& e) { m->errorOut(e, "SffInfoCommand", "SffInfoCommand"); @@ -102,6 +119,7 @@ SffInfoCommand::SffInfoCommand(string option) { try { abort = false; calledHelp = false; hasAccnos = false; + split = 1; //allow user to run help if(option == "help") { help(); abort = true; calledHelp = true; } @@ -126,6 +144,7 @@ SffInfoCommand::SffInfoCommand(string option) { outputTypes["flow"] = tempOutNames; outputTypes["sfftxt"] = tempOutNames; outputTypes["qfile"] = tempOutNames; + outputTypes["sff"] = tempOutNames; //if the user changes the output directory command factory will send this info to us in the output parameter outputDir = validParameter.validFile(parameters, "outputdir", false); if (outputDir == "not found"){ outputDir = ""; } @@ -268,7 +287,80 @@ SffInfoCommand::SffInfoCommand(string option) { //make sure there is at least one valid file left if (accnosFileNames.size() == 0) { m->mothurOut("no valid files."); m->mothurOutEndLine(); abort = true; } } - + + oligosfile = validParameter.validFile(parameters, "oligos", false); + if (oligosfile == "not found") { oligosfile = ""; } + else { + hasOligos = true; + m->splitAtDash(oligosfile, oligosFileNames); + + //go through files and make sure they are good, if not, then disregard them + for (int i = 0; i < oligosFileNames.size(); i++) { + bool ignore = false; + if (oligosFileNames[i] == "current") { + oligosFileNames[i] = m->getOligosFile(); + if (oligosFileNames[i] != "") { m->mothurOut("Using " + oligosFileNames[i] + " as input file for the accnos parameter where you had given current."); m->mothurOutEndLine(); } + else { + m->mothurOut("You have no current oligosfile, ignoring current."); m->mothurOutEndLine(); ignore=true; + //erase from file list + oligosFileNames.erase(oligosFileNames.begin()+i); + i--; + } + } + + if (!ignore) { + + if (inputDir != "") { + string path = m->hasPath(oligosFileNames[i]); + //if the user has not given a path then, add inputdir. else leave path alone. + if (path == "") { oligosFileNames[i] = inputDir + oligosFileNames[i]; } + } + + ifstream in; + int ableToOpen = m->openInputFile(oligosFileNames[i], in, "noerror"); + + //if you can't open it, try default location + if (ableToOpen == 1) { + if (m->getDefaultPath() != "") { //default path is set + string tryPath = m->getDefaultPath() + m->getSimpleName(oligosFileNames[i]); + m->mothurOut("Unable to open " + oligosFileNames[i] + ". Trying default " + tryPath); m->mothurOutEndLine(); + ifstream in2; + ableToOpen = m->openInputFile(tryPath, in2, "noerror"); + in2.close(); + oligosFileNames[i] = tryPath; + } + } + //if you can't open it, try default location + if (ableToOpen == 1) { + if (m->getOutputDir() != "") { //default path is set + string tryPath = m->getOutputDir() + m->getSimpleName(oligosFileNames[i]); + m->mothurOut("Unable to open " + oligosFileNames[i] + ". Trying output directory " + tryPath); m->mothurOutEndLine(); + ifstream in2; + ableToOpen = m->openInputFile(tryPath, in2, "noerror"); + in2.close(); + oligosFileNames[i] = tryPath; + } + } + in.close(); + + if (ableToOpen == 1) { + m->mothurOut("Unable to open " + oligosFileNames[i] + ". It will be disregarded."); m->mothurOutEndLine(); + //erase from file list + oligosFileNames.erase(oligosFileNames.begin()+i); + i--; + } + } + } + + //make sure there is at least one valid file left + if (oligosFileNames.size() == 0) { m->mothurOut("no valid oligos files."); m->mothurOutEndLine(); abort = true; } + } + + if (hasOligos) { + split = 2; + if (oligosFileNames.size() != filenames.size()) { abort = true; m->mothurOut("If you provide a oligos file, you must have one for each sff file."); m->mothurOutEndLine(); } + } + if (hasAccnos) { if (accnosFileNames.size() != filenames.size()) { abort = true; m->mothurOut("If you provide a accnos file, you must have one for each sff file."); m->mothurOutEndLine(); } } @@ -284,7 +376,24 @@ SffInfoCommand::SffInfoCommand(string option) { temp = validParameter.validFile(parameters, "trim", false); if (temp == "not found"){ temp = "T"; } trim = m->isTrue(temp); + + temp = validParameter.validFile(parameters, "bdiffs", false); if (temp == "not found") { temp = "0"; } + m->mothurConvert(temp, bdiffs); + + temp = validParameter.validFile(parameters, "pdiffs", false); if (temp == "not found") { temp = "0"; } + m->mothurConvert(temp, pdiffs); + + temp = validParameter.validFile(parameters, "ldiffs", false); if (temp == "not found") { temp = "0"; } + m->mothurConvert(temp, ldiffs); + + temp = validParameter.validFile(parameters, "sdiffs", false); if (temp == "not found") { temp = "0"; } + m->mothurConvert(temp, sdiffs); + temp = validParameter.validFile(parameters, "tdiffs", false); if (temp == "not found") { int tempTotal = pdiffs + bdiffs + ldiffs + sdiffs; temp = toString(tempTotal); } + m->mothurConvert(temp, tdiffs); + + if(tdiffs == 0){ tdiffs = bdiffs + pdiffs + ldiffs + sdiffs; } + temp = validParameter.validFile(parameters, "sfftxt", false); if (temp == "not found") { temp = "F"; sfftxt = false; sfftxtFilename = ""; } else if (m->isTrue(temp)) { sfftxt = true; sfftxtFilename = ""; } @@ -311,6 +420,8 @@ SffInfoCommand::SffInfoCommand(string option) { if (filename != "") { filenames.push_back(filename); m->mothurOut("Using " + filename + " as input file for the sff parameter."); m->mothurOutEndLine(); } else { m->mothurOut("[ERROR]: you must provide a valid sff or sfftxt file."); m->mothurOutEndLine(); abort=true; } } + + } } catch(exception& e) { @@ -334,8 +445,11 @@ int SffInfoCommand::execute(){ string accnos = ""; if (hasAccnos) { accnos = accnosFileNames[s]; } + + string oligos = ""; + if (hasOligos) { oligos = oligosFileNames[s]; } - int numReads = extractSffInfo(filenames[s], accnos); + int numReads = extractSffInfo(filenames[s], accnos, oligos); m->mothurOut("It took " + toString(time(NULL) - start) + " secs to extract " + toString(numReads) + "."); } @@ -375,13 +489,15 @@ int SffInfoCommand::execute(){ } } //********************************************************************************************************************** -int SffInfoCommand::extractSffInfo(string input, string accnos){ +int SffInfoCommand::extractSffInfo(string input, string accnos, string oligos){ try { - + currentFileName = input; if (outputDir == "") { outputDir += m->hasPath(input); } if (accnos != "") { readAccnosFile(accnos); } else { seqNames.clear(); } + + if (oligos != "") { readOligos(oligos); split = 2; } ofstream outSfftxt, outFasta, outQual, outFlow; string outFastaFileName, outQualFileName; @@ -424,14 +540,10 @@ int SffInfoCommand::extractSffInfo(string input, string accnos){ while (!in.eof()) { bool print = true; - - //read header - Header readheader; - readHeader(in, readheader); - + //read data - seqRead read; - readSeqData(in, read, header.numFlowsPerRead, readheader.numBases); + seqRead read; Header readheader; + readSeqData(in, read, header.numFlowsPerRead, readheader); bool okay = sanityCheck(readheader, read); if (!okay) { break; } @@ -448,7 +560,7 @@ int SffInfoCommand::extractSffInfo(string input, string accnos){ count++; mycount++; - + //report progress if((count+1) % 10000 == 0){ m->mothurOut(toString(count+1)); m->mothurOutEndLine(); } @@ -467,6 +579,54 @@ int SffInfoCommand::extractSffInfo(string input, string accnos){ if (qual) { outQual.close(); } if (flow) { outFlow.close(); } + if (split > 1) { + //create new common headers for each file with the correct number of reads + adjustCommonHeader(header); + + map uniqueSffNames;// so we don't add the same sff multiple times + map::iterator it; + set namesToRemove; + for(int i=0;iisBlank(filehandles[i][j])){ + m->mothurRemove(filehandles[i][j]); + m->mothurRemove(filehandlesHeaders[i][j]); + namesToRemove.insert(filehandles[i][j]); + }else{ + it = uniqueSffNames.find(filehandles[i][j]); + if (it == uniqueSffNames.end()) { + uniqueSffNames[filehandles[i][j]] = barcodeNameVector[i]; + } + } + } + } + } + } + + //append new header to reads + for (int i = 0; i < filehandles.size(); i++) { + for (int j = 0; j < filehandles[i].size(); j++) { + m->appendFiles(filehandles[i][j], filehandlesHeaders[i][j]); + m->renameFile(filehandlesHeaders[i][j], filehandles[i][j]); + m->mothurRemove(filehandlesHeaders[i][j]); + if (numSplitReads[i][j] == 0) { m->mothurRemove(filehandles[i][j]); } + } + } + + //remove names for outputFileNames, just cleans up the output + for(int i = 0; i < outputNames.size(); i++) { + if (namesToRemove.count(outputNames[i]) != 0) { + outputNames.erase(outputNames.begin()+i); + i--; + } + } + + if(m->isBlank(noMatchFile)){ m->mothurRemove(noMatchFile); } + else { outputNames.push_back(noMatchFile); outputTypes["sff"].push_back(noMatchFile); } + } + return count; } catch(exception& e) { @@ -477,20 +637,20 @@ int SffInfoCommand::extractSffInfo(string input, string accnos){ //********************************************************************************************************************** int SffInfoCommand::readCommonHeader(ifstream& in, CommonHeader& header){ try { - + if (!in.eof()) { //read magic number char buffer[4]; in.read(buffer, 4); header.magicNumber = be_int4(*(unsigned int *)(&buffer)); - + //read version char buffer9[4]; in.read(buffer9, 4); header.version = ""; - for (int i = 0; i < 4; i++) { header.version += toString((int)(buffer9[i])); } - + for (int i = 0; i < 4; i++) { header.version += toString((int)(buffer9[i])); } + //read offset char buffer2 [8]; in.read(buffer2, 8); @@ -539,17 +699,18 @@ int SffInfoCommand::readCommonHeader(ifstream& in, CommonHeader& header){ header.keySequence = tempBuffer2; if (header.keySequence.length() > header.keyLength) { header.keySequence = header.keySequence.substr(0, header.keyLength); } delete[] tempBuffer2; - + /* Pad to 8 chars */ unsigned long long spotInFile = in.tellg(); unsigned long long spot = (spotInFile + 7)& ~7; // ~ inverts in.seekg(spot); - - }else{ + + }else{ m->mothurOut("Error reading sff common header."); m->mothurOutEndLine(); } - + return 0; + } catch(exception& e) { m->errorOut(e, "SffInfoCommand", "readCommonHeader"); @@ -557,21 +718,207 @@ int SffInfoCommand::readCommonHeader(ifstream& in, CommonHeader& header){ } } //********************************************************************************************************************** -int SffInfoCommand::readHeader(ifstream& in, Header& header){ +int SffInfoCommand::adjustCommonHeader(CommonHeader header){ try { - - if (!in.eof()) { + + char* mybuffer = new char[4]; + ifstream in; + in.open(currentFileName.c_str(), ios::binary); + + //magic number + in.read(mybuffer,4); + for (int i = 0; i < filehandlesHeaders.size(); i++) { + for (int j = 0; j < filehandlesHeaders[i].size(); j++) { + ofstream out; + m->openOutputFileAppend(filehandlesHeaders[i][j], out); + out.write(mybuffer, in.gcount()); + out.close(); + } + } + delete[] mybuffer; + + //version + mybuffer = new char[4]; + in.read(mybuffer,4); + for (int i = 0; i < filehandlesHeaders.size(); i++) { + for (int j = 0; j < filehandlesHeaders[i].size(); j++) { + ofstream out; + m->openOutputFileAppend(filehandlesHeaders[i][j], out); + out.write(mybuffer, in.gcount()); + out.close(); + } + } + delete[] mybuffer; + + //offset + mybuffer = new char[8]; + in.read(mybuffer,8); + for (int i = 0; i < filehandlesHeaders.size(); i++) { + for (int j = 0; j < filehandlesHeaders[i].size(); j++) { + ofstream out; + m->openOutputFileAppend(filehandlesHeaders[i][j], out); + out.write(mybuffer, in.gcount()); + out.close(); + } + } + delete[] mybuffer; + - //read header length + //read index length + mybuffer = new char[4]; + in.read(mybuffer,4); + for (int i = 0; i < filehandlesHeaders.size(); i++) { + for (int j = 0; j < filehandlesHeaders[i].size(); j++) { + ofstream out; + m->openOutputFileAppend(filehandlesHeaders[i][j], out); + out.write(mybuffer, in.gcount()); + out.close(); + } + } + delete[] mybuffer; + + //change num reads + mybuffer = new char[4]; + in.read(mybuffer,4); + delete[] mybuffer; + for (int i = 0; i < filehandlesHeaders.size(); i++) { + for (int j = 0; j < filehandlesHeaders[i].size(); j++) { + ofstream out; + m->openOutputFileAppend(filehandlesHeaders[i][j], out); + //convert number of reads to 4 byte char* + char* thisbuffer = new char[4]; + thisbuffer[0] = (numSplitReads[i][j] >> 24) & 0xFF; + thisbuffer[1] = (numSplitReads[i][j] >> 16) & 0xFF; + thisbuffer[2] = (numSplitReads[i][j] >> 8) & 0xFF; + thisbuffer[3] = numSplitReads[i][j] & 0xFF; + out.write(thisbuffer, 4); + out.close(); + delete[] thisbuffer; + } + } + + //read header length + mybuffer = new char[2]; + in.read(mybuffer,2); + for (int i = 0; i < filehandlesHeaders.size(); i++) { + for (int j = 0; j < filehandlesHeaders[i].size(); j++) { + ofstream out; + m->openOutputFileAppend(filehandlesHeaders[i][j], out); + out.write(mybuffer, in.gcount()); + out.close(); + } + } + delete[] mybuffer; + + //read key length + mybuffer = new char[2]; + in.read(mybuffer,2); + for (int i = 0; i < filehandlesHeaders.size(); i++) { + for (int j = 0; j < filehandlesHeaders[i].size(); j++) { + ofstream out; + m->openOutputFileAppend(filehandlesHeaders[i][j], out); + out.write(mybuffer, in.gcount()); + out.close(); + } + } + delete[] mybuffer; + + //read number of flow reads + mybuffer = new char[2]; + in.read(mybuffer,2); + for (int i = 0; i < filehandlesHeaders.size(); i++) { + for (int j = 0; j < filehandlesHeaders[i].size(); j++) { + ofstream out; + m->openOutputFileAppend(filehandlesHeaders[i][j], out); + out.write(mybuffer, in.gcount()); + out.close(); + } + } + delete[] mybuffer; + + //read format code + mybuffer = new char[1]; + in.read(mybuffer,1); + for (int i = 0; i < filehandlesHeaders.size(); i++) { + for (int j = 0; j < filehandlesHeaders[i].size(); j++) { + ofstream out; + m->openOutputFileAppend(filehandlesHeaders[i][j], out); + out.write(mybuffer, in.gcount()); + out.close(); + } + } + delete[] mybuffer; + + //read flow chars + mybuffer = new char[header.numFlowsPerRead]; + in.read(mybuffer,header.numFlowsPerRead); + for (int i = 0; i < filehandlesHeaders.size(); i++) { + for (int j = 0; j < filehandlesHeaders[i].size(); j++) { + ofstream out; + m->openOutputFileAppend(filehandlesHeaders[i][j], out); + out.write(mybuffer, in.gcount()); + out.close(); + } + } + delete[] mybuffer; + + //read key + mybuffer = new char[header.keyLength]; + in.read(mybuffer,header.keyLength); + for (int i = 0; i < filehandlesHeaders.size(); i++) { + for (int j = 0; j < filehandlesHeaders[i].size(); j++) { + ofstream out; + m->openOutputFileAppend(filehandlesHeaders[i][j], out); + out.write(mybuffer, in.gcount()); + out.close(); + } + } + delete[] mybuffer; + + + /* Pad to 8 chars */ + unsigned long long spotInFile = in.tellg(); + unsigned long long spot = (spotInFile + 7)& ~7; // ~ inverts + in.seekg(spot); + + mybuffer = new char[spot-spotInFile]; + for (int i = 0; i < filehandlesHeaders.size(); i++) { + for (int j = 0; j < filehandlesHeaders[i].size(); j++) { + ofstream out; + m->openOutputFileAppend(filehandlesHeaders[i][j], out); + out.write(mybuffer, spot-spotInFile); + out.close(); + } + } + delete[] mybuffer; + in.close(); + return 0; + + } + catch(exception& e) { + m->errorOut(e, "SffInfoCommand", "adjustCommonHeader"); + exit(1); + } +} +//********************************************************************************************************************** +int SffInfoCommand::readSeqData(ifstream& in, seqRead& read, int numFlowReads, Header& header){ + try { + unsigned long long startSpotInFile = in.tellg(); + if (!in.eof()) { + + /*****************************************/ + //read header + + //read header length char buffer [2]; in.read(buffer, 2); header.headerLength = be_int2(*(unsigned short *)(&buffer)); - + //read name length char buffer2 [2]; in.read(buffer2, 2); header.nameLength = be_int2(*(unsigned short *)(&buffer2)); - + //read num bases char buffer3 [4]; in.read(buffer3, 4); @@ -592,12 +939,12 @@ int SffInfoCommand::readHeader(ifstream& in, Header& header){ char buffer6 [2]; in.read(buffer6, 2); header.clipAdapterLeft = be_int2(*(unsigned short *)(&buffer6)); - + //read clipAdapterRight char buffer7 [2]; in.read(buffer7, 2); header.clipAdapterRight = be_int2(*(unsigned short *)(&buffer7)); - + //read name char* tempBuffer = new char[header.nameLength]; in.read(&(*tempBuffer), header.nameLength); @@ -612,24 +959,10 @@ int SffInfoCommand::readHeader(ifstream& in, Header& header){ unsigned long long spotInFile = in.tellg(); unsigned long long spot = (spotInFile + 7)& ~7; in.seekg(spot); - - }else{ - m->mothurOut("Error reading sff header info."); m->mothurOutEndLine(); - } - return 0; - } - catch(exception& e) { - m->errorOut(e, "SffInfoCommand", "readHeader"); - exit(1); - } -} -//********************************************************************************************************************** -int SffInfoCommand::readSeqData(ifstream& in, seqRead& read, int numFlowReads, int numBases){ - try { - - if (!in.eof()) { - + /*****************************************/ + //sequence read + //read flowgram read.flowgram.resize(numFlowReads); for (int i = 0; i < numFlowReads; i++) { @@ -639,33 +972,62 @@ int SffInfoCommand::readSeqData(ifstream& in, seqRead& read, int numFlowReads, i } //read flowIndex - read.flowIndex.resize(numBases); - for (int i = 0; i < numBases; i++) { + read.flowIndex.resize(header.numBases); + for (int i = 0; i < header.numBases; i++) { char temp[1]; in.read(temp, 1); read.flowIndex[i] = be_int1(*(unsigned char *)(&temp)); } //read bases - char* tempBuffer = new char[numBases]; - in.read(&(*tempBuffer), numBases); - read.bases = tempBuffer; - if (read.bases.length() > numBases) { read.bases = read.bases.substr(0, numBases); } - delete[] tempBuffer; + char* tempBuffer6 = new char[header.numBases]; + in.read(&(*tempBuffer6), header.numBases); + read.bases = tempBuffer6; + if (read.bases.length() > header.numBases) { read.bases = read.bases.substr(0, header.numBases); } + delete[] tempBuffer6; //read qual scores - read.qualScores.resize(numBases); - for (int i = 0; i < numBases; i++) { + read.qualScores.resize(header.numBases); + for (int i = 0; i < header.numBases; i++) { char temp[1]; in.read(temp, 1); read.qualScores[i] = be_int1(*(unsigned char *)(&temp)); } /* Pad to 8 chars */ - unsigned long long spotInFile = in.tellg(); - unsigned long long spot = (spotInFile + 7)& ~7; + spotInFile = in.tellg(); + spot = (spotInFile + 7)& ~7; in.seekg(spot); - + + if (split > 1) { + char * mybuffer; + mybuffer = new char [spot-startSpotInFile]; + ifstream in2; + m->openInputFile(currentFileName, in2); + in2.seekg(startSpotInFile); + in2.read(mybuffer,spot-startSpotInFile); + in2.close(); + + int barcodeIndex, primerIndex; + int trashCodeLength = findGroup(header, read, barcodeIndex, primerIndex); + + if(trashCodeLength == 0){ + ofstream out; + m->openOutputFileAppend(filehandles[barcodeIndex][primerIndex], out); + out.write(mybuffer, in2.gcount()); + out.close(); + delete[] mybuffer; + numSplitReads[barcodeIndex][primerIndex]++; + } + else{ + ofstream out; + m->openOutputFileAppend(noMatchFile, out); + out.write(mybuffer, in2.gcount()); + out.close(); + delete[] mybuffer; + } + + } }else{ m->mothurOut("Error reading."); m->mothurOutEndLine(); } @@ -678,6 +1040,89 @@ int SffInfoCommand::readSeqData(ifstream& in, seqRead& read, int numFlowReads, i } } //********************************************************************************************************************** +int SffInfoCommand::findGroup(Header header, seqRead read, int& barcode, int& primer) { + try { + //find group read belongs to + TrimOligos trimOligos(pdiffs, bdiffs, ldiffs, sdiffs, primers, barcodes, rbarcodes, revPrimer, linker, spacer); + + int success = 1; + string trashCode = ""; + int currentSeqsDiffs = 0; + + string seq = read.bases; + + if (trim) { + if(header.clipQualRight < header.clipQualLeft){ + seq = "NNNN"; + } + else if((header.clipQualRight != 0) && ((header.clipQualRight-header.clipQualLeft) >= 0)){ + seq = seq.substr((header.clipQualLeft-1), (header.clipQualRight-header.clipQualLeft)); + } + else { + seq = seq.substr(header.clipQualLeft-1); + } + }else{ + //if you wanted the sfftxt then you already converted the bases to the right case + if (!sfftxt) { + //make the bases you want to clip lowercase and the bases you want to keep upper case + if(header.clipQualRight == 0){ header.clipQualRight = seq.length(); } + for (int i = 0; i < (header.clipQualLeft-1); i++) { seq[i] = tolower(seq[i]); } + for (int i = (header.clipQualLeft-1); i < (header.clipQualRight-1); i++) { seq[i] = toupper(seq[i]); } + for (int i = (header.clipQualRight-1); i < seq.length(); i++) { seq[i] = tolower(seq[i]); } + } + } + + Sequence currSeq(header.name, seq); + QualityScores currQual; + + if(numLinkers != 0){ + success = trimOligos.stripLinker(currSeq, currQual); + if(success > ldiffs) { trashCode += 'k'; } + else{ currentSeqsDiffs += success; } + + } + + if(barcodes.size() != 0){ + success = trimOligos.stripBarcode(currSeq, currQual, barcode); + if(success > bdiffs) { trashCode += 'b'; } + else{ currentSeqsDiffs += success; } + } + + if(rbarcodes.size() != 0){ + success = trimOligos.stripRBarcode(currSeq, currQual, barcode); + if(success > bdiffs) { trashCode += 'b'; } + else{ currentSeqsDiffs += success; } + } + + if(numSpacers != 0){ + success = trimOligos.stripSpacer(currSeq, currQual); + if(success > sdiffs) { trashCode += 's'; } + else{ currentSeqsDiffs += success; } + + } + + if(numFPrimers != 0){ + success = trimOligos.stripForward(currSeq, currQual, primer, true); + if(success > pdiffs) { trashCode += 'f'; } + else{ currentSeqsDiffs += success; } + } + + if (currentSeqsDiffs > tdiffs) { trashCode += 't'; } + + if(revPrimer.size() != 0){ + success = trimOligos.stripReverse(currSeq, currQual); + if(!success) { trashCode += 'r'; } + } + + + return trashCode.length(); + } + catch(exception& e) { + m->errorOut(e, "SffInfoCommand", "findGroup"); + exit(1); + } +} +//********************************************************************************************************************** int SffInfoCommand::decodeName(string& timestamp, string& region, string& xy, string name) { try { @@ -1175,6 +1620,247 @@ vector SffInfoCommand::parseHeaderLineToIntVector(ifstream& file, exit(1); } } +//*************************************************************************************************************** + +bool SffInfoCommand::readOligos(string oligoFile){ + try { + filehandles.clear(); + numSplitReads.clear(); + filehandlesHeaders.clear(); + + ifstream inOligos; + m->openInputFile(oligoFile, inOligos); + + string type, oligo, group; + + int indexPrimer = 0; + int indexBarcode = 0; + + while(!inOligos.eof()){ + + inOligos >> type; + + if(type[0] == '#'){ + while (!inOligos.eof()) { char c = inOligos.get(); if (c == 10 || c == 13){ break; } } // get rest of line if there's any crap there + m->gobble(inOligos); + } + else{ + m->gobble(inOligos); + //make type case insensitive + for(int i=0;i> oligo; + + for(int i=0;i::iterator itPrime = primers.find(oligo); + if (itPrime != primers.end()) { m->mothurOut("primer " + oligo + " is in your oligos file already."); m->mothurOutEndLine(); } + + primers[oligo]=indexPrimer; indexPrimer++; + primerNameVector.push_back(group); + }else if(type == "REVERSE"){ + //Sequence oligoRC("reverse", oligo); + //oligoRC.reverseComplement(); + string oligoRC = reverseOligo(oligo); + revPrimer.push_back(oligoRC); + } + else if(type == "BARCODE"){ + inOligos >> group; + + //barcode lines can look like BARCODE atgcatgc groupName - for 454 seqs + //or BARCODE atgcatgc atgcatgc groupName - for illumina data that has forward and reverse info + string temp = ""; + while (!inOligos.eof()) { + char c = inOligos.get(); + if (c == 10 || c == 13){ break; } + else if (c == 32 || c == 9){;} //space or tab + else { temp += c; } + } + + //then this is illumina data with 4 columns + if (temp != "") { + string reverseBarcode = reverseOligo(group); //reverse barcode + group = temp; + + //check for repeat barcodes + map::iterator itBar = rbarcodes.find(reverseBarcode); + if (itBar != rbarcodes.end()) { m->mothurOut("barcode " + reverseBarcode + " is in your oligos file already."); m->mothurOutEndLine(); } + + rbarcodes[reverseBarcode]=indexBarcode; + } + + //check for repeat barcodes + map::iterator itBar = barcodes.find(oligo); + if (itBar != barcodes.end()) { m->mothurOut("barcode " + oligo + " is in your oligos file already."); m->mothurOutEndLine(); } + + barcodes[oligo]=indexBarcode; indexBarcode++; + barcodeNameVector.push_back(group); + + }else if(type == "LINKER"){ + linker.push_back(oligo); + }else if(type == "SPACER"){ + spacer.push_back(oligo); + } + else{ m->mothurOut("[WARNING]: " + type + " is not recognized as a valid type. Choices are forward, reverse, and barcode. Ignoring " + oligo + "."); m->mothurOutEndLine(); } + } + m->gobble(inOligos); + } + inOligos.close(); + + if(barcodeNameVector.size() == 0 && primerNameVector[0] == ""){ split = 1; } + + //add in potential combos + if(barcodeNameVector.size() == 0){ + barcodes[""] = 0; + barcodeNameVector.push_back(""); + } + + if(primerNameVector.size() == 0){ + primers[""] = 0; + primerNameVector.push_back(""); + } + + filehandles.resize(barcodeNameVector.size()); + for(int i=0;i 1){ + set uniqueNames; //used to cleanup outputFileNames + for(map::iterator itBar = barcodes.begin();itBar != barcodes.end();itBar++){ + for(map::iterator itPrimer = primers.begin();itPrimer != primers.end(); itPrimer++){ + + string primerName = primerNameVector[itPrimer->second]; + string barcodeName = barcodeNameVector[itBar->second]; + + string comboGroupName = ""; + string fastaFileName = ""; + string qualFileName = ""; + string nameFileName = ""; + + if(primerName == ""){ + comboGroupName = barcodeNameVector[itBar->second]; + } + else{ + if(barcodeName == ""){ + comboGroupName = primerNameVector[itPrimer->second]; + } + else{ + comboGroupName = barcodeNameVector[itBar->second] + "." + primerNameVector[itPrimer->second]; + } + } + + ofstream temp; + string thisFilename = outputDir + m->getRootName(m->getSimpleName(currentFileName)) + comboGroupName + "." + getOutputFileNameTag("sff"); + if (uniqueNames.count(thisFilename) == 0) { + outputNames.push_back(thisFilename); + outputTypes["sff"].push_back(thisFilename); + uniqueNames.insert(thisFilename); + } + + filehandles[itBar->second][itPrimer->second] = thisFilename; + m->openOutputFile(thisFilename, temp); temp.close(); + } + } + } + numFPrimers = primers.size(); + numLinkers = linker.size(); + numSpacers = spacer.size(); + noMatchFile = outputDir + m->getRootName(m->getSimpleName(currentFileName)) + "scrap." + getOutputFileNameTag("sff"); + m->mothurRemove(noMatchFile); + + bool allBlank = true; + for (int i = 0; i < barcodeNameVector.size(); i++) { + if (barcodeNameVector[i] != "") { + allBlank = false; + break; + } + } + for (int i = 0; i < primerNameVector.size(); i++) { + if (primerNameVector[i] != "") { + allBlank = false; + break; + } + } + + filehandlesHeaders.resize(filehandles.size()); + numSplitReads.resize(filehandles.size()); + for (int i = 0; i < filehandles.size(); i++) { + numSplitReads[i].resize(filehandles[i].size(), 0); + for (int j = 0; j < filehandles[i].size(); j++) { + filehandlesHeaders[i].push_back(filehandles[i][j]+"headers"); + } + } + + if (allBlank) { + m->mothurOut("[WARNING]: your oligos file does not contain any group names. mothur will not create a split the sff file."); m->mothurOutEndLine(); + split = 1; + return false; + } + + return true; + + } + catch(exception& e) { + m->errorOut(e, "SffInfoCommand", "readOligos"); + exit(1); + } +} +//********************************************************************/ +string SffInfoCommand::reverseOligo(string oligo){ + try { + string reverse = ""; + + for(int i=oligo.length()-1;i>=0;i--){ + + if(oligo[i] == 'A') { reverse += 'T'; } + else if(oligo[i] == 'T'){ reverse += 'A'; } + else if(oligo[i] == 'U'){ reverse += 'A'; } + + else if(oligo[i] == 'G'){ reverse += 'C'; } + else if(oligo[i] == 'C'){ reverse += 'G'; } + + else if(oligo[i] == 'R'){ reverse += 'Y'; } + else if(oligo[i] == 'Y'){ reverse += 'R'; } + + else if(oligo[i] == 'M'){ reverse += 'K'; } + else if(oligo[i] == 'K'){ reverse += 'M'; } + + else if(oligo[i] == 'W'){ reverse += 'W'; } + else if(oligo[i] == 'S'){ reverse += 'S'; } + + else if(oligo[i] == 'B'){ reverse += 'V'; } + else if(oligo[i] == 'V'){ reverse += 'B'; } + + else if(oligo[i] == 'D'){ reverse += 'H'; } + else if(oligo[i] == 'H'){ reverse += 'D'; } + + else { reverse += 'N'; } + } + + + return reverse; + } + catch(exception& e) { + m->errorOut(e, "SffInfoCommand", "reverseOligo"); + exit(1); + } +} //********************************************************************************************************************** diff --git a/sffinfocommand.h b/sffinfocommand.h index 4e72a96..dd3c57f 100644 --- a/sffinfocommand.h +++ b/sffinfocommand.h @@ -78,18 +78,25 @@ public: void help() { m->mothurOut(getHelpString()); } private: - string sffFilename, sfftxtFilename, outputDir, accnosName; - vector filenames, outputNames, accnosFileNames; - bool abort, fasta, qual, trim, flow, sfftxt, hasAccnos; - int mycount; + string sffFilename, sfftxtFilename, outputDir, accnosName, currentFileName, oligosfile, noMatchFile; + vector filenames, outputNames, accnosFileNames, oligosFileNames; + bool abort, fasta, qual, trim, flow, sfftxt, hasAccnos, hasOligos; + int mycount, split, numFPrimers, numLinkers, numSpacers, pdiffs, bdiffs, ldiffs, sdiffs, tdiffs; set seqNames; + map barcodes; + map rbarcodes; + map primers; + vector linker, spacer, primerNameVector, barcodeNameVector, revPrimer; + vector > numSplitReads; + vector > filehandles, filehandlesHeaders; //extract sff file functions - int extractSffInfo(string, string); + int extractSffInfo(string, string, string); int readCommonHeader(ifstream&, CommonHeader&); - int readHeader(ifstream&, Header&); - int readSeqData(ifstream&, seqRead&, int, int); + //int readHeader(ifstream&, Header&); + int readSeqData(ifstream&, seqRead&, int, Header&); int decodeName(string&, string&, string&, string); + bool readOligos(string oligosFile); int printCommonHeader(ofstream&, CommonHeader&); int printHeader(ofstream&, Header&); @@ -100,6 +107,9 @@ private: int readAccnosFile(string); int parseSffTxt(); bool sanityCheck(Header&, seqRead&); + int adjustCommonHeader(CommonHeader); + int findGroup(Header header, seqRead read, int& barcode, int& primer); + string reverseOligo(string oligo); //parsesfftxt file functions int parseHeaderLineToInt(ifstream&); diff --git a/sffmultiplecommand.cpp b/sffmultiplecommand.cpp index 1675d24..e9c4784 100644 --- a/sffmultiplecommand.cpp +++ b/sffmultiplecommand.cpp @@ -7,6 +7,11 @@ // #include "sffmultiplecommand.h" +#include "sffinfocommand.h" +#include "seqsummarycommand.h" +#include "trimflowscommand.h" +#include "shhhercommand.h" +#include "trimseqscommand.h" //********************************************************************************************************************** @@ -28,9 +33,34 @@ vector SffMultipleCommand::setParameters(){ CommandParameter ptdiffs("tdiffs", "Number", "", "0", "", "", "",false,false); parameters.push_back(ptdiffs); CommandParameter psignal("signal", "Number", "", "0.50", "", "", "",false,false); parameters.push_back(psignal); CommandParameter pnoise("noise", "Number", "", "0.70", "", "", "",false,false); parameters.push_back(pnoise); - CommandParameter pallfiles("allfiles", "Boolean", "", "t", "", "", "",false,false); parameters.push_back(pallfiles); CommandParameter porder("order", "String", "", "", "", "", "",false,false); parameters.push_back(porder); + //shhh.flows + CommandParameter plookup("lookup", "InputTypes", "", "", "none", "none", "none",false,false); parameters.push_back(plookup); + CommandParameter pcutoff("cutoff", "Number", "", "0.01", "", "", "",false,false); parameters.push_back(pcutoff); + CommandParameter pmaxiter("maxiter", "Number", "", "1000", "", "", "",false,false); parameters.push_back(pmaxiter); + CommandParameter plarge("large", "Number", "", "-1", "", "", "",false,false); parameters.push_back(plarge); + CommandParameter psigma("sigma", "Number", "", "60", "", "", "",false,false); parameters.push_back(psigma); + CommandParameter pmindelta("mindelta", "Number", "", "0.000001", "", "", "",false,false); parameters.push_back(pmindelta); + + //trim.seqs parameters + CommandParameter pallfiles("allfiles", "Boolean", "", "t", "", "", "",false,false); parameters.push_back(pallfiles); + CommandParameter pflip("flip", "Boolean", "", "F", "", "", "",false,false); parameters.push_back(pflip); + CommandParameter pmaxambig("maxambig", "Number", "", "-1", "", "", "",false,false); parameters.push_back(pmaxambig); + CommandParameter pminlength("minlength", "Number", "", "0", "", "", "",false,false); parameters.push_back(pminlength); + CommandParameter pmaxlength("maxlength", "Number", "", "0", "", "", "",false,false); parameters.push_back(pmaxlength); + CommandParameter pkeepforward("keepforward", "Boolean", "", "F", "", "", "",false,false); parameters.push_back(pkeepforward); + CommandParameter pqtrim("qtrim", "Boolean", "", "T", "", "", "",false,false); parameters.push_back(pqtrim); + CommandParameter pqthreshold("qthreshold", "Number", "", "0", "", "", "",false,false); parameters.push_back(pqthreshold); + CommandParameter pqaverage("qaverage", "Number", "", "0", "", "", "",false,false); parameters.push_back(pqaverage); + CommandParameter prollaverage("rollaverage", "Number", "", "0", "", "", "",false,false); parameters.push_back(prollaverage); + CommandParameter pqwindowaverage("qwindowaverage", "Number", "", "0", "", "", "",false,false); parameters.push_back(pqwindowaverage); + CommandParameter pqstepsize("qstepsize", "Number", "", "1", "", "", "",false,false); parameters.push_back(pqstepsize); + CommandParameter pqwindowsize("qwindowsize", "Number", "", "50", "", "", "",false,false); parameters.push_back(pqwindowsize); + CommandParameter pkeepfirst("keepfirst", "Number", "", "0", "", "", "",false,false); parameters.push_back(pkeepfirst); + CommandParameter premovelast("removelast", "Number", "", "0", "", "", "",false,false); parameters.push_back(premovelast); + + CommandParameter pprocessors("processors", "Number", "", "1", "", "", "",false,false); parameters.push_back(pprocessors); CommandParameter pinputdir("inputdir", "String", "", "", "", "", "",false,false); parameters.push_back(pinputdir); CommandParameter poutputdir("outputdir", "String", "", "", "", "", "",false,false); parameters.push_back(poutputdir); @@ -49,9 +79,36 @@ string SffMultipleCommand::getHelpString(){ try { string helpString = ""; helpString += "The sff.multiple command reads a file containing sff filenames and optional oligos filenames. It runs the files through sffinfo, trim.flows, shhh.flows and trim.seqs combining the results.\n"; - helpString += "The sff.multiple command parameters are file, trim, maxhomop, maxflows, minflows, pdiffs, bdiffs, ldiffs, sdiffs, tdiffs, signal, noise, allfiles, order. file is required. \n"; + helpString += "The sff.multiple command parameters are: "; + vector parameters = setParameters(); + for (int i = 0; i < parameters.size()-1; i++) { + helpString += parameters[i] + ", "; + } + helpString += parameters[parameters.size()-1] + ".\n"; helpString += "The file parameter allows you to enter the a file containing the list of sff files and optional oligos files.\n"; helpString += "The trim parameter allows you to indicate if you would like a sequences and quality scores generated by sffinfo trimmed to the clipQualLeft and clipQualRight values. Default=True. \n"; + helpString += "The maxambig parameter allows you to set the maximum number of ambigious bases allowed. The default is -1.\n"; + helpString += "The maxhomop parameter allows you to set a maximum homopolymer length. \n"; + helpString += "The minlength parameter allows you to set and minimum sequence length. \n"; + helpString += "The maxlength parameter allows you to set and maximum sequence length. \n"; + helpString += "The tdiffs parameter is used to specify the total number of differences allowed in the sequence. The default is pdiffs + bdiffs + sdiffs + ldiffs.\n"; + helpString += "The bdiffs parameter is used to specify the number of differences allowed in the barcode. The default is 0.\n"; + helpString += "The pdiffs parameter is used to specify the number of differences allowed in the primer. The default is 0.\n"; + helpString += "The ldiffs parameter is used to specify the number of differences allowed in the linker. The default is 0.\n"; + helpString += "The sdiffs parameter is used to specify the number of differences allowed in the spacer. The default is 0.\n"; + helpString += "The qfile parameter allows you to provide a quality file.\n"; + helpString += "The qthreshold parameter allows you to set a minimum quality score allowed. \n"; + helpString += "The qaverage parameter allows you to set a minimum average quality score allowed. \n"; + helpString += "The qwindowsize parameter allows you to set a number of bases in a window. Default=50.\n"; + helpString += "The qwindowaverage parameter allows you to set a minimum average quality score allowed over a window. \n"; + helpString += "The rollaverage parameter allows you to set a minimum rolling average quality score allowed over a window. \n"; + helpString += "The qstepsize parameter allows you to set a number of bases to move the window over. Default=1.\n"; + helpString += "The allfiles parameter will create separate group and fasta file for each grouping. The default is F.\n"; + helpString += "The keepforward parameter allows you to indicate whether you want the forward primer removed or not. The default is F, meaning remove the forward primer.\n"; + helpString += "The qtrim parameter will trim sequence from the point that they fall below the qthreshold and put it in the .trim file if set to true. The default is T.\n"; + helpString += "The keepfirst parameter trims the sequence to the first keepfirst number of bases after the barcode or primers are removed, before the sequence is checked to see if it meets the other requirements. \n"; + helpString += "The removelast removes the last removelast number of bases after the barcode or primers are removed, before the sequence is checked to see if it meets the other requirements.\n"; + helpString += "Example sff.multiple(file=mySffOligosFile.txt, trim=F).\n"; helpString += "Note: No spaces between parameter labels (i.e. file), '=' and parameters (i.e.mySffOligosFile.txt).\n"; return helpString; @@ -71,13 +128,7 @@ string SffMultipleCommand::getOutputFileNameTag(string type, string inputName="" it = outputTypes.find(type); if (it == outputTypes.end()) { m->mothurOut("[ERROR]: this command doesn't create a " + type + " output file.\n"); } else { - //if (type == "fasta") { outputFileName = "fasta"; } - //else if (type == "flow") { outputFileName = "flow"; } - // else if (type == "sfftxt") { outputFileName = "sff.txt"; } - //else if (type == "qfile") { outputFileName = "qual"; } - //else { - m->mothurOut("[ERROR]: No definition for type " + type + " output file tag.\n"); m->control_pressed = true; - //} + m->mothurOut("[ERROR]: No definition for type " + type + " output file tag.\n"); m->control_pressed = true; } return outputFileName; } @@ -158,6 +209,118 @@ SffMultipleCommand::SffMultipleCommand(string option) { string temp; temp = validParameter.validFile(parameters, "trim", false); if (temp == "not found"){ temp = "T"; } trim = m->isTrue(temp); + + temp = validParameter.validFile(parameters, "minflows", false); if (temp == "not found") { temp = "450"; } + m->mothurConvert(temp, minFlows); + + temp = validParameter.validFile(parameters, "maxflows", false); if (temp == "not found") { temp = "450"; } + m->mothurConvert(temp, maxFlows); + + temp = validParameter.validFile(parameters, "maxhomop", false); if (temp == "not found"){ temp = "9"; } + m->mothurConvert(temp, maxHomoP); + + temp = validParameter.validFile(parameters, "signal", false); if (temp == "not found"){ temp = "0.50"; } + m->mothurConvert(temp, signal); + + temp = validParameter.validFile(parameters, "noise", false); if (temp == "not found"){ temp = "0.70"; } + m->mothurConvert(temp, noise); + + temp = validParameter.validFile(parameters, "bdiffs", false); if (temp == "not found"){ temp = "0"; } + m->mothurConvert(temp, bdiffs); + + temp = validParameter.validFile(parameters, "pdiffs", false); if (temp == "not found"){ temp = "0"; } + m->mothurConvert(temp, pdiffs); + + temp = validParameter.validFile(parameters, "ldiffs", false); if (temp == "not found") { temp = "0"; } + m->mothurConvert(temp, ldiffs); + + temp = validParameter.validFile(parameters, "sdiffs", false); if (temp == "not found") { temp = "0"; } + m->mothurConvert(temp, sdiffs); + + temp = validParameter.validFile(parameters, "tdiffs", false); if (temp == "not found") { int tempTotal = pdiffs + bdiffs + ldiffs + sdiffs; temp = toString(tempTotal); } + m->mothurConvert(temp, tdiffs); + + if(tdiffs == 0){ tdiffs = bdiffs + pdiffs + ldiffs + sdiffs; } + + + temp = validParameter.validFile(parameters, "processors", false); if (temp == "not found"){ temp = m->getProcessors(); } + m->setProcessors(temp); + m->mothurConvert(temp, processors); + + flowOrder = validParameter.validFile(parameters, "order", false); + if (flowOrder == "not found"){ flowOrder = "TACG"; } + else if(flowOrder.length() != 4){ + m->mothurOut("The value of the order option must be four bases long\n"); + } + + temp = validParameter.validFile(parameters, "cutoff", false); if (temp == "not found"){ temp = "0.01"; } + m->mothurConvert(temp, cutoff); + + temp = validParameter.validFile(parameters, "mindelta", false); if (temp == "not found"){ temp = "0.000001"; } + m->mothurConvert(temp, minDelta); + + temp = validParameter.validFile(parameters, "maxiter", false); if (temp == "not found"){ temp = "1000"; } + m->mothurConvert(temp, maxIters); + + temp = validParameter.validFile(parameters, "large", false); if (temp == "not found"){ temp = "0"; } + m->mothurConvert(temp, largeSize); + if (largeSize != 0) { large = true; } + else { large = false; } + if (largeSize < 0) { m->mothurOut("The value of the large cannot be negative.\n"); } + + temp = validParameter.validFile(parameters, "sigma", false);if (temp == "not found") { temp = "60"; } + m->mothurConvert(temp, sigma); + + temp = validParameter.validFile(parameters, "flip", false); + if (temp == "not found") { flip = 0; } + else { flip = m->isTrue(temp); } + + temp = validParameter.validFile(parameters, "maxambig", false); if (temp == "not found") { temp = "-1"; } + m->mothurConvert(temp, maxAmbig); + + temp = validParameter.validFile(parameters, "minlength", false); if (temp == "not found") { temp = "0"; } + m->mothurConvert(temp, minLength); + + temp = validParameter.validFile(parameters, "maxlength", false); if (temp == "not found") { temp = "0"; } + m->mothurConvert(temp, maxLength); + + temp = validParameter.validFile(parameters, "qthreshold", false); if (temp == "not found") { temp = "0"; } + m->mothurConvert(temp, qThreshold); + + temp = validParameter.validFile(parameters, "qtrim", false); if (temp == "not found") { temp = "t"; } + qtrim = m->isTrue(temp); + + temp = validParameter.validFile(parameters, "rollaverage", false); if (temp == "not found") { temp = "0"; } + convert(temp, qRollAverage); + + temp = validParameter.validFile(parameters, "qwindowaverage", false);if (temp == "not found") { temp = "0"; } + convert(temp, qWindowAverage); + + temp = validParameter.validFile(parameters, "qwindowsize", false); if (temp == "not found") { temp = "50"; } + convert(temp, qWindowSize); + + temp = validParameter.validFile(parameters, "qstepsize", false); if (temp == "not found") { temp = "1"; } + convert(temp, qWindowStep); + + temp = validParameter.validFile(parameters, "qaverage", false); if (temp == "not found") { temp = "0"; } + convert(temp, qAverage); + + temp = validParameter.validFile(parameters, "keepfirst", false); if (temp == "not found") { temp = "0"; } + convert(temp, keepFirst); + + temp = validParameter.validFile(parameters, "removelast", false); if (temp == "not found") { temp = "0"; } + convert(temp, removeLast); + + temp = validParameter.validFile(parameters, "allfiles", false); if (temp == "not found") { temp = "F"; } + allFiles = m->isTrue(temp); + + temp = validParameter.validFile(parameters, "keepforward", false); if (temp == "not found") { temp = "F"; } + keepforward = m->isTrue(temp); + + numFPrimers = 0; + numRPrimers = 0; + numLinkers = 0; + numSpacers = 0; } } catch(exception& e) { @@ -175,26 +338,13 @@ int SffMultipleCommand::execute(){ if (m->control_pressed) { return 0; } + if (sffFiles.size() < processors) { processors = sffFiles.size(); } + + if (processors == 1) { driver(sffFiles, oligosFiles, 0, sffFiles.size()); } + else { createProcesses(sffFiles, oligosFiles); } if (m->control_pressed) { for (int i = 0; i < outputNames.size(); i++) { m->mothurRemove(outputNames[i]); } return 0; } - //set fasta file as new current fastafile - string current = ""; - itTypes = outputTypes.find("fasta"); - if (itTypes != outputTypes.end()) { - if ((itTypes->second).size() != 0) { current = (itTypes->second)[0]; m->setFastaFile(current); } - } - - itTypes = outputTypes.find("qfile"); - if (itTypes != outputTypes.end()) { - if ((itTypes->second).size() != 0) { current = (itTypes->second)[0]; m->setQualFile(current); } - } - - itTypes = outputTypes.find("flow"); - if (itTypes != outputTypes.end()) { - if ((itTypes->second).size() != 0) { current = (itTypes->second)[0]; m->setFlowFile(current); } - } - //report output filenames m->mothurOutEndLine(); m->mothurOut("Output File Names: "); m->mothurOutEndLine(); @@ -250,6 +400,175 @@ int SffMultipleCommand::readFile(vector& sffFiles, vector& oligo } } //********************************************************************************************************************** +int SffMultipleCommand::driver(vector sffFiles, vector oligosFiles, int start, int end){ + try { + int count = 0; + for (int i = start; i < end; i++) { + string sff = sffFiles[i]; + string oligos = oligosFiles[i]; + + //run sff.info + string inputString = "sff=" + sff + ", flow=T"; + if (trim) { inputString += ", trim=T"; } + m->mothurOut("/******************************************/"); m->mothurOutEndLine(); + m->mothurOut("Running command: sffinfo(" + inputString + ")"); m->mothurOutEndLine(); + m->mothurCalling = true; + + Command* sffCommand = new SffInfoCommand(inputString); + sffCommand->execute(); + + map > filenames = sffCommand->getOutputFiles(); + + delete sffCommand; + m->mothurCalling = false; + m->mothurOut("/******************************************/"); m->mothurOutEndLine(); + + //run summary.seqs on the fasta file + string fastaFile = ""; + map >::iterator it = filenames.find("fasta"); + if (it != filenames.end()) { if ((it->second).size() != 0) { fastaFile = (it->second)[0]; } } + else { m->mothurOut("[ERROR]: sffinfo did not create a fasta file, quitting.\n"); m->control_pressed = true; break; } + + inputString = "fasta=" + fastaFile; + m->mothurOut("/******************************************/"); m->mothurOutEndLine(); + m->mothurOut("Running command: summary.seqs(" + inputString + ")"); m->mothurOutEndLine(); + m->mothurCalling = true; + + Command* summarySeqsCommand = new SeqSummaryCommand(inputString); + summarySeqsCommand->execute(); + + delete summarySeqsCommand; + m->mothurCalling = false; + m->mothurOut("/******************************************/"); m->mothurOutEndLine(); + + count++; + } + + return count; + } + catch(exception& e) { + m->errorOut(e, "SffMultipleCommand", "driver"); + exit(1); + } +} +//********************************************************************************************************************** +int SffMultipleCommand::createProcesses(vector sffFiles, vector oligosFiles){ + try { + vector processIDS; + int process = 1; + int num = 0; + + //divide the groups between the processors + vector lines; + vector numFilesToComplete; + int numFilesPerProcessor = sffFiles.size() / processors; + for (int i = 0; i < processors; i++) { + int startIndex = i * numFilesPerProcessor; + int endIndex = (i+1) * numFilesPerProcessor; + if(i == (processors - 1)){ endIndex = sffFiles.size(); } + lines.push_back(linePair(startIndex, endIndex)); + numFilesToComplete.push_back((endIndex-startIndex)); + } + +#if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix) + + //loop through and create all the processes you want + while (process != processors) { + int pid = fork(); + + if (pid > 0) { + processIDS.push_back(pid); //create map from line number to pid so you can append files in correct order later + process++; + }else if (pid == 0){ + num = driver(sffFiles, oligosFiles, lines[process].start, lines[process].end); + + //pass numSeqs to parent + ofstream out; + string tempFile = toString(getpid()) + ".num.temp"; + m->openOutputFile(tempFile, out); + out << num << endl; + out.close(); + + exit(0); + }else { + m->mothurOut("[ERROR]: unable to spawn the necessary processes."); m->mothurOutEndLine(); + for (int i = 0; i < processIDS.size(); i++) { kill (processIDS[i], SIGINT); } + exit(0); + } + } + + //do my part + num = driver(sffFiles, oligosFiles, lines[0].start, lines[0].end); + + //force parent to wait until all the processes are done + for (int i=0;i pDataArray; + DWORD dwThreadIdArray[processors-1]; + HANDLE hThreadArray[processors-1]; + + //Create processor worker threads. + for( int i=0; ioutputNames.size(); j++){ outputNames.push_back(pDataArray[i]->outputNames[j]); } + CloseHandle(hThreadArray[i]); + delete pDataArray[i]; + } + */ +#endif + + for (int i=0;iopenInputFile(tempFile, in); + if (!in.eof()) { + int tempNum = 0; + in >> tempNum; + if (tempNum != numFilesToComplete[i+1]) { + m->mothurOut("[ERROR]: main process expected " + toString(processIDS[i]) + " to complete " + toString(numFilesToComplete[i+1]) + " files, and it only reported completing " + toString(tempNum) + ". This will cause file mismatches. The flow files may be too large to process with multiple processors. \n"); + } + } + in.close(); m->mothurRemove(tempFile); + } + + return 0; + + } + catch(exception& e) { + m->errorOut(e, "ShhherCommand", "createProcesses"); + exit(1); + } +} +//********************************************************************************************************************** diff --git a/sffmultiplecommand.h b/sffmultiplecommand.h index ae881aa..ecf33b7 100644 --- a/sffmultiplecommand.h +++ b/sffmultiplecommand.h @@ -30,14 +30,26 @@ public: void help() { m->mothurOut(getHelpString()); } private: + + struct linePair { + int start; + int end; + linePair(int i, int j) : start(i), end(j) {} + }; + string filename, outputDir, flowOrder; vector outputNames; - bool abort, trim; + bool abort, trim, large, flip, qtrim, allFiles, keepforward; int maxFlows, minFlows, minLength, maxLength, maxHomoP, tdiffs, bdiffs, pdiffs, sdiffs, ldiffs, numLinkers, numSpacers; - int numFlows, numFPrimers, numRPrimers, processors; - float signal, noise; + int numFlows, numFPrimers, numRPrimers, processors, maxIters, largeSize; + float signal, noise, cutoff, sigma, minDelta; + int qWindowSize, qWindowStep, keepFirst, removeLast, maxAmbig; + double qRollAverage, qThreshold, qWindowAverage, qAverage; int readFile(vector& sffFiles, vector& oligosFiles); + int createProcesses(vector sffFiles, vector oligosFiles); + int driver(vector sffFiles, vector oligosFiles, int start, int end); + }; diff --git a/trimseqscommand.cpp b/trimseqscommand.cpp index ae8dfb7..88ed32b 100644 --- a/trimseqscommand.cpp +++ b/trimseqscommand.cpp @@ -611,6 +611,7 @@ int TrimSeqsCommand::driverCreateTrim(string filename, string qFileName, string QualityScores currQual; if(qFileName != ""){ currQual = QualityScores(qFile); m->gobble(qFile); + //cout << currQual.getName() << endl; } string origSeq = currSeq.getUnaligned(); -- 2.39.2