+//runs sffinfo, summary.seqs, trim.flows, shhh.flows, trim.seqs, summary.seqs for each sff file.
+int SffMultipleCommand::driver(vector<string> sffFiles, vector<string> oligosFiles, int start, int end, string fasta, string name, string group){
+ try {
+ m->mothurRemove(fasta); m->mothurRemove(name); m->mothurRemove(group);
+ int count = 0;
+ for (int s = start; s < end; s++) {
+
+ string sff = sffFiles[s];
+ string oligos = oligosFiles[s];
+
+ m->mothurOut("\n>>>>>\tProcessing " + sff + " (file " + toString(s+1) + " of " + toString(sffFiles.size()) + ")\t<<<<<\n");
+
+ //run sff.info
+ string inputString = "sff=" + sff + ", flow=T";
+ if (trim) { inputString += ", trim=T"; }
+ m->mothurOut("/******************************************/"); m->mothurOutEndLine();
+ m->mothurOut("Running command: sffinfo(" + inputString + ")"); m->mothurOutEndLine();
+ m->mothurCalling = true;
+
+ Command* sffCommand = new SffInfoCommand(inputString);
+ sffCommand->execute();
+
+ if (m->control_pressed){ break; }
+
+ map<string, vector<string> > filenames = sffCommand->getOutputFiles();
+
+ delete sffCommand;
+ m->mothurCalling = false;
+ m->mothurOutEndLine();
+
+ //run summary.seqs on the fasta file
+ string fastaFile = "";
+ map<string, vector<string> >::iterator it = filenames.find("fasta");
+ if (it != filenames.end()) { if ((it->second).size() != 0) { fastaFile = (it->second)[0]; } }
+ else { m->mothurOut("[ERROR]: sffinfo did not create a fasta file, quitting.\n"); m->control_pressed = true; break; }
+
+ inputString = "fasta=" + fastaFile + ", processors=1";
+ m->mothurOutEndLine();
+ m->mothurOut("Running command: summary.seqs(" + inputString + ")"); m->mothurOutEndLine();
+ m->mothurCalling = true;
+
+ Command* summarySeqsCommand = new SeqSummaryCommand(inputString);
+ summarySeqsCommand->execute();
+
+ if (m->control_pressed){ break; }
+
+ map<string, vector<string> > temp = summarySeqsCommand->getOutputFiles();
+ mergeOutputFileList(filenames, temp);
+
+ delete summarySeqsCommand;
+ m->mothurCalling = false;
+
+ m->mothurOutEndLine();
+
+ //run trim.flows on the fasta file
+ string flowFile = "";
+ it = filenames.find("flow");
+ if (it != filenames.end()) { if ((it->second).size() != 0) { flowFile = (it->second)[0]; } }
+ else { m->mothurOut("[ERROR]: sffinfo did not create a flow file, quitting.\n"); m->control_pressed = true; break; }
+
+ inputString = "flow=" + flowFile;
+ if (oligos != "") { inputString += ", oligos=" + oligos; }
+ inputString += ", maxhomop=" + toString(maxHomoP) + ", maxflows=" + toString(maxFlows) + ", minflows=" + toString(minFlows);
+ inputString += ", pdiffs=" + toString(pdiffs) + ", bdiffs=" + toString(bdiffs) + ", ldiffs=" + toString(ldiffs) + ", sdiffs=" + toString(sdiffs);
+ inputString += ", tdiffs=" + toString(tdiffs) + ", signal=" + toString(signal) + ", noise=" + toString(noise) + ", order=" + flowOrder + ", processors=1";
+
+ m->mothurOutEndLine();
+ m->mothurOut("Running command: trim.flows(" + inputString + ")"); m->mothurOutEndLine();
+ m->mothurCalling = true;
+
+ Command* trimFlowCommand = new TrimFlowsCommand(inputString);
+ trimFlowCommand->execute();
+
+ if (m->control_pressed){ break; }
+
+ temp = trimFlowCommand->getOutputFiles();
+ mergeOutputFileList(filenames, temp);
+
+ delete trimFlowCommand;
+ m->mothurCalling = false;
+
+
+ string fileFileName = "";
+ flowFile = "";
+ if (oligos != "") {
+ it = temp.find("file");
+ if (it != temp.end()) { if ((it->second).size() != 0) { fileFileName = (it->second)[0]; } }
+ else { m->mothurOut("[ERROR]: trim.flows did not create a file file, quitting.\n"); m->control_pressed = true; break; }
+ }else {
+ vector<string> flowFiles;
+ it = temp.find("flow");
+ if (it != temp.end()) { if ((it->second).size() != 0) { flowFiles = (it->second); } }
+ else { m->mothurOut("[ERROR]: trim.flows did not create a flow file, quitting.\n"); m->control_pressed = true; break; }
+
+ for (int i = 0; i < flowFiles.size(); i++) {
+ string end = flowFiles[i].substr(flowFiles[i].length()-9);
+ if (end == "trim.flow") {
+ flowFile = flowFiles[i]; i+=flowFiles.size(); //if we found the trim.flow file stop looking
+ }
+ }
+ }
+
+ if ((fileFileName == "") && (flowFile == "")) { m->mothurOut("[ERROR]: trim.flows did not create a file file or a trim.flow file, quitting.\n"); m->control_pressed = true; break; }
+
+ if (fileFileName != "") { inputString = "file=" + fileFileName; }
+ else { inputString = "flow=" + flowFile; }
+
+ inputString += ", lookup=" + lookupFileName + ", cutoff=" + toString(cutoff); + ", maxiters=" + toString(maxIters);
+ if (large) { inputString += ", large=" + toString(largeSize); }
+ inputString += ", sigma=" +toString(sigma);
+ inputString += ", mindelta=" + toString(minDelta);
+ inputString += ", order=" + flowOrder + ", processors=1";
+
+ //run shhh.flows
+ m->mothurOutEndLine();
+ m->mothurOut("Running command: shhh.flows(" + inputString + ")"); m->mothurOutEndLine();
+ m->mothurCalling = true;
+
+ Command* shhhFlowCommand = new ShhherCommand(inputString);
+ shhhFlowCommand->execute();
+
+ if (m->control_pressed){ break; }
+
+ temp = shhhFlowCommand->getOutputFiles();
+ mergeOutputFileList(filenames, temp);
+
+ delete shhhFlowCommand;
+ m->mothurCalling = false;
+
+ vector<string> fastaFiles;
+ vector<string> nameFiles;
+ it = temp.find("fasta");
+ if (it != temp.end()) { if ((it->second).size() != 0) { fastaFiles = (it->second); } }
+ else { m->mothurOut("[ERROR]: shhh.flows did not create a fasta file, quitting.\n"); m->control_pressed = true; break; }
+
+ it = temp.find("name");
+ if (it != temp.end()) { if ((it->second).size() != 0) { nameFiles = (it->second); } }
+ else { m->mothurOut("[ERROR]: shhh.flows did not create a name file, quitting.\n"); m->control_pressed = true; break; }
+
+ //find fasta and name files with the shortest name. This is because if there is a composite name it will be the shortest.
+ fastaFile = fastaFiles[0];
+ for (int i = 1; i < fastaFiles.size(); i++) { if (fastaFiles[i].length() < fastaFile.length()) { fastaFile = fastaFiles[i]; } }
+ string nameFile = nameFiles[0];
+ for (int i = 1; i < nameFiles.size(); i++) { if (nameFiles[i].length() < nameFile.length()) { nameFile = nameFiles[i]; } }
+
+ inputString = "fasta=" + fastaFile + ", name=" + nameFile;
+ if (oligos != "") { inputString += ", oligos=" + oligos; }
+ if (allFiles) { inputString += ", allfiles=t"; }
+ else { inputString += ", allfiles=f"; }
+ if (flip) { inputString += ", flip=t"; }
+ else { inputString += ", flip=f"; }
+ if (keepforward) { inputString += ", keepforward=t"; }
+ else { inputString += ", keepforward=f"; }
+
+
+ inputString += ", pdiffs=" + toString(pdiffs) + ", bdiffs=" + toString(bdiffs) + ", ldiffs=" + toString(ldiffs) + ", sdiffs=" + toString(sdiffs);
+ inputString += ", tdiffs=" + toString(tdiffs) + ", maxambig=" + toString(maxAmbig) + ", minlength=" + toString(minLength) + ", maxlength=" + toString(maxLength);
+ if (keepFirst != 0) { inputString += ", keepfirst=" + toString(keepFirst); }
+ if (removeLast != 0) { inputString += ", removelast=" + toString(removeLast); }
+ inputString += ", processors=1";
+
+ //run trim.seqs
+ m->mothurOutEndLine();
+ m->mothurOut("Running command: trim.seqs(" + inputString + ")"); m->mothurOutEndLine();
+ m->mothurCalling = true;
+
+ Command* trimseqsCommand = new TrimSeqsCommand(inputString);
+ trimseqsCommand->execute();
+
+ if (m->control_pressed){ break; }
+
+ temp = trimseqsCommand->getOutputFiles();
+ mergeOutputFileList(filenames, temp);
+
+ delete trimseqsCommand;
+ m->mothurCalling = false;
+
+ it = temp.find("fasta");
+ if (it != temp.end()) { if ((it->second).size() != 0) { fastaFiles = (it->second); } }
+ else { m->mothurOut("[ERROR]: trim.seqs did not create a fasta file, quitting.\n"); m->control_pressed = true; break; }
+
+ for (int i = 0; i < fastaFiles.size(); i++) {
+ string end = fastaFiles[i].substr(fastaFiles[i].length()-10);
+ if (end == "trim.fasta") {
+ fastaFile = fastaFiles[i]; i+=fastaFiles.size(); //if we found the trim.fasta file stop looking
+ }
+ }
+
+ it = temp.find("name");
+ if (it != temp.end()) { if ((it->second).size() != 0) { nameFiles = (it->second); } }
+ else { m->mothurOut("[ERROR]: trim.seqs did not create a name file, quitting.\n"); m->control_pressed = true; break; }
+
+ for (int i = 0; i < nameFiles.size(); i++) {
+ string end = nameFiles[i].substr(nameFiles[i].length()-10);
+ if (end == "trim.names") {
+ nameFile = nameFiles[i]; i+=nameFiles.size(); //if we found the trim.names file stop looking
+ }
+ }
+
+ vector<string> groupFiles;
+ string groupFile = "";
+ if (makeGroup) {
+ it = temp.find("group");
+ if (it != temp.end()) { if ((it->second).size() != 0) { groupFiles = (it->second); } }
+
+ //find group file with the shortest name. This is because if there is a composite group file it will be the shortest.
+ groupFile = groupFiles[0];
+ for (int i = 1; i < groupFiles.size(); i++) { if (groupFiles[i].length() < groupFile.length()) { groupFile = groupFiles[i]; } }
+ }
+
+ inputString = "fasta=" + fastaFile + ", processors=1, name=" + nameFile;
+ m->mothurOutEndLine();
+ m->mothurOut("Running command: summary.seqs(" + inputString + ")"); m->mothurOutEndLine();
+ m->mothurCalling = true;
+
+ summarySeqsCommand = new SeqSummaryCommand(inputString);
+ summarySeqsCommand->execute();
+
+ if (m->control_pressed){ break; }
+
+ temp = summarySeqsCommand->getOutputFiles();
+ mergeOutputFileList(filenames, temp);
+
+ delete summarySeqsCommand;
+ m->mothurCalling = false;
+
+ m->mothurOutEndLine();
+ m->mothurOut("/******************************************/"); m->mothurOutEndLine();
+
+ if (append) {
+ m->appendFiles(fastaFile, fasta);
+ m->appendFiles(nameFile, name);
+ if (makeGroup) { m->appendFiles(groupFile, group); }
+ }
+ count++;
+
+ for (it = filenames.begin(); it != filenames.end(); it++) {
+ for (int i = 0; i < (it->second).size(); i++) {
+ outputNames.push_back((it->second)[i]); outputTypes[it->first].push_back((it->second)[i]);
+ }
+ }
+ }
+
+ return count;
+ }
+ catch(exception& e) {
+ m->errorOut(e, "SffMultipleCommand", "driver");
+ exit(1);
+ }
+}
+//**********************************************************************************************************************
+int SffMultipleCommand::mergeOutputFileList(map<string, vector<string> >& files, map<string, vector<string> >& temp){
+ try {
+ map<string, vector<string> >::iterator it;
+ for (it = temp.begin(); it != temp.end(); it++) {
+ map<string, vector<string> >::iterator it2 = files.find(it->first);
+ if (it2 == files.end()) { //we do not already have this type so just add it
+ files[it->first] = it->second;
+ }else { //merge them
+ for (int i = 0; i < (it->second).size(); i++) {
+ files[it->first].push_back((it->second)[i]);
+ }
+ }
+ }
+
+ return 0;
+ }
+ catch(exception& e) {
+ m->errorOut(e, "SffMultipleCommand", "mergeOutputFileList");
+ exit(1);
+ }
+}
+//**********************************************************************************************************************
+int SffMultipleCommand::createProcesses(vector<string> sffFiles, vector<string> oligosFiles, string fasta, string name, string group){
+ try {
+ vector<int> processIDS;
+ int process = 1;
+ int num = 0;
+
+ //divide the groups between the processors
+ vector<linePair> lines;
+ vector<int> numFilesToComplete;
+ int numFilesPerProcessor = sffFiles.size() / processors;
+ for (int i = 0; i < processors; i++) {
+ int startIndex = i * numFilesPerProcessor;
+ int endIndex = (i+1) * numFilesPerProcessor;
+ if(i == (processors - 1)){ endIndex = sffFiles.size(); }
+ lines.push_back(linePair(startIndex, endIndex));
+ numFilesToComplete.push_back((endIndex-startIndex));
+ }
+
+#if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix)
+
+ //loop through and create all the processes you want
+ while (process != processors) {
+ int pid = fork();
+
+ if (pid > 0) {
+ processIDS.push_back(pid); //create map from line number to pid so you can append files in correct order later
+ process++;
+ }else if (pid == 0){
+ num = driver(sffFiles, oligosFiles, lines[process].start, lines[process].end, fasta + toString(getpid()) + ".temp", name + toString(getpid()) + ".temp", group + toString(getpid()) + ".temp");
+
+ //pass numSeqs to parent
+ ofstream out;
+ string tempFile = toString(getpid()) + ".num.temp";
+ m->openOutputFile(tempFile, out);
+ out << num << '\t' << outputNames.size() << endl;
+ for (int i = 0; i < outputNames.size(); i++) { out << outputNames[i] << endl; }
+ out.close();
+
+ exit(0);
+ }else {
+ m->mothurOut("[ERROR]: unable to spawn the necessary processes."); m->mothurOutEndLine();
+ for (int i = 0; i < processIDS.size(); i++) { kill (processIDS[i], SIGINT); }
+ exit(0);
+ }
+ }
+
+ //do my part
+ num = driver(sffFiles, oligosFiles, lines[0].start, lines[0].end, fasta, name, group);
+
+ //force parent to wait until all the processes are done
+ for (int i=0;i<processIDS.size();i++) {
+ int temp = processIDS[i];
+ wait(&temp);
+ }
+
+ for (int i=0;i<processIDS.size();i++) {
+ ifstream in;
+ string tempFile = toString(processIDS[i]) + ".num.temp";
+ m->openInputFile(tempFile, in);
+ if (!in.eof()) {
+ int tempNum = 0; int outputNamesSize = 0;
+ in >> tempNum >> outputNamesSize; m->gobble(in);
+ for (int j = 0; j < outputNamesSize; j++) {
+ string tempName;
+ in >> tempName; m->gobble(in);
+ outputNames.push_back(tempName);
+ }
+ if (tempNum != numFilesToComplete[i+1]) {
+ m->mothurOut("[ERROR]: main process expected " + toString(processIDS[i]) + " to complete " + toString(numFilesToComplete[i+1]) + " files, and it only reported completing " + toString(tempNum) + ". This will cause file mismatches. The flow files may be too large to process with multiple processors. \n");
+ }
+ }
+ in.close(); m->mothurRemove(tempFile);
+
+ if (append) {
+ m->appendFiles(fasta+toString(processIDS[i])+".temp", fasta); m->mothurRemove(fasta+toString(processIDS[i])+".temp");
+ m->appendFiles(name+toString(processIDS[i])+".temp", name); m->mothurRemove(name+toString(processIDS[i])+".temp");
+ if (makeGroup) { m->appendFiles(group+toString(processIDS[i])+".temp", group); m->mothurRemove(group+toString(processIDS[i])+".temp"); }
+ }
+ }
+#endif
+ return 0;
+
+ }
+ catch(exception& e) {
+ m->errorOut(e, "ShhherCommand", "createProcesses");
+ exit(1);
+ }
+}
+//**********************************************************************************************************************