X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=preclustercommand.cpp;h=10ed82571150e9d61e26f62b12508a9e5f967701;hb=a89c6295ae9a35fcaaab7fa50dcb68360dd543b0;hp=c0679155de1514a9ebb6845368b899140718b8d3;hpb=e0c50fc42e953dbe258feb9bc1d1d7dc10fafff0;p=mothur.git diff --git a/preclustercommand.cpp b/preclustercommand.cpp index c067915..10ed825 100644 --- a/preclustercommand.cpp +++ b/preclustercommand.cpp @@ -44,7 +44,7 @@ string PreClusterCommand::getHelpString(){ helpString += "The group parameter allows you to provide a group file so you can cluster by group. \n"; helpString += "The count parameter allows you to provide a count file so you can cluster by group. \n"; helpString += "The diffs parameter allows you to specify maximum number of mismatched bases allowed between sequences in a grouping. The default is 1.\n"; - helpString += "The topdown parameter allows you to specify whether to cluster from largest abundance to smallest or smallest to largest. Default=T, meanging largest to smallest.\n"; + helpString += "The topdown parameter allows you to specify whether to cluster from largest abundance to smallest or smallest to largest. Default=T, meaning largest to smallest.\n"; helpString += "The pre.cluster command should be in the following format: \n"; helpString += "pre.cluster(fasta=yourFastaFile, names=yourNamesFile, diffs=yourMaxDiffs) \n"; helpString += "Example pre.cluster(fasta=amazon.fasta, diffs=2).\n"; @@ -192,7 +192,7 @@ PreClusterCommand::PreClusterCommand(string option) { else if (countfile == "not open") { abort = true; countfile = ""; } else { m->setCountTableFile(countfile); - ct.readTable(countfile); + ct.readTable(countfile, true, false); if (ct.hasGroupInfo()) { bygroup = true; } else { bygroup = false; } } @@ -300,6 +300,7 @@ int PreClusterCommand::execute(){ m->mothurOut("It took " + toString(time(NULL) - start) + " secs to run pre.cluster."); m->mothurOutEndLine(); }else { + if (processors != 1) { m->mothurOut("When using running without group information mothur can only use 1 processor, continuing."); m->mothurOutEndLine(); processors = 1; } if (namefile != "") { readNameFile(); } //reads fasta file and return number of seqs @@ -362,34 +363,37 @@ int PreClusterCommand::createProcessesGroups(string newFName, string newNName, s vector processIDS; int process = 1; int num = 0; - + bool recalc = false; + //sanity check if (groups.size() < processors) { processors = groups.size(); } //divide the groups between the processors vector lines; - int numGroupsPerProcessor = groups.size() / processors; - for (int i = 0; i < processors; i++) { - int startIndex = i * numGroupsPerProcessor; - int endIndex = (i+1) * numGroupsPerProcessor; - if(i == (processors - 1)){ endIndex = groups.size(); } - lines.push_back(linePair(startIndex, endIndex)); - } + int remainingPairs = groups.size(); + int startIndex = 0; + for (int remainingProcessors = processors; remainingProcessors > 0; remainingProcessors--) { + int numPairs = remainingPairs; //case for last processor + if (remainingProcessors != 1) { numPairs = ceil(remainingPairs / remainingProcessors); } + lines.push_back(linePair(startIndex, (startIndex+numPairs))); //startIndex, endIndex + startIndex = startIndex + numPairs; + remainingPairs = remainingPairs - numPairs; + } #if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix) //loop through and create all the processes you want while (process != processors) { - int pid = fork(); + pid_t pid = fork(); if (pid > 0) { processIDS.push_back(pid); //create map from line number to pid so you can append files in correct order later process++; }else if (pid == 0){ outputNames.clear(); - num = driverGroups(newFName + toString(getpid()) + ".temp", newNName + toString(getpid()) + ".temp", newMFile, lines[process].start, lines[process].end, groups); + num = driverGroups(newFName + m->mothurGetpid(process) + ".temp", newNName + m->mothurGetpid(process) + ".temp", newMFile, lines[process].start, lines[process].end, groups); - string tempFile = toString(getpid()) + ".outputNames.temp"; + string tempFile = m->mothurGetpid(process) + ".outputNames.temp"; ofstream outTemp; m->openOutputFile(tempFile, outTemp); @@ -398,12 +402,57 @@ int PreClusterCommand::createProcessesGroups(string newFName, string newNName, s outTemp.close(); exit(0); - }else { - m->mothurOut("[ERROR]: unable to spawn the necessary processes."); m->mothurOutEndLine(); - for (int i = 0; i < processIDS.size(); i++) { kill (processIDS[i], SIGINT); } - exit(0); + }else { + m->mothurOut("[ERROR]: unable to spawn the number of processes you requested, reducing number to " + toString(process) + "\n"); processors = process; + for (int i = 0; i < processIDS.size(); i++) { kill (processIDS[i], SIGINT); } + recalc = true; + break; } } + + if (recalc) { + lines.clear(); + num = 0; + processIDS.resize(0); + process = 1; + + int remainingPairs = groups.size(); + int startIndex = 0; + for (int remainingProcessors = processors; remainingProcessors > 0; remainingProcessors--) { + int numPairs = remainingPairs; //case for last processor + if (remainingProcessors != 1) { numPairs = ceil(remainingPairs / remainingProcessors); } + lines.push_back(linePair(startIndex, (startIndex+numPairs))); //startIndex, endIndex + startIndex = startIndex + numPairs; + remainingPairs = remainingPairs - numPairs; + } + + while (process != processors) { + pid_t pid = fork(); + + if (pid > 0) { + processIDS.push_back(pid); //create map from line number to pid so you can append files in correct order later + process++; + }else if (pid == 0){ + outputNames.clear(); + num = driverGroups(newFName + m->mothurGetpid(process) + ".temp", newNName + m->mothurGetpid(process) + ".temp", newMFile, lines[process].start, lines[process].end, groups); + + string tempFile = m->mothurGetpid(process) + ".outputNames.temp"; + ofstream outTemp; + m->openOutputFile(tempFile, outTemp); + + outTemp << outputNames.size(); + for (int i = 0; i < outputNames.size(); i++) { outTemp << outputNames[i] << endl; } + outTemp.close(); + + exit(0); + }else { + m->mothurOut("[ERROR]: unable to spawn the necessary processes."); m->mothurOutEndLine(); + for (int i = 0; i < processIDS.size(); i++) { kill (processIDS[i], SIGINT); } + exit(0); + } + } + } + //do my part num = driverGroups(newFName, newNName, newMFile, lines[0].start, lines[0].end, groups); @@ -597,7 +646,7 @@ int PreClusterCommand::process(string newMapFile){ out << "ideal_seq_" << (i+1) << '\t' << alignSeqs[i].numIdentical << endl << chunk << endl;; }//end if active i - if(i % 100 == 0) { m->mothurOut(toString(i) + "\t" + toString(numSeqs - count) + "\t" + toString(count)); m->mothurOutEndLine(); } + if(i % 100 == 0) { m->mothurOutJustToScreen(toString(i) + "\t" + toString(numSeqs - count) + "\t" + toString(count)+"\n"); } } }else { map mapFile; @@ -632,7 +681,7 @@ int PreClusterCommand::process(string newMapFile){ }//end abundance check }//end for loop j - if(i % 100 == 0) { m->mothurOut(toString(i) + "\t" + toString(numSeqs - count) + "\t" + toString(count)); m->mothurOutEndLine(); } + if(i % 100 == 0) { m->mothurOutJustToScreen(toString(i) + "\t" + toString(numSeqs - count) + "\t" + toString(count)+"\n"); } } for (int i = 0; i < numSeqs; i++) { @@ -903,15 +952,9 @@ void PreClusterCommand::readNameFile(){ while (!in.eof()) { in >> firstCol >> secondCol; m->gobble(in); - for (int i = 0; i < firstCol.length(); i++) { - if (firstCol[i] == ':') { firstCol[i] = '_'; m->changedSeqNames = true; } - } - - int size = 1; - for (int i = 0; i < secondCol.length(); i++) { - if (secondCol[i] == ':') { secondCol[i] = '_'; m->changedSeqNames = true; } - else if(secondCol[i] == ','){ size++; } - } + m->checkName(firstCol); + m->checkName(secondCol); + int size = m->getNumNames(secondCol); names[firstCol] = secondCol; sizes[firstCol] = size;