]> git.donarmstrong.com Git - mothur.git/blobdiff - shhhercommand.cpp
fixed bug in phylo.diversity rooting. added filename patterns and create filename...
[mothur.git] / shhhercommand.cpp
index be410d9d9f4ecec2fc885a239c9ba57b5f409e89..a859d145b49e261bace85fb973d3aa9466bc1123 100644 (file)
 //**********************************************************************************************************************
 vector<string> ShhherCommand::setParameters(){ 
        try {
-               CommandParameter pflow("flow", "InputTypes", "", "", "none", "fileflow", "none",false,false); parameters.push_back(pflow);
-               CommandParameter pfile("file", "InputTypes", "", "", "none", "fileflow", "none",false,false); parameters.push_back(pfile);
-               CommandParameter plookup("lookup", "InputTypes", "", "", "none", "none", "none",false,false); parameters.push_back(plookup);
-               CommandParameter pcutoff("cutoff", "Number", "", "0.01", "", "", "",false,false); parameters.push_back(pcutoff);
-               CommandParameter pprocessors("processors", "Number", "", "1", "", "", "",false,false); parameters.push_back(pprocessors);
-               CommandParameter pmaxiter("maxiter", "Number", "", "1000", "", "", "",false,false); parameters.push_back(pmaxiter);
-        CommandParameter plarge("large", "Number", "", "-1", "", "", "",false,false); parameters.push_back(plarge);
-               CommandParameter psigma("sigma", "Number", "", "60", "", "", "",false,false); parameters.push_back(psigma);
-               CommandParameter pmindelta("mindelta", "Number", "", "0.000001", "", "", "",false,false); parameters.push_back(pmindelta);
-               CommandParameter porder("order", "String", "", "", "", "", "",false,false); parameters.push_back(porder);
-               CommandParameter pinputdir("inputdir", "String", "", "", "", "", "",false,false); parameters.push_back(pinputdir);
-               CommandParameter poutputdir("outputdir", "String", "", "", "", "", "",false,false); parameters.push_back(poutputdir);
+               CommandParameter pflow("flow", "InputTypes", "", "", "none", "fileflow", "none","fasta-name-group-counts-qfile",false,false,true); parameters.push_back(pflow);
+               CommandParameter pfile("file", "InputTypes", "", "", "none", "fileflow", "none","fasta-name-group-counts-qfile",false,false,true); parameters.push_back(pfile);
+               CommandParameter plookup("lookup", "InputTypes", "", "", "none", "none", "none","",false,false,true); parameters.push_back(plookup);
+               CommandParameter pcutoff("cutoff", "Number", "", "0.01", "", "", "","",false,false); parameters.push_back(pcutoff);
+               CommandParameter pprocessors("processors", "Number", "", "1", "", "", "","",false,false,true); parameters.push_back(pprocessors);
+               CommandParameter pmaxiter("maxiter", "Number", "", "1000", "", "", "","",false,false); parameters.push_back(pmaxiter);
+        CommandParameter plarge("large", "Number", "", "-1", "", "", "","",false,false); parameters.push_back(plarge);
+               CommandParameter psigma("sigma", "Number", "", "60", "", "", "","",false,false); parameters.push_back(psigma);
+               CommandParameter pmindelta("mindelta", "Number", "", "0.000001", "", "", "","",false,false); parameters.push_back(pmindelta);
+               CommandParameter porder("order", "String", "", "", "", "", "","",false,false); parameters.push_back(porder);
+               CommandParameter pinputdir("inputdir", "String", "", "", "", "", "","",false,false); parameters.push_back(pinputdir);
+               CommandParameter poutputdir("outputdir", "String", "", "", "", "", "","",false,false); parameters.push_back(poutputdir);
                
                vector<string> myArray;
                for (int i = 0; i < parameters.size(); i++) {   myArray.push_back(parameters[i].name);          }
@@ -47,6 +47,25 @@ string ShhherCommand::getHelpString(){
        }
 }
 //**********************************************************************************************************************
+string ShhherCommand::getOutputPattern(string type) {
+    try {
+        string pattern = "";
+        
+        if (type == "fasta")            {   pattern = "[filename],shhh.fasta";   }
+        else if (type == "name")    {   pattern = "[filename],shhh.names";   }
+        else if (type == "group")        {   pattern = "[filename],shhh.groups";   }
+        else if (type == "counts")        {   pattern = "[filename],shhh.counts";   }
+        else if (type == "qfile")        {   pattern = "[filename],shhh.qual";   }
+        else { m->mothurOut("[ERROR]: No definition for type " + type + " output pattern.\n"); m->control_pressed = true;  }
+        
+        return pattern;
+    }
+    catch(exception& e) {
+        m->errorOut(e, "ShhherCommand", "getOutputPattern");
+        exit(1);
+    }
+}
+//**********************************************************************************************************************
 
 ShhherCommand::ShhherCommand(){        
        try {
@@ -54,8 +73,12 @@ ShhherCommand::ShhherCommand(){
                setParameters();
                
                //initialize outputTypes
-//             vector<string> tempOutNames;
-//             outputTypes["pn.dist"] = tempOutNames;
+               vector<string> tempOutNames;
+               outputTypes["fasta"] = tempOutNames;
+        outputTypes["name"] = tempOutNames;
+        outputTypes["group"] = tempOutNames;
+        outputTypes["counts"] = tempOutNames;
+        outputTypes["qfile"] = tempOutNames;
 
        }
        catch(exception& e) {
@@ -96,9 +119,13 @@ ShhherCommand::ShhherCommand(string option) {
                        }
                        
                        //initialize outputTypes
-                       vector<string> tempOutNames;
-//                     outputTypes["pn.dist"] = tempOutNames;
-                       //                      outputTypes["fasta"] = tempOutNames;
+            vector<string> tempOutNames;
+            outputTypes["fasta"] = tempOutNames;
+            outputTypes["name"] = tempOutNames;
+            outputTypes["group"] = tempOutNames;
+            outputTypes["counts"] = tempOutNames;
+            outputTypes["qfile"] = tempOutNames;
+
                        
                        //if the user changes the input directory command factory will send this info to us in the output parameter 
                        string inputDir = validParameter.validFile(parameters, "inputdir", false);              
@@ -744,8 +771,8 @@ int ShhherCommand::execute(){
 
                
                if(compositeFASTAFileName != ""){
-                       outputNames.push_back(compositeFASTAFileName);
-                       outputNames.push_back(compositeNamesFileName);
+                       outputNames.push_back(compositeFASTAFileName); outputTypes["fasta"].push_back(compositeFASTAFileName);
+                       outputNames.push_back(compositeNamesFileName); outputTypes["name"].push_back(compositeNamesFileName); 
                }
 
                m->mothurOutEndLine();
@@ -769,8 +796,9 @@ string ShhherCommand::createNamesFile(){
                for(int i=0;i<numSeqs;i++){
                        duplicateNames[mapSeqToUnique[i]] += seqNameVector[i] + ',';
                }
-               
-               string nameFileName = flowFileName.substr(0,flowFileName.find_last_of('.')) + ".shhh.names";
+               map<string, string> variables; 
+               variables["[filename]"] = outputDir + m->getRootName(m->getSimpleName(flowFileName));
+               string nameFileName = getOutputFileName("name",variables);
                
                ofstream nameFile;
                m->openOutputFile(nameFileName, nameFile);
@@ -938,6 +966,8 @@ void ShhherCommand::initPyroCluster(){
     try{
         if (numOTUs < processors) { processors = 1; }
         
+        if (m->debug) { m->mothurOut("[DEBUG]: numSeqs = " + toString(numSeqs) + " numOTUS = " + toString(numOTUs) + " about to alloc a dist vector with size = " + toString((numSeqs * numOTUs)) + ".\n"); }
+        
         dist.assign(numSeqs * numOTUs, 0);
         change.assign(numOTUs, 1);
         centroids.assign(numOTUs, -1);
@@ -947,6 +977,8 @@ void ShhherCommand::initPyroCluster(){
         nSeqsBreaks.assign(processors+1, 0);
         nOTUsBreaks.assign(processors+1, 0);
         
+        if (m->debug) { m->mothurOut("[DEBUG]: made it through the memory allocation.\n"); }
+        
         nSeqsBreaks[0] = 0;
         for(int i=0;i<processors;i++){
             nSeqsBreaks[i+1] = nSeqsBreaks[i] + (int)((double) numSeqs / (double) processors);
@@ -1003,7 +1035,12 @@ void ShhherCommand::getFlowData(){
         
         float intensity;
         
-        flowFile >> numFlowCells;
+        string numFlowTest;
+        flowFile >> numFlowTest;
+        
+        if (!m->isContainingOnlyDigits(numFlowTest)) { m->mothurOut("[ERROR]: expected a number and got " + numFlowTest + ", quitting. Did you use the flow parameter instead of the file parameter?"); m->mothurOutEndLine(); exit(1); }
+        else { convert(numFlowTest, numFlowCells); }
+        
         int index = 0;//pcluster
         while(!flowFile.eof()){
             
@@ -1340,17 +1377,17 @@ string ShhherCommand::cluster(string distFileName, string namesFileName){
     try {
         
         ReadMatrix* read = new ReadColumnMatrix(distFileName);         
-        read->setCutoff(cutoff);
-        
-        NameAssignment* clusterNameMap = new NameAssignment(namesFileName);
-        clusterNameMap->readMap();
-        read->read(clusterNameMap);
-        
-        ListVector* list = read->getListVector();
-        SparseMatrix* matrix = read->getMatrix();
+               read->setCutoff(cutoff);
+               
+               NameAssignment* clusterNameMap = new NameAssignment(namesFileName);
+               clusterNameMap->readMap();
+               read->read(clusterNameMap);
         
-        delete read; 
-        delete clusterNameMap; 
+               ListVector* list = read->getListVector();
+               SparseDistanceMatrix* matrix = read->getDMatrix();
+               
+               delete read; 
+               delete clusterNameMap; 
         
         RAbundVector* rabund = new RAbundVector(list->getRAbundVector());
         
@@ -1610,7 +1647,9 @@ void ShhherCommand::writeQualities(vector<int> otuCounts){
     try {
         string thisOutputDir = outputDir;
         if (outputDir == "") {  thisOutputDir += m->hasPath(flowFileName);  }
-        string qualityFileName = thisOutputDir + m->getRootName(m->getSimpleName(flowFileName)) + "shhh.qual";
+        map<string, string> variables; 
+               variables["[filename]"] = outputDir + m->getRootName(m->getSimpleName(flowFileName));
+        string qualityFileName = getOutputFileName("qfile",variables);
         
         ofstream qualityFile;
         m->openOutputFile(qualityFileName, qualityFile);
@@ -1702,7 +1741,7 @@ void ShhherCommand::writeQualities(vector<int> otuCounts){
             }
         }
         qualityFile.close();
-        outputNames.push_back(qualityFileName);
+        outputNames.push_back(qualityFileName); outputTypes["qfile"].push_back(qualityFileName);
         
     }
     catch(exception& e) {
@@ -1717,7 +1756,9 @@ void ShhherCommand::writeSequences(vector<int> otuCounts){
     try {
         string thisOutputDir = outputDir;
         if (outputDir == "") {  thisOutputDir += m->hasPath(flowFileName);  }
-        string fastaFileName = thisOutputDir + m->getRootName(m->getSimpleName(flowFileName)) + "shhh.fasta";
+        map<string, string> variables; 
+               variables["[filename]"] = thisOutputDir + m->getRootName(m->getSimpleName(flowFileName));
+        string fastaFileName = getOutputFileName("fasta",variables);
         ofstream fastaFile;
         m->openOutputFile(fastaFileName, fastaFile);
         
@@ -1747,7 +1788,7 @@ void ShhherCommand::writeSequences(vector<int> otuCounts){
         }
         fastaFile.close();
         
-        outputNames.push_back(fastaFileName);
+        outputNames.push_back(fastaFileName); outputTypes["fasta"].push_back(fastaFileName);
         
         if(compositeFASTAFileName != ""){
             m->appendFiles(fastaFileName, compositeFASTAFileName);
@@ -1765,7 +1806,9 @@ void ShhherCommand::writeNames(vector<int> otuCounts){
     try {
         string thisOutputDir = outputDir;
         if (outputDir == "") {  thisOutputDir += m->hasPath(flowFileName);  }
-        string nameFileName = thisOutputDir + m->getRootName(m->getSimpleName(flowFileName)) + "shhh.names";
+        map<string, string> variables; 
+               variables["[filename]"] = thisOutputDir + m->getRootName(m->getSimpleName(flowFileName));
+        string nameFileName = getOutputFileName("name",variables);
         ofstream nameFile;
         m->openOutputFile(nameFileName, nameFile);
         
@@ -1784,7 +1827,7 @@ void ShhherCommand::writeNames(vector<int> otuCounts){
             }
         }
         nameFile.close();
-        outputNames.push_back(nameFileName);
+        outputNames.push_back(nameFileName); outputTypes["name"].push_back(nameFileName);
         
         
         if(compositeNamesFileName != ""){
@@ -1803,17 +1846,22 @@ void ShhherCommand::writeGroups(){
     try {
         string thisOutputDir = outputDir;
         if (outputDir == "") {  thisOutputDir += m->hasPath(flowFileName);  }
-        string fileRoot = thisOutputDir + m->getRootName(m->getSimpleName(flowFileName));
-        string groupFileName = fileRoot + "shhh.groups";
+        string fileRoot = m->getRootName(m->getSimpleName(flowFileName));
+        int pos = fileRoot.find_first_of('.');
+        string fileGroup = fileRoot;
+        if (pos != string::npos) {  fileGroup = fileRoot.substr(pos+1, (fileRoot.length()-1-(pos+1)));  }
+        map<string, string> variables; 
+               variables["[filename]"] = thisOutputDir + fileRoot;
+        string groupFileName = getOutputFileName("group",variables);
         ofstream groupFile;
         m->openOutputFile(groupFileName, groupFile);
         
         for(int i=0;i<numSeqs;i++){
             if (m->control_pressed) { break; }
-            groupFile << seqNameVector[i] << '\t' << fileRoot << endl;
+            groupFile << seqNameVector[i] << '\t' << fileGroup << endl;
         }
         groupFile.close();
-        outputNames.push_back(groupFileName);
+        outputNames.push_back(groupFileName); outputTypes["group"].push_back(groupFileName);
         
     }
     catch(exception& e) {
@@ -1828,7 +1876,9 @@ void ShhherCommand::writeClusters(vector<int> otuCounts){
     try {
         string thisOutputDir = outputDir;
         if (outputDir == "") {  thisOutputDir += m->hasPath(flowFileName);  }
-        string otuCountsFileName = thisOutputDir + m->getRootName(m->getSimpleName(flowFileName)) + "shhh.counts";
+        map<string, string> variables; 
+               variables["[filename]"] = thisOutputDir + m->getRootName(m->getSimpleName(flowFileName));
+        string otuCountsFileName = getOutputFileName("counts",variables);
         ofstream otuCountsFile;
         m->openOutputFile(otuCountsFileName, otuCountsFile);
         
@@ -1873,7 +1923,7 @@ void ShhherCommand::writeClusters(vector<int> otuCounts){
             }
         }
         otuCountsFile.close();
-        outputNames.push_back(otuCountsFileName);
+        outputNames.push_back(otuCountsFileName); outputTypes["counts"].push_back(otuCountsFileName);
         
     }
     catch(exception& e) {
@@ -1887,7 +1937,7 @@ void ShhherCommand::writeClusters(vector<int> otuCounts){
 
 int ShhherCommand::execute(){
        try {
-               if (abort == true) { return 0; }
+               if (abort == true) { if (calledHelp) { return 0; }  return 2;   }
                
                getSingleLookUp();      if (m->control_pressed) { return 0; }
                getJointLookUp();       if (m->control_pressed) { return 0; }
@@ -1904,8 +1954,8 @@ int ShhherCommand::execute(){
 #endif
         
                if(compositeFASTAFileName != ""){
-                       outputNames.push_back(compositeFASTAFileName);
-                       outputNames.push_back(compositeNamesFileName);
+                       outputNames.push_back(compositeFASTAFileName); outputTypes["fasta"].push_back(compositeFASTAFileName);
+                       outputNames.push_back(compositeNamesFileName); outputTypes["name"].push_back(compositeNamesFileName);
                }
 
                m->mothurOutEndLine();
@@ -1934,12 +1984,14 @@ int ShhherCommand::createProcesses(vector<string> filenames){
                
                //divide the groups between the processors
                vector<linePair> lines;
+        vector<int> numFilesToComplete;
                int numFilesPerProcessor = filenames.size() / processors;
                for (int i = 0; i < processors; i++) {
                        int startIndex =  i * numFilesPerProcessor;
                        int endIndex = (i+1) * numFilesPerProcessor;
                        if(i == (processors - 1)){      endIndex = filenames.size();    }
                        lines.push_back(linePair(startIndex, endIndex));
+            numFilesToComplete.push_back((endIndex-startIndex));
                }
                
         #if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix)         
@@ -1953,6 +2005,14 @@ int ShhherCommand::createProcesses(vector<string> filenames){
                                process++;
                        }else if (pid == 0){
                                num = driver(filenames, compositeFASTAFileName + toString(getpid()) + ".temp", compositeNamesFileName  + toString(getpid()) + ".temp", lines[process].start, lines[process].end);
+                
+                //pass numSeqs to parent
+                               ofstream out;
+                               string tempFile = compositeFASTAFileName + toString(getpid()) + ".num.temp";
+                               m->openOutputFile(tempFile, out);
+                               out << num << endl;
+                               out.close();
+                
                                exit(0);
                        }else { 
                                m->mothurOut("[ERROR]: unable to spawn the necessary processes."); m->mothurOutEndLine(); 
@@ -1980,7 +2040,7 @@ int ShhherCommand::createProcesses(vector<string> filenames){
                //Windows version shared memory, so be careful when passing variables through the shhhFlowsData struct. 
                //Above fork() will clone, so memory is separate, but that's not the case with windows, 
                //////////////////////////////////////////////////////////////////////////////////////////////////////
-               
+               /*
                vector<shhhFlowsData*> pDataArray; 
                DWORD   dwThreadIdArray[processors-1];
                HANDLE  hThreadArray[processors-1]; 
@@ -2011,10 +2071,22 @@ int ShhherCommand::createProcesses(vector<string> filenames){
                        CloseHandle(hThreadArray[i]);
                        delete pDataArray[i];
                }
-               
+               */
         #endif
         
         for (int i=0;i<processIDS.size();i++) { 
+            ifstream in;
+                       string tempFile =  compositeFASTAFileName + toString(processIDS[i]) + ".num.temp";
+                       m->openInputFile(tempFile, in);
+                       if (!in.eof()) { 
+                int tempNum = 0; 
+                in >> tempNum; 
+                if (tempNum != numFilesToComplete[i+1]) {
+                    m->mothurOut("[ERROR]: main process expected " + toString(processIDS[i]) + " to complete " + toString(numFilesToComplete[i+1]) + " files, and it only reported completing " + toString(tempNum) + ". This will cause file mismatches.  The flow files may be too large to process with multiple processors. \n");
+                }
+            }
+                       in.close(); m->mothurRemove(tempFile);
+            
             if (compositeFASTAFileName != "") {
                 m->appendFiles((compositeFASTAFileName + toString(processIDS[i]) + ".temp"), compositeFASTAFileName);
                 m->appendFiles((compositeNamesFileName + toString(processIDS[i]) + ".temp"), compositeNamesFileName);
@@ -2083,6 +2155,8 @@ vector<string> ShhherCommand::parseFlowFiles(string filename){
 int ShhherCommand::driver(vector<string> filenames, string thisCompositeFASTAFileName, string thisCompositeNamesFileName, int start, int end){
     try {
         
+        int numCompleted = 0;
+        
         for(int i=start;i<end;i++){
                        
                        if (m->control_pressed) { break; }
@@ -2113,6 +2187,7 @@ int ShhherCommand::driver(vector<string> filenames, string thisCompositeFASTAFil
                 vector<int> uniqueLengths;
                 int numFlowCells;
                 
+                if (m->debug) { m->mothurOut("[DEBUG]: About to read flowgrams.\n"); }
                 int numSeqs = getFlowData(flowFileName, seqNameVector, lengths, flowDataIntI, nameMap, numFlowCells);
                 
                 if (m->control_pressed) { break; }
@@ -2169,6 +2244,8 @@ int ShhherCommand::driver(vector<string> filenames, string thisCompositeFASTAFil
                 vector<int> nSeqsBreaks;
                 vector<int> nOTUsBreaks;
                 
+                if (m->debug) { m->mothurOut("[DEBUG]: numSeqs = " + toString(numSeqs) + " numOTUS = " + toString(numOTUs) + " about to alloc a dist vector with size = " + toString((numSeqs * numOTUs)) + ".\n"); }
+                
                 dist.assign(numSeqs * numOTUs, 0);
                 change.assign(numOTUs, 1);
                 centroids.assign(numOTUs, -1);
@@ -2182,6 +2259,8 @@ int ShhherCommand::driver(vector<string> filenames, string thisCompositeFASTAFil
                 nSeqsBreaks[1] = numSeqs;
                 nOTUsBreaks[1] = numOTUs;
                 
+                if (m->debug) { m->mothurOut("[DEBUG]: done allocating memory, about to denoise.\n"); }
+                
                 if (m->control_pressed) { break; }
                 
                 double maxDelta = 0;
@@ -2241,7 +2320,7 @@ int ShhherCommand::driver(vector<string> filenames, string thisCompositeFASTAFil
                 if (m->control_pressed) { break; }
                 
                 vector<int> otuCounts(numOTUs, 0);
-                for(int i=0;i<numSeqs;i++)     {       otuCounts[otuData[i]]++;        }
+                for(int j=0;j<numSeqs;j++)     {       otuCounts[otuData[j]]++;        }
                 
                 calcCentroidsDriver(numOTUs, cumNumSeqs, nSeqsPerOTU, seqIndex, change, centroids, singleTau, mapSeqToUnique, uniqueFlowgrams, flowDataIntI, lengths, numFlowCells, seqNumber);        
                 
@@ -2250,43 +2329,50 @@ int ShhherCommand::driver(vector<string> filenames, string thisCompositeFASTAFil
                 if ((large) && (g == 0)) {  flowFileName = filenames[i]; theseFlowFileNames[0] = filenames[i]; }
                 string thisOutputDir = outputDir;
                 if (outputDir == "") {  thisOutputDir = m->hasPath(flowFileName);  }
-                string qualityFileName = thisOutputDir + m->getRootName(m->getSimpleName(flowFileName)) + "shhh.qual";
-                string fastaFileName = thisOutputDir + m->getRootName(m->getSimpleName(flowFileName)) + "shhh.fasta";
-                string nameFileName = thisOutputDir + m->getRootName(m->getSimpleName(flowFileName)) + "shhh.names";
-                string otuCountsFileName = thisOutputDir + m->getRootName(m->getSimpleName(flowFileName)) + "shhh.counts";
-                string fileRoot = thisOutputDir + m->getRootName(m->getSimpleName(flowFileName));
-                string groupFileName = fileRoot + "shhh.groups";
+                map<string, string> variables; 
+                variables["[filename]"] = thisOutputDir + m->getRootName(m->getSimpleName(flowFileName));
+                string qualityFileName = getOutputFileName("qfile",variables);
+                string fastaFileName = getOutputFileName("fasta",variables);
+                string nameFileName = getOutputFileName("name",variables);
+                string otuCountsFileName = getOutputFileName("counts",variables);
+                string fileRoot = m->getRootName(m->getSimpleName(flowFileName));
+                int pos = fileRoot.find_first_of('.');
+                string fileGroup = fileRoot;
+                if (pos != string::npos) {  fileGroup = fileRoot.substr(pos+1, (fileRoot.length()-1-(pos+1)));  }
+                string groupFileName = getOutputFileName("group",variables);
 
                 
                 writeQualities(numOTUs, numFlowCells, qualityFileName, otuCounts, nSeqsPerOTU, seqNumber, singleTau, flowDataIntI, uniqueFlowgrams, cumNumSeqs, mapUniqueToSeq, seqNameVector, centroids, aaI); if (m->control_pressed) { break; }
                 writeSequences(thisCompositeFASTAFileName, numOTUs, numFlowCells, fastaFileName, otuCounts, uniqueFlowgrams, seqNameVector, aaI, centroids);if (m->control_pressed) { break; }
                 writeNames(thisCompositeNamesFileName, numOTUs, nameFileName, otuCounts, seqNameVector, aaI, nSeqsPerOTU);                             if (m->control_pressed) { break; }
                 writeClusters(otuCountsFileName, numOTUs, numFlowCells,otuCounts, centroids, uniqueFlowgrams, seqNameVector, aaI, nSeqsPerOTU, lengths, flowDataIntI);                 if (m->control_pressed) { break; }
-                writeGroups(groupFileName, fileRoot, numSeqs, seqNameVector);                                          if (m->control_pressed) { break; }
+                writeGroups(groupFileName, fileGroup, numSeqs, seqNameVector);                                         if (m->control_pressed) { break; }
                 
                 if (large) {
                     if (g > 0) {
-                        m->appendFiles(qualityFileName, (thisOutputDir + m->getRootName(m->getSimpleName(theseFlowFileNames[0])) + "shhh.qual"));
+                        variables["[filename]"] = thisOutputDir + m->getRootName(m->getSimpleName(theseFlowFileNames[0]));
+                        m->appendFiles(qualityFileName, getOutputFileName("qfile",variables));
                         m->mothurRemove(qualityFileName);
-                        m->appendFiles(fastaFileName, (thisOutputDir + m->getRootName(m->getSimpleName(theseFlowFileNames[0])) + "shhh.fasta"));
+                        m->appendFiles(fastaFileName, getOutputFileName("fasta",variables));
                         m->mothurRemove(fastaFileName);
-                        m->appendFiles(nameFileName, (thisOutputDir + m->getRootName(m->getSimpleName(theseFlowFileNames[0])) + "shhh.names"));
+                        m->appendFiles(nameFileName, getOutputFileName("name",variables));
                         m->mothurRemove(nameFileName);
-                        m->appendFiles(otuCountsFileName, (thisOutputDir + m->getRootName(m->getSimpleName(theseFlowFileNames[0])) + "shhh.counts"));
+                        m->appendFiles(otuCountsFileName, getOutputFileName("counts",variables));
                         m->mothurRemove(otuCountsFileName);
-                        m->appendFiles(groupFileName, (thisOutputDir + m->getRootName(m->getSimpleName(theseFlowFileNames[0])) + "shhh.groups"));
+                        m->appendFiles(groupFileName, getOutputFileName("group",variables));
                         m->mothurRemove(groupFileName);
                     }
                     m->mothurRemove(theseFlowFileNames[g]);
                 }
                        }
             
+            numCompleted++;
                        m->mothurOut("Total time to process " + flowFileName + ":\t" + toString(time(NULL) - begTime) + '\t' + toString((clock() - begClock)/(double)CLOCKS_PER_SEC) + '\n');
                }
                
         if (m->control_pressed) { for (int i = 0; i < outputNames.size(); i++) { m->mothurRemove(outputNames[i]); } return 0; }
         
-        return 0;
+        return numCompleted;
         
     }catch(exception& e) {
             m->errorOut(e, "ShhherCommand", "driver");
@@ -2310,18 +2396,27 @@ int ShhherCommand::getFlowData(string filename, vector<string>& thisSeqNameVecto
                thisFlowDataIntI.clear();
                thisNameMap.clear();
                
-               flowFile >> numFlowCells;
+               string numFlowTest;
+        flowFile >> numFlowTest;
+        
+        if (!m->isContainingOnlyDigits(numFlowTest)) { m->mothurOut("[ERROR]: expected a number and got " + numFlowTest + ", quitting. Did you use the flow parameter instead of the file parameter?"); m->mothurOutEndLine(); exit(1); }
+        else { convert(numFlowTest, numFlowCells); }
+        
+        if (m->debug) { m->mothurOut("[DEBUG]: numFlowCells = " + toString(numFlowCells) + ".\n"); }
                int index = 0;//pcluster
                while(!flowFile.eof()){
                        
                        if (m->control_pressed) { break; }
                        
                        flowFile >> seqName >> currentNumFlowCells;
+            
                        thisLengths.push_back(currentNumFlowCells);
            
                        thisSeqNameVector.push_back(seqName);
                        thisNameMap[seqName] = index++;//pcluster
-
+            
+            if (m->debug) { m->mothurOut("[DEBUG]: seqName = " + seqName + " length = " + toString(currentNumFlowCells) + " index = " + toString(index) + "\n"); }
+            
                        for(int i=0;i<numFlowCells;i++){
                                flowFile >> intensity;
                                if(intensity > 9.99)    {       intensity = 9.99;       }
@@ -2556,7 +2651,7 @@ int ShhherCommand::cluster(string filename, string distFileName, string namesFil
                read->read(clusterNameMap);
         
                ListVector* list = read->getListVector();
-               SparseMatrix* matrix = read->getMatrix();
+               SparseDistanceMatrix* matrix = read->getDMatrix();
                
                delete read; 
                delete clusterNameMap; 
@@ -2609,6 +2704,8 @@ int ShhherCommand::getOTUData(int numSeqs, string fileName,  vector<int>& otuDat
                
                listFile >> label >> numOTUs;
         
+        if (m->debug) { m->mothurOut("[DEBUG]: Getting OTU Data...\n"); }
+        
                otuData.assign(numSeqs, 0);
                cumNumSeqs.assign(numOTUs, 0);
                nSeqsPerOTU.assign(numOTUs, 0);
@@ -2623,6 +2720,7 @@ int ShhherCommand::getOTUData(int numSeqs, string fileName,  vector<int>& otuDat
                for(int i=0;i<numOTUs;i++){
                        
                        if (m->control_pressed) { break; }
+            if (m->debug) { m->mothurOut("[DEBUG]: processing OTU " + toString(i) + ".\n"); }
             
                        listFile >> singleOTU;
                        
@@ -3177,7 +3275,7 @@ void ShhherCommand::writeQualities(int numOTUs, int numFlowCells, string quality
                        }
                }
                qualityFile.close();
-               outputNames.push_back(qualityFileName);
+               outputNames.push_back(qualityFileName); outputTypes["qfile"].push_back(qualityFileName);
         
        }
        catch(exception& e) {
@@ -3221,7 +3319,7 @@ void ShhherCommand::writeSequences(string thisCompositeFASTAFileName, int numOTU
                }
                fastaFile.close();
         
-               outputNames.push_back(fastaFileName);
+               outputNames.push_back(fastaFileName); outputTypes["fasta"].push_back(fastaFileName);
         
                if(thisCompositeFASTAFileName != ""){
                        m->appendFiles(fastaFileName, thisCompositeFASTAFileName);
@@ -3256,7 +3354,7 @@ void ShhherCommand::writeNames(string thisCompositeNamesFileName, int numOTUs, s
                        }
                }
                nameFile.close();
-               outputNames.push_back(nameFileName);
+               outputNames.push_back(nameFileName); outputTypes["name"].push_back(nameFileName);
                
                
                if(thisCompositeNamesFileName != ""){
@@ -3281,7 +3379,7 @@ void ShhherCommand::writeGroups(string groupFileName, string fileRoot, int numSe
                        groupFile << seqNameVector[i] << '\t' << fileRoot << endl;
                }
                groupFile.close();
-               outputNames.push_back(groupFileName);
+               outputNames.push_back(groupFileName); outputTypes["group"].push_back(groupFileName);
         
        }
        catch(exception& e) {
@@ -3340,7 +3438,7 @@ void ShhherCommand::writeClusters(string otuCountsFileName, int numOTUs, int num
                        }
                }
                otuCountsFile.close();
-               outputNames.push_back(otuCountsFileName);
+               outputNames.push_back(otuCountsFileName); outputTypes["counts"].push_back(otuCountsFileName);
         
        }
        catch(exception& e) {