]> git.donarmstrong.com Git - mothur.git/blobdiff - clustersplitcommand.cpp
added set.current and get.current commands and modified existing commands to update...
[mothur.git] / clustersplitcommand.cpp
index 861c4fbf3da78e42313d7e76751efd2f5d05f3bf..f9d38227f2d59709235bfe69f87385067317b1ab 100644 (file)
 #include "readmatrix.hpp"
 #include "inputdata.h"
 
+
+//**********************************************************************************************************************
+vector<string> ClusterSplitCommand::getValidParameters(){      
+       try {
+               string AlignArray[] =  {"fasta","phylip","column","name","cutoff","precision","method","splitmethod","taxonomy","taxlevel","large","showabund","timing","hard","processors","outputdir","inputdir"};
+               vector<string> myArray (AlignArray, AlignArray+(sizeof(AlignArray)/sizeof(string)));
+               return myArray;
+       }
+       catch(exception& e) {
+               m->errorOut(e, "ClusterSplitCommand", "getValidParameters");
+               exit(1);
+       }
+}
+//**********************************************************************************************************************
+ClusterSplitCommand::ClusterSplitCommand(){    
+       try {
+               abort = true; calledHelp = true; 
+               vector<string> tempOutNames;
+               outputTypes["list"] = tempOutNames;
+               outputTypes["rabund"] = tempOutNames;
+               outputTypes["sabund"] = tempOutNames;
+               outputTypes["column"] = tempOutNames;
+       }
+       catch(exception& e) {
+               m->errorOut(e, "ClusterSplitCommand", "ClusterSplitCommand");
+               exit(1);
+       }
+}
+//**********************************************************************************************************************
+vector<string> ClusterSplitCommand::getRequiredParameters(){   
+       try {
+               string Array[] =  {"fasta","phylip","column","or"};
+               vector<string> myArray (Array, Array+(sizeof(Array)/sizeof(string)));
+               return myArray;
+       }
+       catch(exception& e) {
+               m->errorOut(e, "ClusterSplitCommand", "getRequiredParameters");
+               exit(1);
+       }
+}
+//**********************************************************************************************************************
+vector<string> ClusterSplitCommand::getRequiredFiles(){        
+       try {
+               vector<string> myArray;
+               return myArray;
+       }
+       catch(exception& e) {
+               m->errorOut(e, "ClusterSplitCommand", "getRequiredFiles");
+               exit(1);
+       }
+}
 //**********************************************************************************************************************
 //This function checks to make sure the cluster command has no errors and then clusters based on the method chosen.
 ClusterSplitCommand::ClusterSplitCommand(string option)  {
        try{
                globaldata = GlobalData::getInstance();
-               abort = false;
+               abort = false; calledHelp = false;   
+               format = "";
                
                //allow user to run help
-               if(option == "help") { help(); abort = true; }
+               if(option == "help") { help(); abort = true; calledHelp = true; }
                
                else {
                        //valid paramters for this command
-                       string Array[] =  {"phylip","column","name","cutoff","precision","method","splitmethod","taxonomy","taxlevel","showabund","timing","hard","processors","outputdir","inputdir"};
+                       string Array[] =  {"fasta","phylip","column","name","cutoff","precision","method","splitmethod","taxonomy","taxlevel","large","showabund","timing","hard","processors","outputdir","inputdir"};
                        vector<string> myArray (Array, Array+(sizeof(Array)/sizeof(string)));
                        
                        OptionParser parser(option);
                        map<string,string> parameters = parser.getParameters();
                        
-                       ValidParameters validParameter;
+                       ValidParameters validParameter("cluster.split");
                
                        //check to make sure all parameters are valid for command
                        map<string,string>::iterator it;
@@ -43,6 +95,13 @@ ClusterSplitCommand::ClusterSplitCommand(string option)  {
                                }
                        }
                        
+                       //initialize outputTypes
+                       vector<string> tempOutNames;
+                       outputTypes["list"] = tempOutNames;
+                       outputTypes["rabund"] = tempOutNames;
+                       outputTypes["sabund"] = tempOutNames;
+                       outputTypes["column"] = tempOutNames;
+                       
                        globaldata->newRead();
                        
                        //if the user changes the output directory command factory will send this info to us in the output parameter 
@@ -56,7 +115,7 @@ ClusterSplitCommand::ClusterSplitCommand(string option)  {
                                it = parameters.find("phylip");
                                //user has given a template file
                                if(it != parameters.end()){ 
-                                       path = hasPath(it->second);
+                                       path = m->hasPath(it->second);
                                        //if the user has not given a path then, add inputdir. else leave path alone.
                                        if (path == "") {       parameters["phylip"] = inputDir + it->second;           }
                                }
@@ -64,7 +123,7 @@ ClusterSplitCommand::ClusterSplitCommand(string option)  {
                                it = parameters.find("column");
                                //user has given a template file
                                if(it != parameters.end()){ 
-                                       path = hasPath(it->second);
+                                       path = m->hasPath(it->second);
                                        //if the user has not given a path then, add inputdir. else leave path alone.
                                        if (path == "") {       parameters["column"] = inputDir + it->second;           }
                                }
@@ -72,7 +131,7 @@ ClusterSplitCommand::ClusterSplitCommand(string option)  {
                                it = parameters.find("name");
                                //user has given a template file
                                if(it != parameters.end()){ 
-                                       path = hasPath(it->second);
+                                       path = m->hasPath(it->second);
                                        //if the user has not given a path then, add inputdir. else leave path alone.
                                        if (path == "") {       parameters["name"] = inputDir + it->second;             }
                                }
@@ -80,10 +139,18 @@ ClusterSplitCommand::ClusterSplitCommand(string option)  {
                                it = parameters.find("taxonomy");
                                //user has given a template file
                                if(it != parameters.end()){ 
-                                       path = hasPath(it->second);
+                                       path = m->hasPath(it->second);
                                        //if the user has not given a path then, add inputdir. else leave path alone.
                                        if (path == "") {       parameters["taxonomy"] = inputDir + it->second;         }
                                }
+                               
+                               it = parameters.find("fasta");
+                               //user has given a template file
+                               if(it != parameters.end()){ 
+                                       path = m->hasPath(it->second);
+                                       //if the user has not given a path then, add inputdir. else leave path alone.
+                                       if (path == "") {       parameters["fasta"] = inputDir + it->second;            }
+                               }
                        }
                        
                        //check for required parameters
@@ -101,16 +168,26 @@ ClusterSplitCommand::ClusterSplitCommand(string option)  {
                        if (namefile == "not open") { abort = true; }   
                        else if (namefile == "not found") { namefile = ""; }
                        
+                       fastafile = validParameter.validFile(parameters, "fasta", true);
+                       if (fastafile == "not open") { abort = true; }  
+                       else if (fastafile == "not found") { fastafile = ""; }
+                       else { distfile = fastafile;  splitmethod = "fasta";  }
+                       
                        taxFile = validParameter.validFile(parameters, "taxonomy", true);
                        if (taxFile == "not open") { abort = true; }    
                        else if (taxFile == "not found") { taxFile = ""; }
                        
-                       if ((phylipfile == "") && (columnfile == "")) { m->mothurOut("When executing a cluster.split command you must enter a phylip or a column."); m->mothurOutEndLine(); abort = true; }
-                       else if ((phylipfile != "") && (columnfile != "")) { m->mothurOut("When executing a cluster.split command you must enter ONLY ONE of the following: phylip or column."); m->mothurOutEndLine(); abort = true; }
+                       if ((phylipfile == "") && (columnfile == "") && (fastafile == "")) { m->mothurOut("When executing a cluster.split command you must enter a phylip or a column or fastafile."); m->mothurOutEndLine(); abort = true; }
+                       else if ((phylipfile != "") && (columnfile != "") && (fastafile != "")) { m->mothurOut("When executing a cluster.split command you must enter ONLY ONE of the following: fasta, phylip or column."); m->mothurOutEndLine(); abort = true; }
                
                        if (columnfile != "") {
                                if (namefile == "") { m->mothurOut("You need to provide a namefile if you are going to use the column format."); m->mothurOutEndLine(); abort = true; }
                        }
+                       
+                       if (fastafile != "") {
+                               if (taxFile == "") { m->mothurOut("You need to provide a taxonomy file if you are using a fasta file to generate the split."); m->mothurOutEndLine(); abort = true; }
+                               if (namefile == "") { m->mothurOut("You need to provide a names file if you are using a fasta file to generate the split."); m->mothurOutEndLine(); abort = true; }
+                       }
                                        
                        //check for optional parameter and set defaults
                        // ...at some point should added some additional type checking...
@@ -123,12 +200,19 @@ ClusterSplitCommand::ClusterSplitCommand(string option)  {
                        convert(temp, precision); 
                        
                        temp = validParameter.validFile(parameters, "hard", false);                     if (temp == "not found") { temp = "F"; }
-                       hard = isTrue(temp);
+                       hard = m->isTrue(temp);
+                       
+                       temp = validParameter.validFile(parameters, "large", false);                    if (temp == "not found") { temp = "F"; }
+                       large = m->isTrue(temp);
                        
                        temp = validParameter.validFile(parameters, "processors", false);       if (temp == "not found"){       temp = "1";                             }
                        convert(temp, processors); 
                        
-                       splitmethod = validParameter.validFile(parameters, "splitmethod", false);               if (splitmethod == "not found") { splitmethod = "distance"; }
+                       temp = validParameter.validFile(parameters, "splitmethod", false);      
+                       if (splitmethod != "fasta") {
+                               if (temp == "not found")  { splitmethod = "distance"; }
+                               else {  splitmethod = temp; }
+                       }
                        
                        temp = validParameter.validFile(parameters, "cutoff", false);           if (temp == "not found")  { temp = "10"; }
                        convert(temp, cutoff); 
@@ -142,8 +226,8 @@ ClusterSplitCommand::ClusterSplitCommand(string option)  {
                        if ((method == "furthest") || (method == "nearest") || (method == "average")) { }
                        else { m->mothurOut("Not a valid clustering method.  Valid clustering algorithms are furthest, nearest or average."); m->mothurOutEndLine(); abort = true; }
                        
-                       if ((splitmethod == "distance") || (splitmethod == "classify")) { }
-                       else { m->mothurOut("Not a valid splitting method.  Valid splitting algorithms are distance or classify."); m->mothurOutEndLine(); abort = true; }
+                       if ((splitmethod == "distance") || (splitmethod == "classify") || (splitmethod == "fasta")) { }
+                       else { m->mothurOut(splitmethod + " is not a valid splitting method.  Valid splitting algorithms are distance, classify or fasta."); m->mothurOutEndLine(); abort = true; }
                        
                        if ((splitmethod == "classify") && (taxFile == "")) {  m->mothurOut("You need to provide a taxonomy file if you are going to use the classify splitmethod."); m->mothurOutEndLine(); abort = true;  }
 
@@ -165,15 +249,26 @@ ClusterSplitCommand::ClusterSplitCommand(string option)  {
 
 void ClusterSplitCommand::help(){
        try {
-               m->mothurOut("The cluster.split command parameter options are phylip, column, name, cutoff, precision, method, splitmethod, taxonomy, taxlevel, showabund, timing, hard, processors. Phylip or column and name are required.\n");
+               m->mothurOut("The cluster.split command parameter options are fasta, phylip, column, name, cutoff, precision, method, splitmethod, taxonomy, taxlevel, showabund, timing, hard, large, processors. Fasta or Phylip or column and name are required.\n");
+               m->mothurOut("The cluster.split command can split your files in 3 ways. Splitting by distance file, by classification, or by classification also using a fasta file. \n");
+               m->mothurOut("For the distance file method, you need only provide your distance file and mothur will split the file into distinct groups. \n");
+               m->mothurOut("For the classification method, you need to provide your distance file and taxonomy file, and set the splitmethod to classify.  \n");
+               m->mothurOut("You will also need to set the taxlevel you want to split by. mothur will split the sequences into distinct taxonomy groups, and split the distance file based on those groups. \n");
+               m->mothurOut("For the classification method using a fasta file, you need to provide your fasta file, names file and taxonomy file.  \n");
+               m->mothurOut("You will also need to set the taxlevel you want to split by. mothur will split the sequence into distinct taxonomy groups, and create distance files for each grouping. \n");
                m->mothurOut("The phylip and column parameter allow you to enter your distance file. \n");
+               m->mothurOut("The fasta parameter allows you to enter your aligned fasta file. \n");
                m->mothurOut("The name parameter allows you to enter your name file and is required if your distance file is in column format. \n");
                m->mothurOut("The cutoff parameter allow you to set the distance you want to cluster to, default is 10.0. \n");
                m->mothurOut("The precision parameter allows you specify the precision of the precision of the distances outputted, default=100, meaning 2 decimal places. \n");
                m->mothurOut("The method allows you to specify what clustering algorythm you want to use, default=furthest, option furthest, nearest, or average. \n");
-               m->mothurOut("The splitmethod parameter allows you to specify how you want to split your distance file before you cluster, default=distance, options distance or classify. \n");
+               m->mothurOut("The splitmethod parameter allows you to specify how you want to split your distance file before you cluster, default=distance, options distance, classify or fasta. \n");
                m->mothurOut("The taxonomy parameter allows you to enter the taxonomy file for your sequences, this is only valid if you are using splitmethod=classify. Be sure your taxonomy file does not include the probability scores. \n");
-               m->mothurOut("The taxlevel parameter allows you to specify the taxonomy level you want to use to split the distance file, default=1. \n");
+               m->mothurOut("The taxlevel parameter allows you to specify the taxonomy level you want to use to split the distance file, default=1, meaning use the first taxon in each list. \n");
+               m->mothurOut("The large parameter allows you to indicate that your distance matrix is too large to fit in RAM.  The default value is false.\n");
+               #ifdef USE_MPI
+               m->mothurOut("When using MPI, the processors parameter is set to the number of MPI processes running. \n");
+               #endif
                m->mothurOut("The cluster.split command should be in the following format: \n");
                m->mothurOut("cluster.split(column=youDistanceFile, name=yourNameFile, method=yourMethod, cutoff=yourCutoff, precision=yourPrecision, splitmethod=yourSplitmethod, taxonomy=yourTaxonomyfile, taxlevel=yourtaxlevel) \n");
                m->mothurOut("Example: cluster.split(column=abrecovery.dist, name=abrecovery.names, method=furthest, cutoff=0.10, precision=1000, splitmethod=classify, taxonomy=abrecovery.silva.slv.taxonomy, taxlevel=5) \n");       
@@ -194,13 +289,31 @@ ClusterSplitCommand::~ClusterSplitCommand(){}
 int ClusterSplitCommand::execute(){
        try {
        
-               if (abort == true) {    return 0;       }
+               if (abort == true) { if (calledHelp) { return 0; }  return 2;   }
                
+               time_t estart;
+               vector<string> listFileNames;
+               set<string> labels;
+               string singletonName = "";
+               double saveCutoff = cutoff;
+
                //****************** file prep work ******************************//
+               #ifdef USE_MPI
+                       int pid;
+                       int tag = 2001;
+                       MPI_Status status; 
+                       MPI_Comm_size(MPI_COMM_WORLD, &processors); //set processors to the number of mpi processes running
+                       MPI_Comm_rank(MPI_COMM_WORLD, &pid); //find out who we are
+                       
+                       if (pid == 0) { //only process 0 converts and splits
+                       
+               #endif
                
                //if user gave a phylip file convert to column file
                if (format == "phylip") {
-       
+                       estart = time(NULL);
+                       m->mothurOut("Converting to column format..."); m->mothurOutEndLine();
+                       
                        ReadCluster* convert = new ReadCluster(distfile, cutoff, outputDir, false);
                        
                        NameAssignment* nameMap = NULL;
@@ -216,7 +329,7 @@ int ClusterSplitCommand::execute(){
                        if (namefile == "") {  //you need to make a namefile for split matrix
                                ofstream out;
                                namefile = phylipfile + ".names";
-                               openOutputFile(namefile, out);
+                               m->openOutputFile(namefile, out);
                                for (int i = 0; i < listToMakeNameFile->getNumBins(); i++) {
                                        string bin = listToMakeNameFile->get(i);
                                        out << bin << '\t' << bin << endl;
@@ -225,32 +338,195 @@ int ClusterSplitCommand::execute(){
                        }
                        delete listToMakeNameFile;
                        delete convert;
+                       
+                       m->mothurOut("It took " + toString(time(NULL) - estart) + " seconds to convert the distance file."); m->mothurOutEndLine();
                }
                if (m->control_pressed) { return 0; }
                
-               time_t estart = time(NULL);
+               estart = time(NULL);
+               m->mothurOut("Splitting the file..."); m->mothurOutEndLine();
                
                //split matrix into non-overlapping groups
                SplitMatrix* split;
-               if (splitmethod == "distance")  {       split = new SplitMatrix(distfile, namefile, taxFile, cutoff, splitmethod);                      }
-               else                                                    {       split = new SplitMatrix(distfile, namefile, taxFile, taxLevelCutoff, splitmethod);  }
+               if (splitmethod == "distance")                  {       split = new SplitMatrix(distfile, namefile, taxFile, cutoff, splitmethod, large);                                                       }
+               else if (splitmethod == "classify")             {       split = new SplitMatrix(distfile, namefile, taxFile, taxLevelCutoff, splitmethod, large);                                       }
+               else if (splitmethod == "fasta")                {       split = new SplitMatrix(fastafile, namefile, taxFile, taxLevelCutoff, cutoff, splitmethod, processors, outputDir);      }
+               else { m->mothurOut("Not a valid splitting method.  Valid splitting algorithms are distance, classify or fasta."); m->mothurOutEndLine(); return 0;             }
                
                split->split();
                
                if (m->control_pressed) { delete split; return 0; }
                
-               string singletonName = split->getSingletonNames();
+               singletonName = split->getSingletonNames();
                vector< map<string, string> > distName = split->getDistanceFiles();  //returns map of distance files -> namefile sorted by distance file size
                delete split;
                
+               //output a merged distance file
+               if (splitmethod == "fasta")             { createMergedDistanceFile(distName); }
+                       
+                               
                if (m->control_pressed) { return 0; }
                
                m->mothurOut("It took " + toString(time(NULL) - estart) + " seconds to split the distance file."); m->mothurOutEndLine();
                estart = time(NULL);
                
                //****************** break up files between processes and cluster each file set ******************************//
-               vector<string> listFileNames;
-               set<string> labels;
+       #ifdef USE_MPI
+                       ////you are process 0 from above////
+                       
+                       vector < vector < map<string, string> > > dividedNames; //distNames[1] = vector of filenames for process 1...                           
+                       dividedNames.resize(processors);
+                                       
+                       //for each file group figure out which process will complete it
+                       //want to divide the load intelligently so the big files are spread between processes
+                       for (int i = 0; i < distName.size(); i++) { 
+                               int processToAssign = (i+1) % processors; 
+                               if (processToAssign == 0) { processToAssign = processors; }
+                                               
+                               dividedNames[(processToAssign-1)].push_back(distName[i]);
+                       }
+                                       
+                       //not lets reverse the order of ever other process, so we balance big files running with little ones
+                       for (int i = 0; i < processors; i++) {
+                               int remainder = ((i+1) % processors);
+                               if (remainder) {  reverse(dividedNames[i].begin(), dividedNames[i].end());  }
+                       }
+                       
+                       
+                       //send each child the list of files it needs to process
+                       for(int i = 1; i < processors; i++) { 
+                               //send number of file pairs
+                               int num = dividedNames[i].size();
+                               MPI_Send(&num, 1, MPI_INT, i, tag, MPI_COMM_WORLD);
+                               
+                               for (int j = 0; j < num; j++) { //send filenames to process i
+                                       char tempDistFileName[1024];
+                                       strcpy(tempDistFileName, (dividedNames[i][j].begin()->first).c_str());
+                                       int lengthDist = (dividedNames[i][j].begin()->first).length();
+                                       
+                                       MPI_Send(&lengthDist, 1, MPI_INT, i, tag, MPI_COMM_WORLD);
+                                       MPI_Send(tempDistFileName, 1024, MPI_CHAR, i, tag, MPI_COMM_WORLD);
+                                       
+                                       char tempNameFileName[1024];
+                                       strcpy(tempNameFileName, (dividedNames[i][j].begin()->second).c_str());
+                                       int lengthName = (dividedNames[i][j].begin()->second).length();
+
+                                       MPI_Send(&lengthName, 1, MPI_INT, i, tag, MPI_COMM_WORLD);
+                                       MPI_Send(tempNameFileName, 1024, MPI_CHAR, i, tag, MPI_COMM_WORLD);
+                               }
+                       }
+                       
+                       //process your share
+                       listFileNames = cluster(dividedNames[0], labels);
+                       
+                       //receive the other processes info
+                       for(int i = 1; i < processors; i++) { 
+                               int num = dividedNames[i].size();
+                               
+                               double tempCutoff;
+                               MPI_Recv(&tempCutoff, 1, MPI_DOUBLE, i, tag, MPI_COMM_WORLD, &status);
+                               if (tempCutoff < cutoff) { cutoff = tempCutoff; }
+                               
+                               //send list filenames to root process
+                               for (int j = 0; j < num; j++) {  
+                                       int lengthList = 0;
+                                       char tempListFileName[1024];
+                               
+                                       MPI_Recv(&lengthList, 1, MPI_INT, i, tag, MPI_COMM_WORLD, &status);
+                                       MPI_Recv(tempListFileName, 1024, MPI_CHAR, i, tag, MPI_COMM_WORLD, &status); 
+                               
+                                       string myListFileName = tempListFileName;
+                                       myListFileName = myListFileName.substr(0, lengthList);
+                                       
+                                       listFileNames.push_back(myListFileName);
+                               }
+                               
+                               //send Labels to root process
+                               int numLabels = 0;
+                               MPI_Recv(&numLabels, 1, MPI_INT, i, tag, MPI_COMM_WORLD, &status);
+                               
+                               for (int j = 0; j < numLabels; j++) {  
+                                       int lengthLabel = 0;
+                                       char tempLabel[100];
+                               
+                                       MPI_Recv(&lengthLabel, 1, MPI_INT, i, tag, MPI_COMM_WORLD, &status);
+                                       MPI_Recv(tempLabel, 100, MPI_CHAR, i, tag, MPI_COMM_WORLD, &status); 
+                               
+                                       string myLabel = tempLabel;
+                                       myLabel = myLabel.substr(0, lengthLabel);
+                       
+                                       if (labels.count(myLabel) == 0) { labels.insert(myLabel); }
+                               }
+                       }
+                       
+               }else { //you are a child process
+                       vector < map<string, string> >  myNames;
+                       
+                       //recieve the files you need to process
+                       //receive number of file pairs
+                       int num = 0;
+                       MPI_Recv(&num, 1, MPI_INT, 0, tag, MPI_COMM_WORLD, &status);
+                       
+                       myNames.resize(num);
+       
+                       for (int j = 0; j < num; j++) { //receive filenames to process 
+                               int lengthDist = 0;
+                               char tempDistFileName[1024];
+                               
+                               MPI_Recv(&lengthDist, 1, MPI_INT, 0, tag, MPI_COMM_WORLD, &status);
+                               MPI_Recv(tempDistFileName, 1024, MPI_CHAR, 0, tag, MPI_COMM_WORLD, &status); 
+                               
+                               string myDistFileName = tempDistFileName;
+                               myDistFileName = myDistFileName.substr(0, lengthDist);
+                       
+                               int lengthName = 0;
+                               char tempNameFileName[1024];
+                               
+                               MPI_Recv(&lengthName, 1, MPI_INT, 0, tag, MPI_COMM_WORLD, &status);
+                               MPI_Recv(tempNameFileName, 1024, MPI_CHAR, 0, tag, MPI_COMM_WORLD, &status); 
+                               
+                               string myNameFileName = tempNameFileName;
+                               myNameFileName = myNameFileName.substr(0, lengthName);
+                               
+                               //save file name
+                               myNames[j][myDistFileName] = myNameFileName;
+                       }
+       
+                       //process them
+                       listFileNames = cluster(myNames, labels);
+                       
+                       //send cutoff
+                       MPI_Send(&cutoff, 1, MPI_DOUBLE, 0, tag, MPI_COMM_WORLD);
+                       
+                       //send list filenames to root process
+                       for (int j = 0; j < num; j++) {  
+                               char tempListFileName[1024];
+                               strcpy(tempListFileName, listFileNames[j].c_str());
+                               int lengthList = listFileNames[j].length();
+                                       
+                               MPI_Send(&lengthList, 1, MPI_INT, 0, tag, MPI_COMM_WORLD);
+                               MPI_Send(tempListFileName, 1024, MPI_CHAR, 0, tag, MPI_COMM_WORLD);
+                       }
+                       
+                       //send Labels to root process
+                       int numLabels = labels.size();
+                       MPI_Send(&numLabels, 1, MPI_INT, 0, tag, MPI_COMM_WORLD);
+                       
+                       for(set<string>::iterator it = labels.begin(); it != labels.end(); ++it) {
+                               char tempLabel[100];
+                               strcpy(tempLabel, (*it).c_str());
+                               int lengthLabel = (*it).length();
+                                       
+                               MPI_Send(&lengthLabel, 1, MPI_INT, 0, tag, MPI_COMM_WORLD);
+                               MPI_Send(tempLabel, 100, MPI_CHAR, 0, tag, MPI_COMM_WORLD);
+                       }
+               }
+               
+               //make everyone wait
+               MPI_Barrier(MPI_COMM_WORLD);
+               
+       #else
+
                #if defined (__APPLE__) || (__MACH__) || (linux) || (__linux)
                                if(processors == 1){
                                        listFileNames = cluster(distName, labels); //clusters individual files and returns names of list files
@@ -260,7 +536,6 @@ int ClusterSplitCommand::execute(){
                                        
                                        //for each file group figure out which process will complete it
                                        //want to divide the load intelligently so the big files are spread between processes
-                                       int count = 1;
                                        for (int i = 0; i < distName.size(); i++) { 
                                                int processToAssign = (i+1) % processors; 
                                                if (processToAssign == 0) { processToAssign = processors; }
@@ -282,11 +557,13 @@ int ClusterSplitCommand::execute(){
                                        for(int i=0;i<processors;i++){
                                                string filename = toString(processIDS[i]) + ".temp";
                                                ifstream in;
-                                               openInputFile(filename, in);
+                                               m->openInputFile(filename, in);
+                                               
+                                               in >> tag; m->gobble(in);
                                                
                                                while(!in.eof()) {
                                                        string tempName;
-                                                       in >> tempName; gobble(in);
+                                                       in >> tempName; m->gobble(in);
                                                        listFileNames.push_back(tempName);
                                                }
                                                in.close();
@@ -295,11 +572,15 @@ int ClusterSplitCommand::execute(){
                                                //get labels
                                                filename = toString(processIDS[i]) + ".temp.labels";
                                                ifstream in2;
-                                               openInputFile(filename, in2);
+                                               m->openInputFile(filename, in2);
+                                               
+                                               float tempCutoff;
+                                               in2 >> tempCutoff; m->gobble(in2);
+                                               if (tempCutoff < cutoff) { cutoff = tempCutoff; }
                                                
                                                while(!in2.eof()) {
                                                        string tempName;
-                                                       in2 >> tempName; gobble(in);
+                                                       in2 >> tempName; m->gobble(in2);
                                                        if (labels.count(tempName) == 0) { labels.insert(tempName); }
                                                }
                                                in2.close();
@@ -309,10 +590,21 @@ int ClusterSplitCommand::execute(){
                #else
                                listFileNames = cluster(distName, labels); //clusters individual files and returns names of list files
                #endif
-               
+       #endif  
                if (m->control_pressed) { for (int i = 0; i < listFileNames.size(); i++) { remove(listFileNames[i].c_str()); } return 0; }
                
+               if (saveCutoff != cutoff) { m->mothurOut("Cutoff was " + toString(saveCutoff) + " changed cutoff to " + toString(cutoff)); m->mothurOutEndLine();  }
+               
+               m->mothurOut("It took " + toString(time(NULL) - estart) + " seconds to cluster"); m->mothurOutEndLine();
+               
                //****************** merge list file and create rabund and sabund files ******************************//
+               estart = time(NULL);
+               m->mothurOut("Merging the clustered files..."); m->mothurOutEndLine();
+               
+               #ifdef USE_MPI
+                       if (pid == 0) { //only process 0 merges
+               #endif
+
                ListVector* listSingle;
                map<float, int> labelBins = completeListFile(listFileNames, singletonName, labels, listSingle); //returns map of label to numBins
                
@@ -322,12 +614,38 @@ int ClusterSplitCommand::execute(){
 
                if (m->control_pressed) { for (int i = 0; i < outputNames.size(); i++) { remove(outputNames[i].c_str()); } return 0; }
                
-               m->mothurOut("It took " + toString(time(NULL) - estart) + " seconds to cluster"); m->mothurOutEndLine();
+               m->mothurOut("It took " + toString(time(NULL) - estart) + " seconds to merge."); m->mothurOutEndLine();
+               
+               //set list file as new current listfile
+               string current = "";
+               itTypes = outputTypes.find("list");
+               if (itTypes != outputTypes.end()) {
+                       if ((itTypes->second).size() != 0) { current = (itTypes->second)[0]; m->setListFile(current); }
+               }
+               
+               //set rabund file as new current rabundfile
+               itTypes = outputTypes.find("rabund");
+               if (itTypes != outputTypes.end()) {
+                       if ((itTypes->second).size() != 0) { current = (itTypes->second)[0]; m->setRabundFile(current); }
+               }
                
+               //set sabund file as new current sabundfile
+               itTypes = outputTypes.find("sabund");
+               if (itTypes != outputTypes.end()) {
+                       if ((itTypes->second).size() != 0) { current = (itTypes->second)[0]; m->setSabundFile(current); }
+               }
+                               
                m->mothurOutEndLine();
                m->mothurOut("Output File Names: "); m->mothurOutEndLine();
                for (int i = 0; i < outputNames.size(); i++) {  m->mothurOut(outputNames[i]); m->mothurOutEndLine();    }
                m->mothurOutEndLine();
+               
+               #ifdef USE_MPI
+                       } //only process 0 merges
+                       
+                       //make everyone wait
+                       MPI_Barrier(MPI_COMM_WORLD);
+               #endif
 
                return 0;
        }
@@ -337,7 +655,7 @@ int ClusterSplitCommand::execute(){
        }
 }
 //**********************************************************************************************************************
-map<float, int> ClusterSplitCommand::completeListFile(vector<string> listNames, string singleton, set<string> userLabels, ListVector*& listSingle){
+map<float, int> ClusterSplitCommand::completeListFile(vector<string> listNames, string singleton, set<string>& userLabels, ListVector*& listSingle){
        try {
                                
                map<float, int> labelBin;
@@ -347,12 +665,12 @@ map<float, int> ClusterSplitCommand::completeListFile(vector<string> listNames,
                //read in singletons
                if (singleton != "none") {
                        ifstream in;
-                       openInputFile(singleton, in);
+                       m->openInputFile(singleton, in);
                                
                        string firstCol, secondCol;
                        listSingle = new ListVector();
                        while (!in.eof()) {
-                               in >> firstCol >> secondCol; gobble(in);
+                               in >> firstCol >> secondCol; m->gobble(in);
                                listSingle->push_back(secondCol);
                        }
                        in.close();
@@ -368,8 +686,10 @@ map<float, int> ClusterSplitCommand::completeListFile(vector<string> listNames,
                        if ((*it != "unique") && (convertTestFloat(*it, temp) == true)) {       convert(*it, temp);     }
                        else if (*it == "unique")                                                                               {       temp = -1.0;            }
                        
-                       orderFloat.push_back(temp);
-                       labelBin[temp] = numSingleBins; //initialize numbins 
+                       if (temp <= cutoff) {
+                               orderFloat.push_back(temp);
+                               labelBin[temp] = numSingleBins; //initialize numbins 
+                       }
                }
        
                //sort order
@@ -391,7 +711,7 @@ map<float, int> ClusterSplitCommand::completeListFile(vector<string> listNames,
                        
                        string filledInList = listNames[k] + "filledInTemp";
                        ofstream outFilled;
-                       openOutputFile(filledInList, outFilled);
+                       m->openOutputFile(filledInList, outFilled);
        
                        //for each label needed
                        for(int l = 0; l < orderFloat.size(); l++){
@@ -448,16 +768,16 @@ map<float, int> ClusterSplitCommand::completeListFile(vector<string> listNames,
 //**********************************************************************************************************************
 int ClusterSplitCommand::mergeLists(vector<string> listNames, map<float, int> userLabels, ListVector* listSingle){
        try {
-               if (outputDir == "") { outputDir += hasPath(distfile); }
-               fileroot = outputDir + getRootName(getSimpleName(distfile));
+               if (outputDir == "") { outputDir += m->hasPath(distfile); }
+               fileroot = outputDir + m->getRootName(m->getSimpleName(distfile));
                
-               openOutputFile(fileroot+ tag + ".sabund",       outSabund);
-               openOutputFile(fileroot+ tag + ".rabund",       outRabund);
-               openOutputFile(fileroot+ tag + ".list",         outList);
+               m->openOutputFile(fileroot+ tag + ".sabund",    outSabund);
+               m->openOutputFile(fileroot+ tag + ".rabund",    outRabund);
+               m->openOutputFile(fileroot+ tag + ".list",              outList);
                                
-               outputNames.push_back(fileroot+ tag + ".sabund");
-               outputNames.push_back(fileroot+ tag + ".rabund");
-               outputNames.push_back(fileroot+ tag + ".list");
+               outputNames.push_back(fileroot+ tag + ".sabund");  outputTypes["list"].push_back(fileroot+ tag + ".list");
+               outputNames.push_back(fileroot+ tag + ".rabund");  outputTypes["rabund"].push_back(fileroot+ tag + ".rabund");
+               outputNames.push_back(fileroot+ tag + ".list");    outputTypes["sabund"].push_back(fileroot+ tag + ".sabund");
                
                map<float, int>::iterator itLabel;
 
@@ -477,7 +797,7 @@ int ClusterSplitCommand::mergeLists(vector<string> listNames, map<float, int> us
                        if (listSingle != NULL) {
                                for (int j = 0; j < listSingle->getNumBins(); j++) {
                                        outList << listSingle->get(j) << '\t';
-                                       rabund->push_back(getNumNames(listSingle->get(j)));
+                                       rabund->push_back(m->getNumNames(listSingle->get(j)));
                                }
                        }
                        
@@ -494,7 +814,7 @@ int ClusterSplitCommand::mergeLists(vector<string> listNames, map<float, int> us
                                else {          
                                        for (int j = 0; j < list->getNumBins(); j++) {
                                                outList << list->get(j) << '\t';
-                                               rabund->push_back(getNumNames(list->get(j)));
+                                               rabund->push_back(m->getNumNames(list->get(j)));
                                        }
                                        delete list;
                                }
@@ -534,7 +854,7 @@ void ClusterSplitCommand::printData(ListVector* oldList){
                RAbundVector oldRAbund = oldList->getRAbundVector();
                
                oldRAbund.setLabel(label);
-               if (isTrue(showabund)) {
+               if (m->isTrue(showabund)) {
                        oldRAbund.getSAbundVector().print(cout);
                }
                oldRAbund.print(outRabund);
@@ -570,22 +890,28 @@ int ClusterSplitCommand::createProcesses(vector < vector < map<string, string> >
                                //write out names to file
                                string filename = toString(getpid()) + ".temp";
                                ofstream out;
-                               openOutputFile(filename, out);
+                               m->openOutputFile(filename, out);
+                               out << tag << endl;
                                for (int j = 0; j < listFileNames.size(); j++) { out << listFileNames[j] << endl;  }
                                out.close();
                                
                                //print out labels
                                ofstream outLabels;
                                filename = toString(getpid()) + ".temp.labels";
-                               openOutputFile(filename, outLabels);
-               
+                               m->openOutputFile(filename, outLabels);
+                               
+                               outLabels << cutoff << endl;
                                for (set<string>::iterator it = labels.begin(); it != labels.end(); it++) {
                                        outLabels << (*it) << endl;
                                }
                                outLabels.close();
 
                                exit(0);
-                       }else { m->mothurOut("unable to spawn the necessary processes."); m->mothurOutEndLine(); exit(0); }
+                       }else { 
+                               m->mothurOut("[ERROR]: unable to spawn the necessary processes."); m->mothurOutEndLine(); 
+                               for (int i = 0; i < processIDS.size(); i++) { kill (processIDS[i], SIGINT); }
+                               exit(0);
+                       }
                }
                
                //force parent to wait until all the processes are done
@@ -615,8 +941,11 @@ vector<string> ClusterSplitCommand::cluster(vector< map<string, string> > distNa
                
                vector<string> listFileNames;
                
+               double smallestCutoff = cutoff;
+               
                //cluster each distance file
                for (int i = 0; i < distNames.size(); i++) {
+                       if (m->control_pressed) { return listFileNames; }
                        
                        string thisNamefile = distNames[i].begin()->second;
                        string thisDistFile = distNames[i].begin()->first;
@@ -625,6 +954,16 @@ vector<string> ClusterSplitCommand::cluster(vector< map<string, string> > distNa
                        globaldata->setNameFile(thisNamefile);
                        globaldata->setColumnFile(thisDistFile); globaldata->setFormat("column");
                        
+                       #ifdef USE_MPI
+                               int pid;
+                               MPI_Comm_rank(MPI_COMM_WORLD, &pid); //find out who we are
+                               
+                               //output your files too
+                               if (pid != 0) {
+                                       cout << endl << "Reading " << thisDistFile << endl;
+                               }
+                       #endif
+                       
                        m->mothurOutEndLine(); m->mothurOut("Reading " + thisDistFile); m->mothurOutEndLine();
                        
                        ReadMatrix* read = new ReadColumnMatrix(thisDistFile);  
@@ -643,6 +982,14 @@ vector<string> ClusterSplitCommand::cluster(vector< map<string, string> > distNa
                        delete read; 
                        delete nameMap; 
                        
+                       
+                       #ifdef USE_MPI
+                               //output your files too
+                               if (pid != 0) {
+                                       cout << endl << "Clustering " << thisDistFile << endl;
+                               }
+                       #endif
+                       
                        m->mothurOutEndLine(); m->mothurOut("Clustering " + thisDistFile); m->mothurOutEndLine();
                
                        rabund = new RAbundVector(list->getRAbundVector());
@@ -653,16 +1000,14 @@ vector<string> ClusterSplitCommand::cluster(vector< map<string, string> > distNa
                        else if(method == "average"){   cluster = new AverageLinkage(rabund, list, matrix, cutoff, method);     }
                        tag = cluster->getTag();
                
-                       if (outputDir == "") { outputDir += hasPath(thisDistFile); }
-                       fileroot = outputDir + getRootName(getSimpleName(thisDistFile));
+                       if (outputDir == "") { outputDir += m->hasPath(thisDistFile); }
+                       fileroot = outputDir + m->getRootName(m->getSimpleName(thisDistFile));
                        
                        ofstream listFile;
-                       openOutputFile(fileroot+ tag + ".list", listFile);
+                       m->openOutputFile(fileroot+ tag + ".list",      listFile);
                
                        listFileNames.push_back(fileroot+ tag + ".list");
-               
-                       time_t estart = time(NULL);
-                       
+                               
                        float previousDist = 0.00000;
                        float rndPreviousDist = 0.00000;
                        
@@ -681,14 +1026,14 @@ vector<string> ClusterSplitCommand::cluster(vector< map<string, string> > distNa
                                        listFileNames.clear(); return listFileNames;
                                }
                
-                               cluster->update(cutoff);
+                               cluster->update(saveCutoff);
        
                                float dist = matrix->getSmallDist();
                                float rndDist;
                                if (hard) {
-                                       rndDist = ceilDist(dist, precision); 
+                                       rndDist = m->ceilDist(dist, precision); 
                                }else{
-                                       rndDist = roundDist(dist, precision); 
+                                       rndDist = m->roundDist(dist, precision); 
                                }
 
                                if(previousDist <= 0.0000 && dist != previousDist){
@@ -729,8 +1074,18 @@ vector<string> ClusterSplitCommand::cluster(vector< map<string, string> > distNa
                        
                        remove(thisDistFile.c_str());
                        remove(thisNamefile.c_str());
+                       
+                       if (saveCutoff != cutoff) { 
+                               if (hard)       {  saveCutoff = m->ceilDist(saveCutoff, precision);     }
+                               else            {       saveCutoff = m->roundDist(saveCutoff, precision);  }
+                       
+                               m->mothurOut("Cutoff was " + toString(cutoff) + " changed cutoff to " + toString(saveCutoff)); m->mothurOutEndLine();  
+                       }
+                       
+                       if (saveCutoff < smallestCutoff) { smallestCutoff = saveCutoff;  }
                }
                
+               cutoff = smallestCutoff;
                                        
                return listFileNames;
        
@@ -742,5 +1097,45 @@ vector<string> ClusterSplitCommand::cluster(vector< map<string, string> > distNa
 
 
 }
+//**********************************************************************************************************************
 
+int ClusterSplitCommand::createMergedDistanceFile(vector< map<string, string> > distNames) {
+       try{
+               
+#ifdef USE_MPI
+               int pid;
+               MPI_Comm_rank(MPI_COMM_WORLD, &pid); //find out who we are
+               
+               if (pid != 0) {
+#endif
+               
+               string thisOutputDir = outputDir;
+               if (outputDir == "") { thisOutputDir = m->hasPath(fastafile); }
+               string outputFileName = thisOutputDir + m->getRootName(m->getSimpleName(fastafile)) + "dist";
+               remove(outputFileName.c_str());
+               
+               
+               for (int i = 0; i < distNames.size(); i++) {
+                       if (m->control_pressed) {  return 0; }
+                       
+                       string thisDistFile = distNames[i].begin()->first;
+                       
+                       m->appendFiles(thisDistFile, outputFileName);
+               }       
+                       
+               outputTypes["column"].push_back(outputFileName); outputNames.push_back(outputFileName);
+                       
+#ifdef USE_MPI
+               }
+#endif
+                               
+               return 0;       
+               
+               
+       }
+       catch(exception& e) {
+               m->errorOut(e, "ClusterSplitCommand", "createMergedDistanceFile");
+               exit(1);
+       }
+}
 //**********************************************************************************************************************