X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=clustersplitcommand.cpp;h=f9d38227f2d59709235bfe69f87385067317b1ab;hb=348de0f8b17d84ede77081dcf67bd6ef43496677;hp=ced5a6384c792188d34068f4d899ff5ef306adeb;hpb=e4c80376cc4533f66c8dfc18f3e1a86a60ac17fe;p=mothur.git diff --git a/clustersplitcommand.cpp b/clustersplitcommand.cpp index ced5a63..f9d3822 100644 --- a/clustersplitcommand.cpp +++ b/clustersplitcommand.cpp @@ -15,16 +15,67 @@ #include "readmatrix.hpp" #include "inputdata.h" + +//********************************************************************************************************************** +vector ClusterSplitCommand::getValidParameters(){ + try { + string AlignArray[] = {"fasta","phylip","column","name","cutoff","precision","method","splitmethod","taxonomy","taxlevel","large","showabund","timing","hard","processors","outputdir","inputdir"}; + vector myArray (AlignArray, AlignArray+(sizeof(AlignArray)/sizeof(string))); + return myArray; + } + catch(exception& e) { + m->errorOut(e, "ClusterSplitCommand", "getValidParameters"); + exit(1); + } +} +//********************************************************************************************************************** +ClusterSplitCommand::ClusterSplitCommand(){ + try { + abort = true; calledHelp = true; + vector tempOutNames; + outputTypes["list"] = tempOutNames; + outputTypes["rabund"] = tempOutNames; + outputTypes["sabund"] = tempOutNames; + outputTypes["column"] = tempOutNames; + } + catch(exception& e) { + m->errorOut(e, "ClusterSplitCommand", "ClusterSplitCommand"); + exit(1); + } +} +//********************************************************************************************************************** +vector ClusterSplitCommand::getRequiredParameters(){ + try { + string Array[] = {"fasta","phylip","column","or"}; + vector myArray (Array, Array+(sizeof(Array)/sizeof(string))); + return myArray; + } + catch(exception& e) { + m->errorOut(e, "ClusterSplitCommand", "getRequiredParameters"); + exit(1); + } +} +//********************************************************************************************************************** +vector ClusterSplitCommand::getRequiredFiles(){ + try { + vector myArray; + return myArray; + } + catch(exception& e) { + m->errorOut(e, "ClusterSplitCommand", "getRequiredFiles"); + exit(1); + } +} //********************************************************************************************************************** //This function checks to make sure the cluster command has no errors and then clusters based on the method chosen. ClusterSplitCommand::ClusterSplitCommand(string option) { try{ globaldata = GlobalData::getInstance(); - abort = false; + abort = false; calledHelp = false; format = ""; //allow user to run help - if(option == "help") { help(); abort = true; } + if(option == "help") { help(); abort = true; calledHelp = true; } else { //valid paramters for this command @@ -44,6 +95,13 @@ ClusterSplitCommand::ClusterSplitCommand(string option) { } } + //initialize outputTypes + vector tempOutNames; + outputTypes["list"] = tempOutNames; + outputTypes["rabund"] = tempOutNames; + outputTypes["sabund"] = tempOutNames; + outputTypes["column"] = tempOutNames; + globaldata->newRead(); //if the user changes the output directory command factory will send this info to us in the output parameter @@ -57,7 +115,7 @@ ClusterSplitCommand::ClusterSplitCommand(string option) { it = parameters.find("phylip"); //user has given a template file if(it != parameters.end()){ - path = hasPath(it->second); + path = m->hasPath(it->second); //if the user has not given a path then, add inputdir. else leave path alone. if (path == "") { parameters["phylip"] = inputDir + it->second; } } @@ -65,7 +123,7 @@ ClusterSplitCommand::ClusterSplitCommand(string option) { it = parameters.find("column"); //user has given a template file if(it != parameters.end()){ - path = hasPath(it->second); + path = m->hasPath(it->second); //if the user has not given a path then, add inputdir. else leave path alone. if (path == "") { parameters["column"] = inputDir + it->second; } } @@ -73,7 +131,7 @@ ClusterSplitCommand::ClusterSplitCommand(string option) { it = parameters.find("name"); //user has given a template file if(it != parameters.end()){ - path = hasPath(it->second); + path = m->hasPath(it->second); //if the user has not given a path then, add inputdir. else leave path alone. if (path == "") { parameters["name"] = inputDir + it->second; } } @@ -81,7 +139,7 @@ ClusterSplitCommand::ClusterSplitCommand(string option) { it = parameters.find("taxonomy"); //user has given a template file if(it != parameters.end()){ - path = hasPath(it->second); + path = m->hasPath(it->second); //if the user has not given a path then, add inputdir. else leave path alone. if (path == "") { parameters["taxonomy"] = inputDir + it->second; } } @@ -89,7 +147,7 @@ ClusterSplitCommand::ClusterSplitCommand(string option) { it = parameters.find("fasta"); //user has given a template file if(it != parameters.end()){ - path = hasPath(it->second); + path = m->hasPath(it->second); //if the user has not given a path then, add inputdir. else leave path alone. if (path == "") { parameters["fasta"] = inputDir + it->second; } } @@ -142,10 +200,10 @@ ClusterSplitCommand::ClusterSplitCommand(string option) { convert(temp, precision); temp = validParameter.validFile(parameters, "hard", false); if (temp == "not found") { temp = "F"; } - hard = isTrue(temp); + hard = m->isTrue(temp); temp = validParameter.validFile(parameters, "large", false); if (temp == "not found") { temp = "F"; } - large = isTrue(temp); + large = m->isTrue(temp); temp = validParameter.validFile(parameters, "processors", false); if (temp == "not found"){ temp = "1"; } convert(temp, processors); @@ -195,7 +253,7 @@ void ClusterSplitCommand::help(){ m->mothurOut("The cluster.split command can split your files in 3 ways. Splitting by distance file, by classification, or by classification also using a fasta file. \n"); m->mothurOut("For the distance file method, you need only provide your distance file and mothur will split the file into distinct groups. \n"); m->mothurOut("For the classification method, you need to provide your distance file and taxonomy file, and set the splitmethod to classify. \n"); - m->mothurOut("You will also need to set the taxlevel you want to split by. mothur will split the sequence into distinct taxonomy groups, and split the distance file based on those groups. \n"); + m->mothurOut("You will also need to set the taxlevel you want to split by. mothur will split the sequences into distinct taxonomy groups, and split the distance file based on those groups. \n"); m->mothurOut("For the classification method using a fasta file, you need to provide your fasta file, names file and taxonomy file. \n"); m->mothurOut("You will also need to set the taxlevel you want to split by. mothur will split the sequence into distinct taxonomy groups, and create distance files for each grouping. \n"); m->mothurOut("The phylip and column parameter allow you to enter your distance file. \n"); @@ -206,7 +264,7 @@ void ClusterSplitCommand::help(){ m->mothurOut("The method allows you to specify what clustering algorythm you want to use, default=furthest, option furthest, nearest, or average. \n"); m->mothurOut("The splitmethod parameter allows you to specify how you want to split your distance file before you cluster, default=distance, options distance, classify or fasta. \n"); m->mothurOut("The taxonomy parameter allows you to enter the taxonomy file for your sequences, this is only valid if you are using splitmethod=classify. Be sure your taxonomy file does not include the probability scores. \n"); - m->mothurOut("The taxlevel parameter allows you to specify the taxonomy level you want to use to split the distance file, default=1. \n"); + m->mothurOut("The taxlevel parameter allows you to specify the taxonomy level you want to use to split the distance file, default=1, meaning use the first taxon in each list. \n"); m->mothurOut("The large parameter allows you to indicate that your distance matrix is too large to fit in RAM. The default value is false.\n"); #ifdef USE_MPI m->mothurOut("When using MPI, the processors parameter is set to the number of MPI processes running. \n"); @@ -231,7 +289,7 @@ ClusterSplitCommand::~ClusterSplitCommand(){} int ClusterSplitCommand::execute(){ try { - if (abort == true) { return 0; } + if (abort == true) { if (calledHelp) { return 0; } return 2; } time_t estart; vector listFileNames; @@ -271,7 +329,7 @@ int ClusterSplitCommand::execute(){ if (namefile == "") { //you need to make a namefile for split matrix ofstream out; namefile = phylipfile + ".names"; - openOutputFile(namefile, out); + m->openOutputFile(namefile, out); for (int i = 0; i < listToMakeNameFile->getNumBins(); i++) { string bin = listToMakeNameFile->get(i); out << bin << '\t' << bin << endl; @@ -292,7 +350,7 @@ int ClusterSplitCommand::execute(){ SplitMatrix* split; if (splitmethod == "distance") { split = new SplitMatrix(distfile, namefile, taxFile, cutoff, splitmethod, large); } else if (splitmethod == "classify") { split = new SplitMatrix(distfile, namefile, taxFile, taxLevelCutoff, splitmethod, large); } - else if (splitmethod == "fasta") { split = new SplitMatrix(fastafile, namefile, taxFile, taxLevelCutoff, splitmethod, processors, outputDir); } + else if (splitmethod == "fasta") { split = new SplitMatrix(fastafile, namefile, taxFile, taxLevelCutoff, cutoff, splitmethod, processors, outputDir); } else { m->mothurOut("Not a valid splitting method. Valid splitting algorithms are distance, classify or fasta."); m->mothurOutEndLine(); return 0; } split->split(); @@ -303,6 +361,10 @@ int ClusterSplitCommand::execute(){ vector< map > distName = split->getDistanceFiles(); //returns map of distance files -> namefile sorted by distance file size delete split; + //output a merged distance file + if (splitmethod == "fasta") { createMergedDistanceFile(distName); } + + if (m->control_pressed) { return 0; } m->mothurOut("It took " + toString(time(NULL) - estart) + " seconds to split the distance file."); m->mothurOutEndLine(); @@ -317,7 +379,6 @@ int ClusterSplitCommand::execute(){ //for each file group figure out which process will complete it //want to divide the load intelligently so the big files are spread between processes - int count = 1; for (int i = 0; i < distName.size(); i++) { int processToAssign = (i+1) % processors; if (processToAssign == 0) { processToAssign = processors; } @@ -475,7 +536,6 @@ int ClusterSplitCommand::execute(){ //for each file group figure out which process will complete it //want to divide the load intelligently so the big files are spread between processes - int count = 1; for (int i = 0; i < distName.size(); i++) { int processToAssign = (i+1) % processors; if (processToAssign == 0) { processToAssign = processors; } @@ -497,13 +557,13 @@ int ClusterSplitCommand::execute(){ for(int i=0;iopenInputFile(filename, in); - in >> tag; gobble(in); + in >> tag; m->gobble(in); while(!in.eof()) { string tempName; - in >> tempName; gobble(in); + in >> tempName; m->gobble(in); listFileNames.push_back(tempName); } in.close(); @@ -512,15 +572,15 @@ int ClusterSplitCommand::execute(){ //get labels filename = toString(processIDS[i]) + ".temp.labels"; ifstream in2; - openInputFile(filename, in2); + m->openInputFile(filename, in2); float tempCutoff; - in2 >> tempCutoff; gobble(in2); + in2 >> tempCutoff; m->gobble(in2); if (tempCutoff < cutoff) { cutoff = tempCutoff; } while(!in2.eof()) { string tempName; - in2 >> tempName; gobble(in2); + in2 >> tempName; m->gobble(in2); if (labels.count(tempName) == 0) { labels.insert(tempName); } } in2.close(); @@ -556,6 +616,25 @@ int ClusterSplitCommand::execute(){ m->mothurOut("It took " + toString(time(NULL) - estart) + " seconds to merge."); m->mothurOutEndLine(); + //set list file as new current listfile + string current = ""; + itTypes = outputTypes.find("list"); + if (itTypes != outputTypes.end()) { + if ((itTypes->second).size() != 0) { current = (itTypes->second)[0]; m->setListFile(current); } + } + + //set rabund file as new current rabundfile + itTypes = outputTypes.find("rabund"); + if (itTypes != outputTypes.end()) { + if ((itTypes->second).size() != 0) { current = (itTypes->second)[0]; m->setRabundFile(current); } + } + + //set sabund file as new current sabundfile + itTypes = outputTypes.find("sabund"); + if (itTypes != outputTypes.end()) { + if ((itTypes->second).size() != 0) { current = (itTypes->second)[0]; m->setSabundFile(current); } + } + m->mothurOutEndLine(); m->mothurOut("Output File Names: "); m->mothurOutEndLine(); for (int i = 0; i < outputNames.size(); i++) { m->mothurOut(outputNames[i]); m->mothurOutEndLine(); } @@ -586,12 +665,12 @@ map ClusterSplitCommand::completeListFile(vector listNames, //read in singletons if (singleton != "none") { ifstream in; - openInputFile(singleton, in); + m->openInputFile(singleton, in); string firstCol, secondCol; listSingle = new ListVector(); while (!in.eof()) { - in >> firstCol >> secondCol; gobble(in); + in >> firstCol >> secondCol; m->gobble(in); listSingle->push_back(secondCol); } in.close(); @@ -632,7 +711,7 @@ map ClusterSplitCommand::completeListFile(vector listNames, string filledInList = listNames[k] + "filledInTemp"; ofstream outFilled; - openOutputFile(filledInList, outFilled); + m->openOutputFile(filledInList, outFilled); //for each label needed for(int l = 0; l < orderFloat.size(); l++){ @@ -689,16 +768,16 @@ map ClusterSplitCommand::completeListFile(vector listNames, //********************************************************************************************************************** int ClusterSplitCommand::mergeLists(vector listNames, map userLabels, ListVector* listSingle){ try { - if (outputDir == "") { outputDir += hasPath(distfile); } - fileroot = outputDir + getRootName(getSimpleName(distfile)); + if (outputDir == "") { outputDir += m->hasPath(distfile); } + fileroot = outputDir + m->getRootName(m->getSimpleName(distfile)); - openOutputFile(fileroot+ tag + ".sabund", outSabund); - openOutputFile(fileroot+ tag + ".rabund", outRabund); - openOutputFile(fileroot+ tag + ".list", outList); + m->openOutputFile(fileroot+ tag + ".sabund", outSabund); + m->openOutputFile(fileroot+ tag + ".rabund", outRabund); + m->openOutputFile(fileroot+ tag + ".list", outList); - outputNames.push_back(fileroot+ tag + ".sabund"); - outputNames.push_back(fileroot+ tag + ".rabund"); - outputNames.push_back(fileroot+ tag + ".list"); + outputNames.push_back(fileroot+ tag + ".sabund"); outputTypes["list"].push_back(fileroot+ tag + ".list"); + outputNames.push_back(fileroot+ tag + ".rabund"); outputTypes["rabund"].push_back(fileroot+ tag + ".rabund"); + outputNames.push_back(fileroot+ tag + ".list"); outputTypes["sabund"].push_back(fileroot+ tag + ".sabund"); map::iterator itLabel; @@ -718,7 +797,7 @@ int ClusterSplitCommand::mergeLists(vector listNames, map us if (listSingle != NULL) { for (int j = 0; j < listSingle->getNumBins(); j++) { outList << listSingle->get(j) << '\t'; - rabund->push_back(getNumNames(listSingle->get(j))); + rabund->push_back(m->getNumNames(listSingle->get(j))); } } @@ -735,7 +814,7 @@ int ClusterSplitCommand::mergeLists(vector listNames, map us else { for (int j = 0; j < list->getNumBins(); j++) { outList << list->get(j) << '\t'; - rabund->push_back(getNumNames(list->get(j))); + rabund->push_back(m->getNumNames(list->get(j))); } delete list; } @@ -775,7 +854,7 @@ void ClusterSplitCommand::printData(ListVector* oldList){ RAbundVector oldRAbund = oldList->getRAbundVector(); oldRAbund.setLabel(label); - if (isTrue(showabund)) { + if (m->isTrue(showabund)) { oldRAbund.getSAbundVector().print(cout); } oldRAbund.print(outRabund); @@ -811,7 +890,7 @@ int ClusterSplitCommand::createProcesses(vector < vector < map > //write out names to file string filename = toString(getpid()) + ".temp"; ofstream out; - openOutputFile(filename, out); + m->openOutputFile(filename, out); out << tag << endl; for (int j = 0; j < listFileNames.size(); j++) { out << listFileNames[j] << endl; } out.close(); @@ -819,7 +898,7 @@ int ClusterSplitCommand::createProcesses(vector < vector < map > //print out labels ofstream outLabels; filename = toString(getpid()) + ".temp.labels"; - openOutputFile(filename, outLabels); + m->openOutputFile(filename, outLabels); outLabels << cutoff << endl; for (set::iterator it = labels.begin(); it != labels.end(); it++) { @@ -828,7 +907,11 @@ int ClusterSplitCommand::createProcesses(vector < vector < map > outLabels.close(); exit(0); - }else { m->mothurOut("unable to spawn the necessary processes."); m->mothurOutEndLine(); exit(0); } + }else { + m->mothurOut("[ERROR]: unable to spawn the necessary processes."); m->mothurOutEndLine(); + for (int i = 0; i < processIDS.size(); i++) { kill (processIDS[i], SIGINT); } + exit(0); + } } //force parent to wait until all the processes are done @@ -917,16 +1000,14 @@ vector ClusterSplitCommand::cluster(vector< map > distNa else if(method == "average"){ cluster = new AverageLinkage(rabund, list, matrix, cutoff, method); } tag = cluster->getTag(); - if (outputDir == "") { outputDir += hasPath(thisDistFile); } - fileroot = outputDir + getRootName(getSimpleName(thisDistFile)); + if (outputDir == "") { outputDir += m->hasPath(thisDistFile); } + fileroot = outputDir + m->getRootName(m->getSimpleName(thisDistFile)); ofstream listFile; - openOutputFile(fileroot+ tag + ".list", listFile); + m->openOutputFile(fileroot+ tag + ".list", listFile); listFileNames.push_back(fileroot+ tag + ".list"); - - time_t estart = time(NULL); - + float previousDist = 0.00000; float rndPreviousDist = 0.00000; @@ -950,9 +1031,9 @@ vector ClusterSplitCommand::cluster(vector< map > distNa float dist = matrix->getSmallDist(); float rndDist; if (hard) { - rndDist = ceilDist(dist, precision); + rndDist = m->ceilDist(dist, precision); }else{ - rndDist = roundDist(dist, precision); + rndDist = m->roundDist(dist, precision); } if(previousDist <= 0.0000 && dist != previousDist){ @@ -995,8 +1076,8 @@ vector ClusterSplitCommand::cluster(vector< map > distNa remove(thisNamefile.c_str()); if (saveCutoff != cutoff) { - if (hard) { saveCutoff = ceilDist(saveCutoff, precision); } - else { saveCutoff = roundDist(saveCutoff, precision); } + if (hard) { saveCutoff = m->ceilDist(saveCutoff, precision); } + else { saveCutoff = m->roundDist(saveCutoff, precision); } m->mothurOut("Cutoff was " + toString(cutoff) + " changed cutoff to " + toString(saveCutoff)); m->mothurOutEndLine(); } @@ -1016,5 +1097,45 @@ vector ClusterSplitCommand::cluster(vector< map > distNa } +//********************************************************************************************************************** +int ClusterSplitCommand::createMergedDistanceFile(vector< map > distNames) { + try{ + +#ifdef USE_MPI + int pid; + MPI_Comm_rank(MPI_COMM_WORLD, &pid); //find out who we are + + if (pid != 0) { +#endif + + string thisOutputDir = outputDir; + if (outputDir == "") { thisOutputDir = m->hasPath(fastafile); } + string outputFileName = thisOutputDir + m->getRootName(m->getSimpleName(fastafile)) + "dist"; + remove(outputFileName.c_str()); + + + for (int i = 0; i < distNames.size(); i++) { + if (m->control_pressed) { return 0; } + + string thisDistFile = distNames[i].begin()->first; + + m->appendFiles(thisDistFile, outputFileName); + } + + outputTypes["column"].push_back(outputFileName); outputNames.push_back(outputFileName); + +#ifdef USE_MPI + } +#endif + + return 0; + + + } + catch(exception& e) { + m->errorOut(e, "ClusterSplitCommand", "createMergedDistanceFile"); + exit(1); + } +} //**********************************************************************************************************************