5 * Created by Sarah Westcott on 1/2/09.
6 * Copyright 2009 Schloss Lab UMASS Amherst. All rights reserved.
10 #include "clustercommand.h"
11 #include "readphylip.h"
12 #include "readcolumn.h"
13 #include "readmatrix.hpp"
14 #include "clusterdoturcommand.h"
17 //**********************************************************************************************************************
18 vector<string> ClusterCommand::setParameters(){
20 CommandParameter pphylip("phylip", "InputTypes", "", "", "PhylipColumn", "PhylipColumn", "none","list",false,false,true); parameters.push_back(pphylip);
21 CommandParameter pname("name", "InputTypes", "", "", "NameCount", "none", "ColumnName","rabund-sabund",false,false,true); parameters.push_back(pname);
22 CommandParameter pcount("count", "InputTypes", "", "", "NameCount", "none", "none","",false,false,true); parameters.push_back(pcount);
23 CommandParameter pcolumn("column", "InputTypes", "", "", "PhylipColumn", "PhylipColumn", "ColumnName","list",false,false,true); parameters.push_back(pcolumn);
24 CommandParameter pcutoff("cutoff", "Number", "", "10", "", "", "","",false,false,true); parameters.push_back(pcutoff);
25 CommandParameter pprecision("precision", "Number", "", "100", "", "", "","",false,false); parameters.push_back(pprecision);
26 CommandParameter pmethod("method", "Multiple", "furthest-nearest-average-weighted", "average", "", "", "","",false,false,true); parameters.push_back(pmethod);
27 CommandParameter pshowabund("showabund", "Boolean", "", "T", "", "", "","",false,false); parameters.push_back(pshowabund);
28 CommandParameter ptiming("timing", "Boolean", "", "F", "", "", "","",false,false); parameters.push_back(ptiming);
29 CommandParameter psim("sim", "Boolean", "", "F", "", "", "","",false,false); parameters.push_back(psim);
30 CommandParameter phard("hard", "Boolean", "", "T", "", "", "","",false,false); parameters.push_back(phard);
31 CommandParameter pinputdir("inputdir", "String", "", "", "", "", "","",false,false); parameters.push_back(pinputdir);
32 //CommandParameter padjust("adjust", "String", "", "F", "", "", "","",false,false); parameters.push_back(padjust);
33 CommandParameter poutputdir("outputdir", "String", "", "", "", "", "","",false,false); parameters.push_back(poutputdir);
35 vector<string> myArray;
36 for (int i = 0; i < parameters.size(); i++) { myArray.push_back(parameters[i].name); }
40 m->errorOut(e, "ClusterCommand", "setParameters");
44 //**********************************************************************************************************************
45 string ClusterCommand::getHelpString(){
47 string helpString = "";
48 helpString += "The cluster command parameter options are phylip, column, name, count, method, cuttoff, hard, precision, sim, showabund and timing. Phylip or column and name are required, unless you have a valid current file.\n";
49 //helpString += "The adjust parameter is used to handle missing distances. If you set a cutoff, adjust=f by default. If not, adjust=t by default. Adjust=f, means ignore missing distances and adjust cutoff as needed with the average neighbor method. Adjust=t, will treat missing distances as 1.0. You can also set the value the missing distances should be set to, adjust=0.5 would give missing distances a value of 0.5.\n";
50 helpString += "The cluster command should be in the following format: \n";
51 helpString += "cluster(method=yourMethod, cutoff=yourCutoff, precision=yourPrecision) \n";
52 helpString += "The acceptable cluster methods are furthest, nearest, average and weighted. If no method is provided then average is assumed.\n";
56 m->errorOut(e, "ClusterCommand", "getHelpString");
60 //**********************************************************************************************************************
61 string ClusterCommand::getOutputPattern(string type) {
65 if (type == "list") { pattern = "[filename],[clustertag],list-[filename],[clustertag],[tag2],list"; }
66 else if (type == "rabund") { pattern = "[filename],[clustertag],rabund"; }
67 else if (type == "sabund") { pattern = "[filename],[clustertag],sabund"; }
68 else { m->mothurOut("[ERROR]: No definition for type " + type + " output pattern.\n"); m->control_pressed = true; }
73 m->errorOut(e, "ClusterCommand", "getOutputPattern");
77 //**********************************************************************************************************************
78 ClusterCommand::ClusterCommand(){
80 abort = true; calledHelp = true;
82 vector<string> tempOutNames;
83 outputTypes["list"] = tempOutNames;
84 outputTypes["rabund"] = tempOutNames;
85 outputTypes["sabund"] = tempOutNames;
88 m->errorOut(e, "ClusterCommand", "ClusterCommand");
92 //**********************************************************************************************************************
93 //This function checks to make sure the cluster command has no errors and then clusters based on the method chosen.
94 ClusterCommand::ClusterCommand(string option) {
96 abort = false; calledHelp = false;
98 //allow user to run help
99 if(option == "help") { help(); abort = true; calledHelp = true; }
100 else if(option == "citation") { citation(); abort = true; calledHelp = true;}
103 vector<string> myArray = setParameters();
105 OptionParser parser(option);
106 map<string,string> parameters = parser.getParameters();
107 map<string,string>::iterator it;
109 ValidParameters validParameter;
111 //check to make sure all parameters are valid for command
112 for (it = parameters.begin(); it != parameters.end(); it++) {
113 if (validParameter.isValidParameter(it->first, myArray, it->second) != true) {
118 //initialize outputTypes
119 vector<string> tempOutNames;
120 outputTypes["list"] = tempOutNames;
121 outputTypes["rabund"] = tempOutNames;
122 outputTypes["sabund"] = tempOutNames;
124 //if the user changes the output directory command factory will send this info to us in the output parameter
125 outputDir = validParameter.validFile(parameters, "outputdir", false); if (outputDir == "not found"){ outputDir = ""; }
127 string inputDir = validParameter.validFile(parameters, "inputdir", false);
128 if (inputDir == "not found"){ inputDir = ""; }
131 it = parameters.find("phylip");
132 //user has given a template file
133 if(it != parameters.end()){
134 path = m->hasPath(it->second);
135 //if the user has not given a path then, add inputdir. else leave path alone.
136 if (path == "") { parameters["phylip"] = inputDir + it->second; }
139 it = parameters.find("column");
140 //user has given a template file
141 if(it != parameters.end()){
142 path = m->hasPath(it->second);
143 //if the user has not given a path then, add inputdir. else leave path alone.
144 if (path == "") { parameters["column"] = inputDir + it->second; }
147 it = parameters.find("name");
148 //user has given a template file
149 if(it != parameters.end()){
150 path = m->hasPath(it->second);
151 //if the user has not given a path then, add inputdir. else leave path alone.
152 if (path == "") { parameters["name"] = inputDir + it->second; }
155 it = parameters.find("count");
156 //user has given a template file
157 if(it != parameters.end()){
158 path = m->hasPath(it->second);
159 //if the user has not given a path then, add inputdir. else leave path alone.
160 if (path == "") { parameters["count"] = inputDir + it->second; }
164 //check for required parameters
165 phylipfile = validParameter.validFile(parameters, "phylip", true);
166 if (phylipfile == "not open") { phylipfile = ""; abort = true; }
167 else if (phylipfile == "not found") { phylipfile = ""; }
168 else { distfile = phylipfile; format = "phylip"; m->setPhylipFile(phylipfile); }
170 columnfile = validParameter.validFile(parameters, "column", true);
171 if (columnfile == "not open") { columnfile = ""; abort = true; }
172 else if (columnfile == "not found") { columnfile = ""; }
173 else { distfile = columnfile; format = "column"; m->setColumnFile(columnfile); }
175 namefile = validParameter.validFile(parameters, "name", true);
176 if (namefile == "not open") { abort = true; }
177 else if (namefile == "not found") { namefile = ""; }
178 else { m->setNameFile(namefile); }
180 countfile = validParameter.validFile(parameters, "count", true);
181 if (countfile == "not open") { abort = true; countfile = ""; }
182 else if (countfile == "not found") { countfile = ""; }
183 else { m->setCountTableFile(countfile); }
185 if ((phylipfile == "") && (columnfile == "")) {
186 //is there are current file available for either of these?
187 //give priority to column, then phylip
188 columnfile = m->getColumnFile();
189 if (columnfile != "") { distfile = columnfile; format = "column"; m->mothurOut("Using " + columnfile + " as input file for the column parameter."); m->mothurOutEndLine(); }
191 phylipfile = m->getPhylipFile();
192 if (phylipfile != "") { distfile = phylipfile; format = "phylip"; m->mothurOut("Using " + phylipfile + " as input file for the phylip parameter."); m->mothurOutEndLine(); }
194 m->mothurOut("No valid current files. You must provide a phylip or column file before you can use the cluster command."); m->mothurOutEndLine();
199 else if ((phylipfile != "") && (columnfile != "")) { m->mothurOut("When executing a cluster command you must enter ONLY ONE of the following: phylip or column."); m->mothurOutEndLine(); abort = true; }
201 if (columnfile != "") {
202 if ((namefile == "") && (countfile == "")){
203 namefile = m->getNameFile();
204 if (namefile != "") { m->mothurOut("Using " + namefile + " as input file for the name parameter."); m->mothurOutEndLine(); }
206 countfile = m->getCountTableFile();
207 if (countfile != "") { m->mothurOut("Using " + countfile + " as input file for the count parameter."); m->mothurOutEndLine(); }
209 m->mothurOut("You need to provide a namefile or countfile if you are going to use the column format."); m->mothurOutEndLine();
216 if ((countfile != "") && (namefile != "")) { m->mothurOut("When executing a cluster command you must enter ONLY ONE of the following: count or name."); m->mothurOutEndLine(); abort = true; }
218 //check for optional parameter and set defaults
219 // ...at some point should added some additional type checking...
220 //get user cutoff and precision or use defaults
222 temp = validParameter.validFile(parameters, "precision", false);
223 if (temp == "not found") { temp = "100"; }
224 //saves precision legnth for formatting below
225 length = temp.length();
226 m->mothurConvert(temp, precision);
228 temp = validParameter.validFile(parameters, "hard", false); if (temp == "not found") { temp = "T"; }
229 hard = m->isTrue(temp);
231 temp = validParameter.validFile(parameters, "sim", false); if (temp == "not found") { temp = "F"; }
232 sim = m->isTrue(temp);
234 //bool cutoffSet = false;
235 temp = validParameter.validFile(parameters, "cutoff", false);
236 if (temp == "not found") { temp = "10"; }
237 //else { cutoffSet = true; }
238 m->mothurConvert(temp, cutoff);
239 cutoff += (5 / (precision * 10.0));
241 //temp = validParameter.validFile(parameters, "adjust", false); if (temp == "not found") { temp = "F"; }
242 //if (m->isNumeric1(temp)) { m->mothurConvert(temp, adjust); }
243 //else if (m->isTrue(temp)) { adjust = 1.0; }
244 //else { adjust = -1.0; }
247 method = validParameter.validFile(parameters, "method", false);
248 if (method == "not found") { method = "average"; }
250 if ((method == "furthest") || (method == "nearest") || (method == "average") || (method == "weighted")) { }
251 else { m->mothurOut("Not a valid clustering method. Valid clustering algorithms are furthest, nearest, average, and weighted."); m->mothurOutEndLine(); abort = true; }
253 showabund = validParameter.validFile(parameters, "showabund", false);
254 if (showabund == "not found") { showabund = "T"; }
256 timing = validParameter.validFile(parameters, "timing", false);
257 if (timing == "not found") { timing = "F"; }
261 catch(exception& e) {
262 m->errorOut(e, "ClusterCommand", "ClusterCommand");
266 //**********************************************************************************************************************
267 ClusterCommand::~ClusterCommand(){}
268 //**********************************************************************************************************************
270 int ClusterCommand::execute(){
273 if (abort == true) { if (calledHelp) { return 0; } return 2; }
275 //phylip file given and cutoff not given - use cluster.classic because it uses less memory and is faster
276 if ((format == "phylip") && (cutoff > 10.0)) {
277 m->mothurOutEndLine(); m->mothurOut("You are using a phylip file and no cutoff. I will run cluster.classic to save memory and time."); m->mothurOutEndLine();
279 //run unique.seqs for deconvolute results
280 string inputString = "phylip=" + distfile;
281 if (namefile != "") { inputString += ", name=" + namefile; }
282 else if (countfile != "") { inputString += ", count=" + countfile; }
283 inputString += ", precision=" + toString(precision);
284 inputString += ", method=" + method;
285 if (hard) { inputString += ", hard=T"; }
286 else { inputString += ", hard=F"; }
287 if (sim) { inputString += ", sim=T"; }
288 else { inputString += ", sim=F"; }
291 m->mothurOutEndLine();
292 m->mothurOut("/------------------------------------------------------------/"); m->mothurOutEndLine();
293 m->mothurOut("Running command: cluster.classic(" + inputString + ")"); m->mothurOutEndLine();
295 Command* clusterClassicCommand = new ClusterDoturCommand(inputString);
296 clusterClassicCommand->execute();
297 delete clusterClassicCommand;
299 m->mothurOut("/------------------------------------------------------------/"); m->mothurOutEndLine();
305 if (format == "column") { read = new ReadColumnMatrix(columnfile, sim); } //sim indicates whether its a similarity matrix
306 else if (format == "phylip") { read = new ReadPhylipMatrix(phylipfile, sim); }
308 read->setCutoff(cutoff);
310 NameAssignment* nameMap = NULL;
311 CountTable* ct = NULL;
313 nameMap = new NameAssignment(namefile);
316 }else if (countfile != "") {
317 ct = new CountTable();
318 ct->readTable(countfile, false);
320 }else { read->read(nameMap); }
322 list = read->getListVector();
323 matrix = read->getDMatrix();
325 if(countfile != "") {
326 rabund = new RAbundVector();
327 createRabund(ct, list, rabund); //creates an rabund that includes the counts for the unique list
329 }else { rabund = new RAbundVector(list->getRAbundVector()); }
332 if (m->control_pressed) { //clean up
333 delete list; delete matrix; delete rabund; if(countfile == ""){rabundFile.close(); sabundFile.close(); m->mothurRemove((fileroot+ tag + ".rabund")); m->mothurRemove((fileroot+ tag + ".sabund")); }
334 listFile.close(); m->mothurRemove((fileroot+ tag + ".list")); outputTypes.clear(); return 0;
338 if (method == "furthest") { cluster = new CompleteLinkage(rabund, list, matrix, cutoff, method, adjust); }
339 else if(method == "nearest"){ cluster = new SingleLinkage(rabund, list, matrix, cutoff, method, adjust); }
340 else if(method == "average"){ cluster = new AverageLinkage(rabund, list, matrix, cutoff, method, adjust); }
341 else if(method == "weighted"){ cluster = new WeightedLinkage(rabund, list, matrix, cutoff, method, adjust); }
342 tag = cluster->getTag();
344 if (outputDir == "") { outputDir += m->hasPath(distfile); }
345 fileroot = outputDir + m->getRootName(m->getSimpleName(distfile));
347 map<string, string> variables;
348 variables["[filename]"] = fileroot;
349 variables["[clustertag]"] = tag;
350 string sabundFileName = getOutputFileName("sabund", variables);
351 string rabundFileName = getOutputFileName("rabund", variables);
352 if (countfile != "") { variables["[tag2]"] = "unique_list"; }
353 string listFileName = getOutputFileName("list", variables);
355 if (countfile == "") {
356 m->openOutputFile(sabundFileName, sabundFile);
357 m->openOutputFile(rabundFileName, rabundFile);
358 outputNames.push_back(sabundFileName); outputTypes["sabund"].push_back(sabundFileName);
359 outputNames.push_back(rabundFileName); outputTypes["rabund"].push_back(rabundFileName);
362 m->openOutputFile(listFileName, listFile);
363 outputNames.push_back(listFileName); outputTypes["list"].push_back(listFileName);
366 time_t estart = time(NULL);
367 float previousDist = 0.00000;
368 float rndPreviousDist = 0.00000;
375 double saveCutoff = cutoff;
377 while (matrix->getSmallDist() < cutoff && matrix->getNNodes() > 0){
379 if (m->control_pressed) { //clean up
380 delete list; delete matrix; delete rabund; delete cluster;
381 if(countfile == "") {rabundFile.close(); sabundFile.close(); m->mothurRemove((fileroot+ tag + ".rabund")); m->mothurRemove((fileroot+ tag + ".sabund")); }
382 listFile.close(); m->mothurRemove((fileroot+ tag + ".list")); outputTypes.clear(); return 0;
385 if (print_start && m->isTrue(timing)) {
386 m->mothurOut("Clustering (" + tag + ") dist " + toString(matrix->getSmallDist()) + "/"
387 + toString(m->roundDist(matrix->getSmallDist(), precision))
388 + "\t(precision: " + toString(precision) + ", Nodes: " + toString(matrix->getNNodes()) + ")");
395 cluster->update(cutoff);
397 float dist = matrix->getSmallDist();
400 rndDist = m->ceilDist(dist, precision);
402 rndDist = m->roundDist(dist, precision);
405 if(previousDist <= 0.0000 && dist != previousDist){
408 else if(rndDist != rndPreviousDist){
409 printData(toString(rndPreviousDist, length-1));
413 rndPreviousDist = rndDist;
418 if (print_start && m->isTrue(timing)) {
419 m->mothurOut("Clustering (" + tag + ") for distance " + toString(previousDist) + "/" + toString(rndPreviousDist)
420 + "\t(precision: " + toString(precision) + ", Nodes: " + toString(matrix->getNNodes()) + ")");
425 if(previousDist <= 0.0000){
428 else if(rndPreviousDist<cutoff){
429 printData(toString(rndPreviousDist, length-1));
436 if (countfile == "") {
442 if (saveCutoff != cutoff) {
443 if (hard) { saveCutoff = m->ceilDist(saveCutoff, precision); }
444 else { saveCutoff = m->roundDist(saveCutoff, precision); }
446 m->mothurOut("changed cutoff to " + toString(cutoff)); m->mothurOutEndLine();
449 //set list file as new current listfile
451 itTypes = outputTypes.find("list");
452 if (itTypes != outputTypes.end()) {
453 if ((itTypes->second).size() != 0) { current = (itTypes->second)[0]; m->setListFile(current); }
456 //set rabund file as new current rabundfile
457 itTypes = outputTypes.find("rabund");
458 if (itTypes != outputTypes.end()) {
459 if ((itTypes->second).size() != 0) { current = (itTypes->second)[0]; m->setRabundFile(current); }
462 //set sabund file as new current sabundfile
463 itTypes = outputTypes.find("sabund");
464 if (itTypes != outputTypes.end()) {
465 if ((itTypes->second).size() != 0) { current = (itTypes->second)[0]; m->setSabundFile(current); }
468 m->mothurOutEndLine();
469 m->mothurOut("Output File Names: "); m->mothurOutEndLine();
470 for (int i = 0; i < outputNames.size(); i++) { m->mothurOut(outputNames[i]); m->mothurOutEndLine(); }
471 m->mothurOutEndLine();
474 //if (m->isTrue(timing)) {
475 m->mothurOut("It took " + toString(time(NULL) - estart) + " seconds to cluster"); m->mothurOutEndLine();
481 catch(exception& e) {
482 m->errorOut(e, "ClusterCommand", "execute");
487 //**********************************************************************************************************************
489 void ClusterCommand::printData(string label){
491 if (m->isTrue(timing)) {
492 m->mothurOut("\tTime: " + toString(time(NULL) - start) + "\tsecs for " + toString(oldRAbund.getNumBins())
493 + "\tclusters. Updates: " + toString(loops)); m->mothurOutEndLine();
499 oldRAbund.setLabel(label);
500 if (countfile == "") {
501 oldRAbund.print(rabundFile);
502 oldRAbund.getSAbundVector().print(sabundFile);
505 if (m->isTrue(showabund)) {
506 oldRAbund.getSAbundVector().print(cout);
509 oldList.setLabel(label);
510 oldList.print(listFile);
512 catch(exception& e) {
513 m->errorOut(e, "ClusterCommand", "printData");
519 //**********************************************************************************************************************
521 int ClusterCommand::createRabund(CountTable*& ct, ListVector*& list, RAbundVector*& rabund){
523 rabund->setLabel(list->getLabel());
524 for(int i = 0; i < list->getNumBins(); i++) {
525 if (m->control_pressed) { break; }
526 vector<string> binNames;
527 string bin = list->get(i);
528 m->splitAtComma(bin, binNames);
530 for (int j = 0; j < binNames.size(); j++) { total += ct->getNumSeqs(binNames[j]); }
531 rabund->push_back(total);
535 catch(exception& e) {
536 m->errorOut(e, "ClusterCommand", "createRabund");
541 //**********************************************************************************************************************