matrixoutputcommand.cpp

   1 /*
   2  *  matrixoutputcommand.cpp
   3  *  Mothur
   4  *
   5  *  Created by Sarah Westcott on 5/20/09.
   6  *  Copyright 2009 Schloss Lab UMASS Amherst. All rights reserved.
   7  *
   8  */
   9
  10 #include "matrixoutputcommand.h"
  11 #include "sharedsobscollectsummary.h"
  12 #include "sharedchao1.h"
  13 #include "sharedace.h"
  14 #include "sharednseqs.h"
  15 #include "sharedjabund.h"
  16 #include "sharedsorabund.h"
  17 #include "sharedjclass.h"
  18 #include "sharedsorclass.h"
  19 #include "sharedjest.h"
  20 #include "sharedsorest.h"
  21 #include "sharedthetayc.h"
  22 #include "sharedthetan.h"
  23 #include "sharedkstest.h"
  24 #include "whittaker.h"
  25 #include "sharedochiai.h"
  26 #include "sharedanderbergs.h"
  27 #include "sharedkulczynski.h"
  28 #include "sharedkulczynskicody.h"
  29 #include "sharedlennon.h"
  30 #include "sharedmorisitahorn.h"
  31 #include "sharedbraycurtis.h"
  32 #include "sharedjackknife.h"
  33 #include "whittaker.h"
  34 #include "odum.h"
  35 #include "canberra.h"
  36 #include "structeuclidean.h"
  37 #include "structchord.h"
  38 #include "hellinger.h"
  39 #include "manhattan.h"
  40 #include "structpearson.h"
  41 #include "soergel.h"
  42 #include "spearman.h"
  43 #include "structkulczynski.h"
  44 #include "structchi2.h"
  45 #include "speciesprofile.h"
  46 #include "hamming.h"
  47 #include "gower.h"
  48 #include "memchi2.h"
  49 #include "memchord.h"
  50 #include "memeuclidean.h"
  51 #include "mempearson.h"
  52 //**********************************************************************************************************************
  53 vector<string> MatrixOutputCommand::getValidParameters(){
  54         try {
  55                 string Array[] =  {"label","calc","groups","outputdir","inputdir", "output"};
  56                 vector<string> myArray (Array, Array+(sizeof(Array)/sizeof(string)));
  57                 return myArray;
  58         }
  59         catch(exception& e) {
  60                 m->errorOut(e, "MatrixOutputCommand", "getValidParameters");
  61                 exit(1);
  62         }
  63 }
  64 //**********************************************************************************************************************
  65 MatrixOutputCommand::MatrixOutputCommand(){
  66         try {
  67                 abort = true; calledHelp = true;
  68                 vector<string> tempOutNames;
  69                 outputTypes["phylip"] = tempOutNames;
  70         }
  71         catch(exception& e) {
  72                 m->errorOut(e, "MatrixOutputCommand", "MatrixOutputCommand");
  73                 exit(1);
  74         }
  75 }
  76 //**********************************************************************************************************************
  77 vector<string> MatrixOutputCommand::getRequiredParameters(){
  78         try {
  79                 vector<string> myArray;
  80                 return myArray;
  81         }
  82         catch(exception& e) {
  83                 m->errorOut(e, "MatrixOutputCommand", "getRequiredParameters");
  84                 exit(1);
  85         }
  86 }
  87 //**********************************************************************************************************************
  88 vector<string> MatrixOutputCommand::getRequiredFiles(){
  89         try {
  90                 string Array[] =  {"shared"};
  91                 vector<string> myArray (Array, Array+(sizeof(Array)/sizeof(string)));
  92                 return myArray;
  93         }
  94         catch(exception& e) {
  95                 m->errorOut(e, "MatrixOutputCommand", "getRequiredFiles");
  96                 exit(1);
  97         }
  98 }
  99 //**********************************************************************************************************************
 100
 101 MatrixOutputCommand::MatrixOutputCommand(string option)  {
 102         try {
 103                 globaldata = GlobalData::getInstance();
 104                 abort = false; calledHelp = false;
 105                 allLines = 1;
 106                 labels.clear();
 107                 Groups.clear();
 108                 Estimators.clear();
 109
 110                 //allow user to run help
 111                 if(option == "help") { validCalculator = new ValidCalculators(); help(); abort = true; calledHelp = true; }
 112
 113                 else {
 114                         //valid paramters for this command
 115                         string Array[] =  {"label","calc","groups","outputdir","inputdir", "output"};
 116                         vector<string> myArray (Array, Array+(sizeof(Array)/sizeof(string)));
 117
 118                         OptionParser parser(option);
 119                         map<string,string> parameters  = parser.getParameters();
 120
 121                         ValidParameters validParameter;
 122
 123                         //check to make sure all parameters are valid for command
 124                         for (map<string,string>::iterator it = parameters.begin(); it != parameters.end(); it++) {
 125                                 if (validParameter.isValidParameter(it->first, myArray, it->second) != true) {  abort = true;  }
 126                         }
 127
 128                         //initialize outputTypes
 129                         vector<string> tempOutNames;
 130                         outputTypes["phylip"] = tempOutNames;
 131
 132                         //if the user changes the output directory command factory will send this info to us in the output parameter
 133                         outputDir = validParameter.validFile(parameters, "outputdir", false);           if (outputDir == "not found"){
 134                                 outputDir = "";
 135                                 outputDir += m->hasPath(globaldata->inputFileName); //if user entered a file with a path then preserve it
 136                         }
 137
 138                         //make sure the user has already run the read.otu command
 139                         if (globaldata->getSharedFile() == "") {
 140                                 if (globaldata->getListFile() == "") { m->mothurOut("You must read a list and a group, or a shared before you can use the dist.shared command."); m->mothurOutEndLine(); abort = true; }
 141                                 else if (globaldata->getGroupFile() == "") { m->mothurOut("You must read a list and a group, or a shared before you can use the dist.shared command."); m->mothurOutEndLine(); abort = true; }
 142                         }
 143
 144                         //check for optional parameter and set defaults
 145                         // ...at some point should added some additional type checking...
 146                         label = validParameter.validFile(parameters, "label", false);
 147                         if (label == "not found") { label = ""; }
 148                         else {
 149                                 if(label != "all") {  m->splitAtDash(label, labels);  allLines = 0;  }
 150                                 else { allLines = 1;  }
 151                         }
 152
 153                         output = validParameter.validFile(parameters, "output", false);         if(output == "not found"){      output = "lt"; }
 154                         if ((output != "lt") && (output != "square")) { m->mothurOut(output + " is not a valid output form. Options are lt and square. I will use lt."); m->mothurOutEndLine(); output = "lt"; }
 155
 156                         //if the user has not specified any labels use the ones from read.otu
 157                         if (label == "") {
 158                                 allLines = globaldata->allLines;
 159                                 labels = globaldata->labels;
 160                         }
 161
 162                         groups = validParameter.validFile(parameters, "groups", false);
 163                         if (groups == "not found") { groups = ""; }
 164                         else {
 165                                 m->splitAtDash(groups, Groups);
 166                                 globaldata->Groups = Groups;
 167                         }
 168
 169                         calc = validParameter.validFile(parameters, "calc", false);
 170                         if (calc == "not found") { calc = "jclass-thetayc";  }
 171                         else {
 172                                  if (calc == "default")  {  calc = "jclass-thetayc";  }
 173                         }
 174                         m->splitAtDash(calc, Estimators);
 175
 176                         if (abort == false) {
 177
 178                                 validCalculator = new ValidCalculators();
 179
 180                                 int i;
 181                                 for (i=0; i<Estimators.size(); i++) {
 182                                         if (validCalculator->isValidCalculator("matrix", Estimators[i]) == true) {
 183                                                 if (Estimators[i] == "sharedsobs") {
 184                                                         matrixCalculators.push_back(new SharedSobsCS());
 185                                                 }else if (Estimators[i] == "sharedchao") {
 186                                                         matrixCalculators.push_back(new SharedChao1());
 187                                                 }else if (Estimators[i] == "sharedace") {
 188                                                         matrixCalculators.push_back(new SharedAce());
 189                                                 }else if (Estimators[i] == "jabund") {
 190                                                         matrixCalculators.push_back(new JAbund());
 191                                                 }else if (Estimators[i] == "sorabund") {
 192                                                         matrixCalculators.push_back(new SorAbund());
 193                                                 }else if (Estimators[i] == "jclass") {
 194                                                         matrixCalculators.push_back(new Jclass());
 195                                                 }else if (Estimators[i] == "sorclass") {
 196                                                         matrixCalculators.push_back(new SorClass());
 197                                                 }else if (Estimators[i] == "jest") {
 198                                                         matrixCalculators.push_back(new Jest());
 199                                                 }else if (Estimators[i] == "sorest") {
 200                                                         matrixCalculators.push_back(new SorEst());
 201                                                 }else if (Estimators[i] == "thetayc") {
 202                                                         matrixCalculators.push_back(new ThetaYC());
 203                                                 }else if (Estimators[i] == "thetan") {
 204                                                         matrixCalculators.push_back(new ThetaN());
 205                                                 }else if (Estimators[i] == "kstest") {
 206                                                         matrixCalculators.push_back(new KSTest());
 207                                                 }else if (Estimators[i] == "sharednseqs") {
 208                                                         matrixCalculators.push_back(new SharedNSeqs());
 209                                                 }else if (Estimators[i] == "ochiai") {
 210                                                         matrixCalculators.push_back(new Ochiai());
 211                                                 }else if (Estimators[i] == "anderberg") {
 212                                                         matrixCalculators.push_back(new Anderberg());
 213                                                 }else if (Estimators[i] == "kulczynski") {
 214                                                         matrixCalculators.push_back(new Kulczynski());
 215                                                 }else if (Estimators[i] == "kulczynskicody") {
 216                                                         matrixCalculators.push_back(new KulczynskiCody());
 217                                                 }else if (Estimators[i] == "lennon") {
 218                                                         matrixCalculators.push_back(new Lennon());
 219                                                 }else if (Estimators[i] == "morisitahorn") {
 220                                                         matrixCalculators.push_back(new MorHorn());
 221                                                 }else if (Estimators[i] == "braycurtis") {
 222                                                         matrixCalculators.push_back(new BrayCurtis());
 223                                                 }else if (Estimators[i] == "whittaker") {
 224                                                         matrixCalculators.push_back(new Whittaker());
 225                                                 }else if (Estimators[i] == "odum") {
 226                                                         matrixCalculators.push_back(new Odum());
 227                                                 }else if (Estimators[i] == "canberra") {
 228                                                         matrixCalculators.push_back(new Canberra());
 229                                                 }else if (Estimators[i] == "structeuclidean") {
 230                                                         matrixCalculators.push_back(new StructEuclidean());
 231                                                 }else if (Estimators[i] == "structchord") {
 232                                                         matrixCalculators.push_back(new StructChord());
 233                                                 }else if (Estimators[i] == "hellinger") {
 234                                                         matrixCalculators.push_back(new Hellinger());
 235                                                 }else if (Estimators[i] == "manhattan") {
 236                                                         matrixCalculators.push_back(new Manhattan());
 237                                                 }else if (Estimators[i] == "structpearson") {
 238                                                         matrixCalculators.push_back(new StructPearson());
 239                                                 }else if (Estimators[i] == "soergel") {
 240                                                         matrixCalculators.push_back(new Soergel());
 241                                                 }else if (Estimators[i] == "spearman") {
 242                                                         matrixCalculators.push_back(new Spearman());
 243                                                 }else if (Estimators[i] == "structkulczynski") {
 244                                                         matrixCalculators.push_back(new StructKulczynski());
 245                                                 }else if (Estimators[i] == "speciesprofile") {
 246                                                         matrixCalculators.push_back(new SpeciesProfile());
 247                                                 }else if (Estimators[i] == "hamming") {
 248                                                         matrixCalculators.push_back(new Hamming());
 249                                                 }else if (Estimators[i] == "structchi2") {
 250                                                         matrixCalculators.push_back(new StructChi2());
 251                                                 }else if (Estimators[i] == "gower") {
 252                                                         matrixCalculators.push_back(new Gower());
 253                                                 }else if (Estimators[i] == "memchi2") {
 254                                                         matrixCalculators.push_back(new MemChi2());
 255                                                 }else if (Estimators[i] == "memchord") {
 256                                                         matrixCalculators.push_back(new MemChord());
 257                                                 }else if (Estimators[i] == "memeuclidean") {
 258                                                         matrixCalculators.push_back(new MemEuclidean());
 259                                                 }else if (Estimators[i] == "mempearson") {
 260                                                         matrixCalculators.push_back(new MemPearson());
 261                                                 }
 262                                         }
 263                                 }
 264
 265                         }
 266                 }
 267
 268         }
 269         catch(exception& e) {
 270                 m->errorOut(e, "MatrixOutputCommand", "MatrixOutputCommand");
 271                 exit(1);
 272         }
 273 }
 274
 275 //**********************************************************************************************************************
 276
 277 void MatrixOutputCommand::help(){
 278         try {
 279                 m->mothurOut("The dist.shared command can only be executed after a successful read.otu command.\n");
 280                 m->mothurOut("The dist.shared command parameters are groups, calc, output and label.  No parameters are required.\n");
 281                 m->mothurOut("The groups parameter allows you to specify which of the groups in your groupfile you would like included used.\n");
 282                 m->mothurOut("The group names are separated by dashes. The label parameter allows you to select what distance levels you would like distance matrices created for, and is also separated by dashes.\n");
 283                 m->mothurOut("The dist.shared command should be in the following format: dist.shared(groups=yourGroups, calc=yourCalcs, label=yourLabels).\n");
 284                 m->mothurOut("The output parameter allows you to specify format of your distance matrix. Options are lt, and square. The default is lt.\n");
 285                 m->mothurOut("Example dist.shared(groups=A-B-C, calc=jabund-sorabund).\n");
 286                 m->mothurOut("The default value for groups is all the groups in your groupfile.\n");
 287                 m->mothurOut("The default value for calc is jclass and thetayc.\n");
 288                 validCalculator->printCalc("matrix", cout);
 289                 m->mothurOut("The dist.shared command outputs a .dist file for each calculator you specify at each distance you choose.\n");
 290                 m->mothurOut("Note: No spaces between parameter labels (i.e. groups), '=' and parameters (i.e.yourGroups).\n\n");
 291         }
 292         catch(exception& e) {
 293                 m->errorOut(e, "MatrixOutputCommand", "help");
 294                 exit(1);
 295         }
 296 }
 297
 298
 299 //**********************************************************************************************************************
 300
 301 MatrixOutputCommand::~MatrixOutputCommand(){
 302         if (abort == false) {
 303                 delete input; globaldata->ginput = NULL;
 304                 delete read;
 305                 delete validCalculator;
 306         }
 307 }
 308
 309 //**********************************************************************************************************************
 310
 311 int MatrixOutputCommand::execute(){
 312         try {
 313
 314                 if (abort == true) { if (calledHelp) { return 0; }  return 2;   }
 315
 316                 //if the users entered no valid calculators don't execute command
 317                 if (matrixCalculators.size() == 0) { m->mothurOut("No valid calculators."); m->mothurOutEndLine();  return 0; }
 318
 319                 //you have groups
 320                 read = new ReadOTUFile(globaldata->inputFileName);
 321                 read->read(&*globaldata);
 322
 323                 input = globaldata->ginput;
 324                 lookup = input->getSharedRAbundVectors();
 325                 string lastLabel = lookup[0]->getLabel();
 326
 327                 //if the users enters label "0.06" and there is no "0.06" in their file use the next lowest label.
 328                 set<string> processedLabels;
 329                 set<string> userLabels = labels;
 330
 331                 if (lookup.size() < 2) { m->mothurOut("You have not provided enough valid groups.  I cannot run the command."); m->mothurOutEndLine(); return 0;}
 332
 333                 numGroups = lookup.size();
 334
 335                 if (m->control_pressed) { delete read; delete input; globaldata->ginput = NULL; for (int i = 0; i < lookup.size(); i++) {  delete lookup[i];  } globaldata->Groups.clear(); return 0;  }
 336
 337                 //as long as you are not at the end of the file or done wih the lines you want
 338                 while((lookup[0] != NULL) && ((allLines == 1) || (userLabels.size() != 0))) {
 339
 340                         if (m->control_pressed) { outputTypes.clear(); delete read; delete input; globaldata->ginput = NULL; for (int i = 0; i < lookup.size(); i++) {  delete lookup[i];  } for (int i = 0; i < outputNames.size(); i++) {     remove(outputNames[i].c_str()); } globaldata->Groups.clear(); return 0;  }
 341
 342                         if(allLines == 1 || labels.count(lookup[0]->getLabel()) == 1){
 343                                 m->mothurOut(lookup[0]->getLabel()); m->mothurOutEndLine();
 344                                 process(lookup);
 345
 346                                 processedLabels.insert(lookup[0]->getLabel());
 347                                 userLabels.erase(lookup[0]->getLabel());
 348                         }
 349
 350                         if ((m->anyLabelsToProcess(lookup[0]->getLabel(), userLabels, "") == true) && (processedLabels.count(lastLabel) != 1)) {
 351                                 string saveLabel = lookup[0]->getLabel();
 352
 353                                 for (int i = 0; i < lookup.size(); i++) {  delete lookup[i];  }
 354                                 lookup = input->getSharedRAbundVectors(lastLabel);
 355
 356                                 m->mothurOut(lookup[0]->getLabel()); m->mothurOutEndLine();
 357                                 process(lookup);
 358
 359                                 processedLabels.insert(lookup[0]->getLabel());
 360                                 userLabels.erase(lookup[0]->getLabel());
 361
 362                                 //restore real lastlabel to save below
 363                                 lookup[0]->setLabel(saveLabel);
 364                         }
 365
 366                         lastLabel = lookup[0]->getLabel();
 367
 368                         //get next line to process
 369                         for (int i = 0; i < lookup.size(); i++) {  delete lookup[i];  }
 370                         lookup = input->getSharedRAbundVectors();
 371                 }
 372
 373                 if (m->control_pressed) { outputTypes.clear(); delete read; delete input; globaldata->ginput = NULL; for (int i = 0; i < outputNames.size(); i++) {     remove(outputNames[i].c_str()); } globaldata->Groups.clear(); return 0;  }
 374
 375                 //output error messages about any remaining user labels
 376                 set<string>::iterator it;
 377                 bool needToRun = false;
 378                 for (it = userLabels.begin(); it != userLabels.end(); it++) {
 379                         m->mothurOut("Your file does not include the label " + *it);
 380                         if (processedLabels.count(lastLabel) != 1) {
 381                                 m->mothurOut(". I will use " + lastLabel + "."); m->mothurOutEndLine();
 382                                 needToRun = true;
 383                         }else {
 384                                 m->mothurOut(". Please refer to " + lastLabel + "."); m->mothurOutEndLine();
 385                         }
 386                 }
 387
 388                 if (m->control_pressed) { outputTypes.clear(); delete read; delete input; globaldata->ginput = NULL;  for (int i = 0; i < outputNames.size(); i++) {    remove(outputNames[i].c_str()); } globaldata->Groups.clear(); return 0;  }
 389
 390                 //run last label if you need to
 391                 if (needToRun == true)  {
 392                         for (int i = 0; i < lookup.size(); i++) {  if (lookup[i] != NULL) {  delete lookup[i]; }  }
 393                         lookup = input->getSharedRAbundVectors(lastLabel);
 394
 395                         m->mothurOut(lookup[0]->getLabel()); m->mothurOutEndLine();
 396                         process(lookup);
 397                         for (int i = 0; i < lookup.size(); i++) {  delete lookup[i];  }
 398                 }
 399
 400                 if (m->control_pressed) { outputTypes.clear();  delete read; delete input; globaldata->ginput = NULL;  for (int i = 0; i < outputNames.size(); i++) {   remove(outputNames[i].c_str()); } globaldata->Groups.clear(); return 0;  }
 401
 402                 //reset groups parameter
 403                 globaldata->Groups.clear();
 404
 405                 m->mothurOutEndLine();
 406                 m->mothurOut("Output File Names: "); m->mothurOutEndLine();
 407                 for (int i = 0; i < outputNames.size(); i++) {  m->mothurOut(outputNames[i]); m->mothurOutEndLine();    }
 408                 m->mothurOutEndLine();
 409
 410
 411                 return 0;
 412         }
 413         catch(exception& e) {
 414                 m->errorOut(e, "MatrixOutputCommand", "execute");
 415                 exit(1);
 416         }
 417 }
 418 /***********************************************************/
 419 void MatrixOutputCommand::printSims(ostream& out) {
 420         try {
 421
 422                 out.setf(ios::fixed, ios::floatfield); out.setf(ios::showpoint);
 423
 424                 //output num seqs
 425                 out << simMatrix.size() << endl;
 426
 427                 if (output == "lt") {
 428                         for (int m = 0; m < simMatrix.size(); m++)      {
 429                                 out << lookup[m]->getGroup() << '\t';
 430                                 for (int n = 0; n < m; n++)     {
 431                                         out << simMatrix[m][n] << '\t';
 432                                 }
 433                                 out << endl;
 434                         }
 435                 }else{
 436                         for (int m = 0; m < simMatrix.size(); m++)      {
 437                                 out << lookup[m]->getGroup() << '\t';
 438                                 for (int n = 0; n < simMatrix[m].size(); n++)   {
 439                                         out << simMatrix[m][n] << '\t';
 440                                 }
 441                                 out << endl;
 442                         }
 443                 }
 444         }
 445         catch(exception& e) {
 446                 m->errorOut(e, "MatrixOutputCommand", "printSims");
 447                 exit(1);
 448         }
 449 }
 450 /***********************************************************/
 451 int MatrixOutputCommand::process(vector<SharedRAbundVector*> thisLookup){
 452         try {
 453
 454                                 EstOutput data;
 455                                 vector<SharedRAbundVector*> subset;
 456
 457                                 //for each calculator
 458                                 for(int i = 0 ; i < matrixCalculators.size(); i++) {
 459
 460                                         //initialize simMatrix
 461                                         simMatrix.clear();
 462                                         simMatrix.resize(numGroups);
 463                                         for (int p = 0; p < simMatrix.size(); p++)      {
 464                                                 for (int j = 0; j < simMatrix.size(); j++)      {
 465                                                         simMatrix[p].push_back(0.0);
 466                                                 }
 467                                         }
 468
 469                                         for (int k = 0; k < thisLookup.size(); k++) {
 470                                                 for (int l = k; l < thisLookup.size(); l++) {
 471                                                         if (k != l) { //we dont need to similiarity of a groups to itself
 472                                                                 //get estimated similarity between 2 groups
 473
 474                                                                 if (m->control_pressed) { return 0; }
 475
 476                                                                 subset.clear(); //clear out old pair of sharedrabunds
 477                                                                 //add new pair of sharedrabunds
 478                                                                 subset.push_back(thisLookup[k]); subset.push_back(thisLookup[l]);
 479
 480                                                                 //if this calc needs all groups to calculate the pair load all groups
 481                                                                 if (matrixCalculators[i]->getNeedsAll()) {
 482                                                                         //load subset with rest of lookup for those calcs that need everyone to calc for a pair
 483                                                                         for (int w = 0; w < thisLookup.size(); w++) {
 484                                                                                 if ((w != k) && (w != l)) { subset.push_back(thisLookup[w]); }
 485                                                                         }
 486                                                                 }
 487
 488                                                                 data = matrixCalculators[i]->getValues(subset); //saves the calculator outputs
 489                                                                 //save values in similarity matrix
 490                                                                 simMatrix[k][l] = 1.0 - data[0];  //convert similiarity to distance
 491                                                                 simMatrix[l][k] = 1.0 - data[0];  //convert similiarity to distance
 492                                                         }
 493                                                 }
 494                                         }
 495
 496                                         exportFileName = outputDir + m->getRootName(m->getSimpleName(globaldata->inputFileName)) + matrixCalculators[i]->getName() + "." + thisLookup[0]->getLabel() + "." + output + ".dist";
 497                                         m->openOutputFile(exportFileName, out);
 498                                         outputNames.push_back(exportFileName); outputTypes["phylip"].push_back(exportFileName);
 499
 500                                         printSims(out);
 501                                         out.close();
 502
 503                                 }
 504
 505                                 return 0;
 506
 507         }
 508         catch(exception& e) {
 509                 m->errorOut(e, "MatrixOutputCommand", "process");
 510                 exit(1);
 511         }
 512 }
 513 /***********************************************************/
 514
 515
 516