phylotree.cpp

   1 /*
   2  *  doTaxonomy.cpp
   3  *
   4  *
   5  *  Created by Pat Schloss on 6/17/09.
   6  *  Copyright 2009 Patrick D. Schloss. All rights reserved.
   7  *
   8  */
   9
  10 #include "phylotree.h"
  11
  12 /**************************************************************************************************/
  13
  14 PhyloTree::PhyloTree(){
  15         try {
  16                 m = MothurOut::getInstance();
  17                 numNodes = 1;
  18                 numSeqs = 0;
  19                 tree.push_back(TaxNode("Root"));
  20                 tree[0].heirarchyID = "0";
  21                 maxLevel = 0;
  22                 calcTotals = true;
  23                 addSeqToTree("unknown", "unknown;");
  24         }
  25         catch(exception& e) {
  26                 m->errorOut(e, "PhyloTree", "PhyloTree");
  27                 exit(1);
  28         }
  29 }
  30 /**************************************************************************************************/
  31
  32 PhyloTree::PhyloTree(ifstream& in, string filename){
  33         try {
  34                 m = MothurOut::getInstance();
  35                 calcTotals = false;
  36                 numNodes = 0;
  37                 numSeqs = 0;
  38
  39                 #ifdef USE_MPI
  40                         MPI_File inMPI;
  41                         MPI_Offset size;
  42                         MPI_Status status;
  43
  44                         char inFileName[1024];
  45                         strcpy(inFileName, filename.c_str());
  46
  47                         MPI_File_open(MPI_COMM_WORLD, inFileName, MPI_MODE_RDONLY, MPI_INFO_NULL, &inMPI);
  48                         MPI_File_get_size(inMPI, &size);
  49
  50                         char* buffer = new char[size];
  51                         MPI_File_read(inMPI, buffer, size, MPI_CHAR, &status);
  52
  53                         string tempBuf = buffer;
  54                         if (tempBuf.length() > size) { tempBuf = tempBuf.substr(0, size);  }
  55                         istringstream iss (tempBuf,istringstream::in);
  56                         delete buffer;
  57
  58                         //read version
  59                         m->getline(iss); m->gobble(iss);
  60
  61                         iss >> numNodes; m->gobble(iss);
  62
  63                         tree.resize(numNodes);
  64
  65                         for (int i = 0; i < tree.size(); i++) {
  66                                 iss >> tree[i].name >> tree[i].level >> tree[i].parent; m->gobble(iss);
  67                         }
  68
  69                         //read genus nodes
  70                         int numGenus = 0;
  71                         iss >> numGenus; m->gobble(iss);
  72
  73                         int gnode, gsize;
  74                         totals.clear();
  75                         for (int i = 0; i < numGenus; i++) {
  76                                 iss >> gnode >> gsize; m->gobble(iss);
  77
  78                                 uniqueTaxonomies.insert(gnode);
  79                                 totals.push_back(gsize);
  80                         }
  81
  82                         MPI_File_close(&inMPI);
  83
  84                 #else
  85                         //read version
  86                         string line = m->getline(in); m->gobble(in);
  87
  88                         in >> numNodes; m->gobble(in);
  89
  90                         tree.resize(numNodes);
  91
  92                         for (int i = 0; i < tree.size(); i++) {
  93                                 in >> tree[i].name >> tree[i].level >> tree[i].parent; m->gobble(in);
  94                         }
  95
  96                         //read genus nodes
  97                         int numGenus = 0;
  98                         in >> numGenus; m->gobble(in);
  99
 100                         int gnode, gsize;
 101                         totals.clear();
 102                         for (int i = 0; i < numGenus; i++) {
 103                                 in >> gnode >> gsize; m->gobble(in);
 104
 105                                 uniqueTaxonomies.insert(gnode);
 106                                 totals.push_back(gsize);
 107                         }
 108
 109                         in.close();
 110
 111                 #endif
 112
 113         }
 114         catch(exception& e) {
 115                 m->errorOut(e, "PhyloTree", "PhyloTree");
 116                 exit(1);
 117         }
 118 }
 119 /**************************************************************************************************/
 120
 121 PhyloTree::PhyloTree(string tfile){
 122         try {
 123                 m = MothurOut::getInstance();
 124                 numNodes = 1;
 125                 numSeqs = 0;
 126                 tree.push_back(TaxNode("Root"));
 127                 tree[0].heirarchyID = "0";
 128                 maxLevel = 0;
 129                 calcTotals = true;
 130                 string name, tax;
 131
 132                 #ifdef USE_MPI
 133                         int pid, num, processors;
 134                         vector<unsigned long long> positions;
 135
 136                         MPI_Status status;
 137                         MPI_File inMPI;
 138                         MPI_Comm_rank(MPI_COMM_WORLD, &pid); //find out who we are
 139                         MPI_Comm_size(MPI_COMM_WORLD, &processors);
 140
 141                         char inFileName[1024];
 142                         strcpy(inFileName, tfile.c_str());
 143
 144                         MPI_File_open(MPI_COMM_WORLD, inFileName, MPI_MODE_RDONLY, MPI_INFO_NULL, &inMPI);  //comm, filename, mode, info, filepointer
 145
 146                         if (pid == 0) {
 147                                 positions = m->setFilePosEachLine(tfile, num);
 148
 149                                 //send file positions to all processes
 150                                 for(int i = 1; i < processors; i++) {
 151                                         MPI_Send(&num, 1, MPI_INT, i, 2001, MPI_COMM_WORLD);
 152                                         MPI_Send(&positions[0], (num+1), MPI_LONG, i, 2001, MPI_COMM_WORLD);
 153                                 }
 154                         }else{
 155                                 MPI_Recv(&num, 1, MPI_INT, 0, 2001, MPI_COMM_WORLD, &status);
 156                                 positions.resize(num+1);
 157                                 MPI_Recv(&positions[0], (num+1), MPI_LONG, 0, 2001, MPI_COMM_WORLD, &status);
 158                         }
 159
 160                         //read file
 161                         for(int i=0;i<num;i++){
 162                                 //read next sequence
 163                                 int length = positions[i+1] - positions[i];
 164                                 char* buf4 = new char[length];
 165
 166                                 MPI_File_read_at(inMPI, positions[i], buf4, length, MPI_CHAR, &status);
 167
 168                                 string tempBuf = buf4;
 169                                 if (tempBuf.length() > length) { tempBuf = tempBuf.substr(0, length); }
 170                                 delete buf4;
 171
 172                                 istringstream iss (tempBuf,istringstream::in);
 173                                 iss >> name >> tax;
 174                                 addSeqToTree(name, tax);
 175                         }
 176
 177                         MPI_File_close(&inMPI);
 178                         MPI_Barrier(MPI_COMM_WORLD); //make everyone wait - just in case
 179
 180                 #else
 181             map<string, string> temp;
 182             m->readTax(tfile, temp);
 183
 184             for (map<string, string>::iterator itTemp = temp.begin(); itTemp != temp.end();) {
 185                 addSeqToTree(itTemp->first, itTemp->second);
 186                 temp.erase(itTemp++);
 187             }
 188                 #endif
 189
 190                 assignHeirarchyIDs(0);
 191
 192
 193         string unknownTax = "unknown;";
 194         //added last taxon until you get desired level
 195                 for (int i = 1; i < maxLevel; i++) {
 196                         unknownTax += "unclassfied;";
 197                 }
 198
 199         addSeqToTree("unknown", unknownTax);
 200
 201                 //create file for summary if needed
 202                 setUp(tfile);
 203         }
 204         catch(exception& e) {
 205                 m->errorOut(e, "PhyloTree", "PhyloTree");
 206                 exit(1);
 207         }
 208 }
 209
 210 /**************************************************************************************************/
 211
 212 string PhyloTree::getNextTaxon(string& heirarchy, string seqname){
 213         try {
 214                 string currentLevel = "";
 215                 if(heirarchy != ""){
 216                         int pos = heirarchy.find_first_of(';');
 217
 218                         if (pos == -1) { //you can't find another ;
 219                                 currentLevel = heirarchy;
 220                                 heirarchy = "";
 221                                 m->mothurOut(seqname + " is missing a ;, please check for other errors."); m->mothurOutEndLine();
 222                         }else{
 223                                 currentLevel=heirarchy.substr(0,pos);
 224                                 if (pos != (heirarchy.length()-1)) {  heirarchy=heirarchy.substr(pos+1);  }
 225                                 else { heirarchy = ""; }
 226                         }
 227
 228                 }
 229                 return currentLevel;
 230         }
 231         catch(exception& e) {
 232                 m->errorOut(e, "PhyloTree", "getNextTaxon");
 233                 exit(1);
 234         }
 235 }
 236
 237 /**************************************************************************************************/
 238
 239 int PhyloTree::addSeqToTree(string seqName, string seqTaxonomy){
 240         try {
 241                 numSeqs++;
 242
 243                 map<string, int>::iterator childPointer;
 244
 245                 int currentNode = 0;
 246                 int level = 1;
 247
 248                 tree[0].accessions.push_back(seqName);
 249                 m->removeConfidences(seqTaxonomy);
 250
 251                 string taxon;// = getNextTaxon(seqTaxonomy);
 252
 253                 while(seqTaxonomy != ""){
 254
 255                         level++;
 256
 257                         if (m->control_pressed) { return 0; }
 258
 259                         //somehow the parent is getting one too many accnos
 260                         //use print to reassign the taxa id
 261                         taxon = getNextTaxon(seqTaxonomy, seqName);
 262
 263                         if (taxon == "") {  m->mothurOut(seqName + " has an error in the taxonomy.  This may be due to a ;;"); m->mothurOutEndLine(); if (currentNode != 0) {  uniqueTaxonomies.insert(currentNode); } break;  }
 264
 265                         childPointer = tree[currentNode].children.find(taxon);
 266
 267                         if(childPointer != tree[currentNode].children.end()){   //if the node already exists, move on
 268                                 currentNode = childPointer->second;
 269                                 tree[currentNode].accessions.push_back(seqName);
 270                                 name2Taxonomy[seqName] = currentNode;
 271                         }
 272                         else{                                                                                   //otherwise, create it
 273                                 tree.push_back(TaxNode(taxon));
 274                                 numNodes++;
 275                                 tree[currentNode].children[taxon] = numNodes-1;
 276                                 tree[numNodes-1].parent = currentNode;
 277
 278                                 currentNode = tree[currentNode].children[taxon];
 279                                 tree[currentNode].accessions.push_back(seqName);
 280                                 name2Taxonomy[seqName] = currentNode;
 281                         }
 282
 283                         if (seqTaxonomy == "") {   uniqueTaxonomies.insert(currentNode);        }
 284                 }
 285
 286                 return 0;
 287         }
 288         catch(exception& e) {
 289                 m->errorOut(e, "PhyloTree", "addSeqToTree");
 290                 exit(1);
 291         }
 292 }
 293 /**************************************************************************************************/
 294 vector<int> PhyloTree::getGenusNodes()  {
 295         try {
 296                 genusIndex.clear();
 297                 //generate genusIndexes
 298                 set<int>::iterator it2;
 299         map<int, int> temp;
 300                 for (it2=uniqueTaxonomies.begin(); it2!=uniqueTaxonomies.end(); it2++) {  genusIndex.push_back(*it2);   temp[*it2] = genusIndex.size()-1; }
 301
 302         for (map<string, int>::iterator itName = name2Taxonomy.begin(); itName != name2Taxonomy.end(); itName++) {
 303             map<int, int>::iterator itTemp = temp.find(itName->second);
 304             if (itTemp != temp.end()) { name2GenusNodeIndex[itName->first] = itTemp->second; }
 305             else {  m->mothurOut("[ERROR]: trouble making name2GenusNodeIndex, aborting.\n"); m->control_pressed = true; }
 306         }
 307
 308                 return genusIndex;
 309         }
 310         catch(exception& e) {
 311                 m->errorOut(e, "PhyloTree", "getGenusNodes");
 312                 exit(1);
 313         }
 314 }
 315 /**************************************************************************************************/
 316 vector<int> PhyloTree::getGenusTotals() {
 317         try {
 318
 319                 if (calcTotals) {
 320                         totals.clear();
 321                         //reset counts because we are on a new word
 322                         for (int j = 0; j < genusIndex.size(); j++) {
 323                                 totals.push_back(tree[genusIndex[j]].accessions.size());
 324                         }
 325                         return totals;
 326                 }else{
 327                         return totals;
 328                 }
 329
 330         }
 331         catch(exception& e) {
 332                 m->errorOut(e, "PhyloTree", "getGenusNodes");
 333                 exit(1);
 334         }
 335 }
 336 /**************************************************************************************************/
 337
 338 void PhyloTree::assignHeirarchyIDs(int index){
 339         try {
 340                 map<string,int>::iterator it;
 341                 int counter = 1;
 342
 343                 for(it=tree[index].children.begin();it!=tree[index].children.end();it++){
 344                         tree[it->second].heirarchyID = tree[index].heirarchyID + '.' + toString(counter);
 345                         counter++;
 346                         tree[it->second].level = tree[index].level + 1;
 347
 348                         //save maxLevel for binning the unclassified seqs
 349                         if (tree[it->second].level > maxLevel) { maxLevel = tree[it->second].level; }
 350
 351                         assignHeirarchyIDs(it->second);
 352                 }
 353         }
 354         catch(exception& e) {
 355                 m->errorOut(e, "PhyloTree", "assignHeirarchyIDs");
 356                 exit(1);
 357         }
 358 }
 359 /**************************************************************************************************/
 360 void PhyloTree::setUp(string tfile){
 361         try{
 362                 string taxFileNameTest = tfile.substr(0,tfile.find_last_of(".")+1) + "tree.sum";
 363
 364                 #ifdef USE_MPI
 365                         int pid;
 366                         MPI_Comm_rank(MPI_COMM_WORLD, &pid); //find out who we are
 367
 368                         if (pid == 0) {  binUnclassified(taxFileNameTest);  }
 369
 370                 #else
 371                         binUnclassified(taxFileNameTest);
 372                 #endif
 373         }
 374         catch(exception& e) {
 375                 m->errorOut(e, "PhyloTree", "setUp");
 376                 exit(1);
 377         }
 378 }
 379 /**************************************************************************************************/
 380 void PhyloTree::binUnclassified(string file){
 381         try {
 382
 383                 ofstream out;
 384                 m->openOutputFile(file, out);
 385
 386                 map<string, int>::iterator itBin;
 387                 map<string, int>::iterator childPointer;
 388
 389                 vector<TaxNode> copy = tree;
 390
 391                 //fill out tree
 392                 fillOutTree(0, copy);
 393
 394                 //get leaf nodes that may need extension
 395                 for (int i = 0; i < copy.size(); i++) {
 396
 397                         if (copy[i].children.size() == 0) {
 398                                 leafNodes[i] = i;
 399                         }
 400                 }
 401
 402                 int copyNodes = copy.size();
 403
 404                 //go through the seqs and if a sequence finest taxon is not the same level as the most finely defined taxon then classify it as unclassified where necessary
 405                 map<int, int>::iterator itLeaf;
 406                 for (itLeaf = leafNodes.begin(); itLeaf != leafNodes.end(); itLeaf++) {
 407
 408                         if (m->control_pressed) {  out.close(); break;  }
 409
 410                         int level = copy[itLeaf->second].level;
 411                         int currentNode = itLeaf->second;
 412
 413                         //this sequence is unclassified at some levels
 414                         while(level < maxLevel){
 415
 416                                 level++;
 417
 418                                 string taxon = "unclassified";
 419
 420                                 //does the parent have a child names 'unclassified'?
 421                                 childPointer = copy[currentNode].children.find(taxon);
 422
 423                                 if(childPointer != copy[currentNode].children.end()){   //if the node already exists, move on
 424                                         currentNode = childPointer->second; //currentNode becomes 'unclassified'
 425                                 }
 426                                 else{                                                                                   //otherwise, create it
 427                                         copy.push_back(TaxNode(taxon));
 428                                         copyNodes++;
 429                                         copy[currentNode].children[taxon] = copyNodes-1;
 430                                         copy[copyNodes-1].parent = currentNode;
 431                                         copy[copyNodes-1].level = copy[currentNode].level + 1;
 432
 433                                         currentNode = copy[currentNode].children[taxon];
 434                                 }
 435                         }
 436                 }
 437
 438                 if (!m->control_pressed) {
 439                         //print copy tree
 440                         print(out, copy);
 441                 }
 442
 443         }
 444         catch(exception& e) {
 445                 m->errorOut(e, "PhyloTree", "binUnclassified");
 446                 exit(1);
 447         }
 448 }
 449 /**************************************************************************************************/
 450 void PhyloTree::fillOutTree(int index, vector<TaxNode>& copy) {
 451         try {
 452
 453                 map<string,int>::iterator it;
 454
 455                 it = copy[index].children.find("unclassified");
 456                 if (it == copy[index].children.end()) { //no unclassified at this level
 457                         string taxon = "unclassified";
 458                         copy.push_back(TaxNode(taxon));
 459                         copy[index].children[taxon] = copy.size()-1;
 460                         copy[copy.size()-1].parent = index;
 461                         copy[copy.size()-1].level = copy[index].level + 1;
 462                 }
 463
 464                 if (tree[index].level < maxLevel) {
 465                         for(it=tree[index].children.begin();it!=tree[index].children.end();it++){ //check your children
 466                                 fillOutTree(it->second, copy);
 467                         }
 468                 }
 469
 470         }
 471         catch(exception& e) {
 472                 m->errorOut(e, "PhyloTree", "fillOutTree");
 473                 exit(1);
 474         }
 475 }
 476 /**************************************************************************************************/
 477 string PhyloTree::getFullTaxonomy(string seqName) {
 478         try {
 479                 string tax = "";
 480
 481                 int currentNode = name2Taxonomy[seqName];
 482
 483                 while (tree[currentNode].parent != -1) {
 484                         tax = tree[currentNode].name + ";" + tax;
 485                         currentNode = tree[currentNode].parent;
 486                 }
 487
 488                 return tax;
 489         }
 490         catch(exception& e) {
 491                 m->errorOut(e, "PhyloTree", "getFullTaxonomy");
 492                 exit(1);
 493         }
 494 }
 495 /**************************************************************************************************/
 496
 497 void PhyloTree::print(ofstream& out, vector<TaxNode>& copy){
 498         try {
 499
 500                 //output mothur version
 501                 out << "#" << m->getVersion() << endl;
 502
 503                 out << copy.size() << endl;
 504
 505                 out << maxLevel << endl;
 506
 507                 for (int i = 0; i < copy.size(); i++) {
 508
 509                         out << copy[i].level << '\t'<< copy[i].name << '\t' << copy[i].children.size() << '\t';
 510
 511                         map<string,int>::iterator it;
 512                         for(it=copy[i].children.begin();it!=copy[i].children.end();it++){
 513                                 out << it->first << '\t' << it->second << '\t';
 514                         }
 515                         out << endl;
 516                 }
 517
 518                 out.close();
 519         }
 520         catch(exception& e) {
 521                 m->errorOut(e, "PhyloTree", "print");
 522                 exit(1);
 523         }
 524 }
 525 /**************************************************************************************************/
 526 void PhyloTree::printTreeNodes(string treefilename) {
 527         try {
 528
 529                 #ifdef USE_MPI
 530                         int pid;
 531                         MPI_Comm_rank(MPI_COMM_WORLD, &pid); //find out who we are
 532
 533                         if (pid == 0) {
 534
 535                 #endif
 536
 537                         ofstream outTree;
 538                         m->openOutputFile(treefilename, outTree);
 539
 540                         //output mothur version
 541                         outTree << "#" << m->getVersion() << endl;
 542
 543                         //print treenodes
 544                         outTree << tree.size() << endl;
 545                         for (int i = 0; i < tree.size(); i++) {
 546                                 outTree << tree[i].name << '\t' << tree[i].level << '\t' << tree[i].parent << endl;
 547                         }
 548
 549                         //print genus nodes
 550                         outTree << endl << uniqueTaxonomies.size() << endl;
 551                         set<int>::iterator it2;
 552                         for (it2=uniqueTaxonomies.begin(); it2!=uniqueTaxonomies.end(); it2++) {  outTree << *it2 << '\t' << tree[*it2].accessions.size() << endl;      }
 553                         outTree << endl;
 554
 555                         outTree.close();
 556
 557                 #ifdef USE_MPI
 558                         }
 559                 #endif
 560
 561
 562         }
 563         catch(exception& e) {
 564                 m->errorOut(e, "PhyloTree", "printTreeNodes");
 565                 exit(1);
 566         }
 567 }
 568 /**************************************************************************************************/
 569 TaxNode PhyloTree::get(int i ){
 570         try {
 571                 if (i < tree.size()) {  return tree[i];  }
 572                 else {  cout << i << '\t' << tree.size() << endl ; m->mothurOut("Mismatch with taxonomy and template files. Cannot continue."); m->mothurOutEndLine(); exit(1); }
 573         }
 574         catch(exception& e) {
 575                 m->errorOut(e, "PhyloTree", "get");
 576                 exit(1);
 577         }
 578 }
 579 /**************************************************************************************************/
 580 TaxNode PhyloTree::get(string seqName){
 581         try {
 582                 map<string, int>::iterator itFind = name2Taxonomy.find(seqName);
 583
 584                 if (itFind != name2Taxonomy.end()) {  return tree[name2Taxonomy[seqName]];  }
 585                 else { m->mothurOut("Cannot find " + seqName + ". Mismatch with taxonomy and template files. Cannot continue."); m->mothurOutEndLine(); exit(1);}
 586         }
 587         catch(exception& e) {
 588                 m->errorOut(e, "PhyloTree", "get");
 589                 exit(1);
 590         }
 591 }
 592 /**************************************************************************************************/
 593 string PhyloTree::getName(int i ){
 594         try {
 595                 if (i < tree.size()) {  return tree[i].name;     }
 596                 else { m->mothurOut("Mismatch with taxonomy and template files. Cannot continue."); m->mothurOutEndLine(); exit(1); }
 597         }
 598         catch(exception& e) {
 599                 m->errorOut(e, "PhyloTree", "get");
 600                 exit(1);
 601         }
 602 }
 603 /**************************************************************************************************/
 604 int PhyloTree::getGenusIndex(string seqName){
 605         try {
 606                 map<string, int>::iterator itFind = name2GenusNodeIndex.find(seqName);
 607
 608                 if (itFind != name2GenusNodeIndex.end()) {  return itFind->second;  }
 609                 else { m->mothurOut("Cannot find " + seqName + ". Could be a mismatch with taxonomy and template files. Cannot continue."); m->mothurOutEndLine(); exit(1);}
 610         }
 611         catch(exception& e) {
 612                 m->errorOut(e, "PhyloTree", "get");
 613                 exit(1);
 614         }
 615 }
 616 /**************************************************************************************************/
 617 bool PhyloTree::ErrorCheck(vector<string> templateFileNames){
 618         try {
 619
 620                 bool okay = true;
 621                 templateFileNames.push_back("unknown");
 622
 623                 map<string, int>::iterator itFind;
 624                 map<string, int> taxonomyFileNames = name2Taxonomy;
 625
 626         if (m->debug) { m->mothurOut("[DEBUG]: in error check. Numseqs in template = " + toString(templateFileNames.size()) + ". Numseqs in taxonomy = " + toString(taxonomyFileNames.size()) + ".\n"); }
 627
 628                 for (int i = 0; i < templateFileNames.size(); i++) {
 629                         itFind = taxonomyFileNames.find(templateFileNames[i]);
 630
 631                         if (itFind != taxonomyFileNames.end()) { //found it so erase it
 632                                 taxonomyFileNames.erase(itFind);
 633                         }else {
 634                                 m->mothurOut("'" +templateFileNames[i] + "' is in your template file and is not in your taxonomy file. Please correct."); m->mothurOutEndLine();
 635                                 okay = false;
 636                         }
 637
 638                         //templateFileNames.erase(templateFileNames.begin()+i);
 639                         //i--;
 640                 }
 641                 templateFileNames.clear();
 642
 643                 if (taxonomyFileNames.size() > 0) { //there are names in tax file that are not in template
 644                         okay = false;
 645
 646                         for (itFind = taxonomyFileNames.begin(); itFind != taxonomyFileNames.end(); itFind++) {
 647                                 m->mothurOut(itFind->first + " is in your taxonomy file and is not in your template file. Please correct."); m->mothurOutEndLine();
 648                         }
 649                 }
 650
 651                 return okay;
 652         }
 653         catch(exception& e) {
 654                 m->errorOut(e, "PhyloTree", "ErrorCheck");
 655                 exit(1);
 656         }
 657 }
 658 /**************************************************************************************************/
 659
 660
 661
 662