classifyseqscommand.cpp

   1 /*
   2  *  classifyseqscommand.cpp
   3  *  Mothur
   4  *
   5  *  Created by westcott on 11/2/09.
   6  *  Copyright 2009 Schloss Lab. All rights reserved.
   7  *
   8  */
   9
  10 #include "classifyseqscommand.h"
  11 #include "sequence.hpp"
  12 #include "bayesian.h"
  13 #include "phylotree.h"
  14 #include "knn.h"
  15
  16 //**********************************************************************************************************************
  17
  18 ClassifySeqsCommand::ClassifySeqsCommand(string option){
  19         try {
  20                 abort = false;
  21
  22                 //allow user to run help
  23                 if(option == "help") { help(); abort = true; }
  24
  25                 else {
  26
  27                         //valid paramters for this command
  28                         string AlignArray[] =  {"template","fasta","name","search","ksize","method","processors","taxonomy","match","mismatch","gapopen","gapextend","numwanted","cutoff","probs","iters"};
  29                         vector<string> myArray (AlignArray, AlignArray+(sizeof(AlignArray)/sizeof(string)));
  30
  31                         OptionParser parser(option);
  32                         map<string, string> parameters = parser.getParameters();
  33
  34                         ValidParameters validParameter;
  35
  36                         //check to make sure all parameters are valid for command
  37                         for (map<string, string>::iterator it = parameters.begin(); it != parameters.end(); it++) {
  38                                 if (validParameter.isValidParameter(it->first, myArray, it->second) != true) {  abort = true;  }
  39                         }
  40
  41                         //check for required parameters
  42                         templateFileName = validParameter.validFile(parameters, "template", true);
  43                         if (templateFileName == "not found") {
  44                                 mothurOut("template is a required parameter for the classify.seqs command.");
  45                                 mothurOutEndLine();
  46                                 abort = true;
  47                         }
  48                         else if (templateFileName == "not open") { abort = true; }
  49
  50                         fastaFileName = validParameter.validFile(parameters, "fasta", false);
  51                         if (fastaFileName == "not found") { mothurOut("fasta is a required parameter for the classify.seqs command."); mothurOutEndLine(); abort = true;  }
  52                         else {
  53                                 splitAtDash(fastaFileName, fastaFileNames);
  54
  55                                 //go through files and make sure they are good, if not, then disregard them
  56                                 for (int i = 0; i < fastaFileNames.size(); i++) {
  57                                         int ableToOpen;
  58                                         ifstream in;
  59                                         ableToOpen = openInputFile(fastaFileNames[i], in);
  60                                         if (ableToOpen == 1) {
  61                                                 mothurOut(fastaFileNames[i] + " will be disregarded."); mothurOutEndLine();
  62                                                 //erase from file list
  63                                                 fastaFileNames.erase(fastaFileNames.begin()+i);
  64                                                 i--;
  65                                         }
  66                                         in.close();
  67                                 }
  68
  69                                 //make sure there is at least one valid file left
  70                                 if (fastaFileNames.size() == 0) { mothurOut("no valid files."); mothurOutEndLine(); abort = true; }
  71                         }
  72
  73
  74                         taxonomyFileName = validParameter.validFile(parameters, "taxonomy", true);
  75                         if (taxonomyFileName == "not found") {
  76                                 mothurOut("taxonomy is a required parameter for the classify.seqs command.");
  77                                 mothurOutEndLine();
  78                                 abort = true;
  79                         }
  80                         else if (taxonomyFileName == "not open") { abort = true; }
  81
  82
  83                         namefile = validParameter.validFile(parameters, "name", false);
  84                         if (namefile == "not found") { namefile = "";  }
  85
  86                         else {
  87                                 splitAtDash(namefile, namefileNames);
  88
  89                                 //go through files and make sure they are good, if not, then disregard them
  90                                 for (int i = 0; i < namefileNames.size(); i++) {
  91                                         int ableToOpen;
  92                                         ifstream in;
  93                                         ableToOpen = openInputFile(namefileNames[i], in);
  94                                         if (ableToOpen == 1) {  mothurOut("Unable to match name file with fasta file."); mothurOutEndLine(); abort = true;      }
  95                                         in.close();
  96                                 }
  97                         }
  98
  99                         if (namefile != "") {
 100                                 if (namefileNames.size() != fastaFileNames.size()) { abort = true; mothurOut("If you provide a name file, you must have one for each fasta file."); mothurOutEndLine(); }
 101                         }
 102
 103                         //check for optional parameter and set defaults
 104                         // ...at some point should added some additional type checking...
 105                         string temp;
 106                         temp = validParameter.validFile(parameters, "ksize", false);            if (temp == "not found"){       temp = "8";                             }
 107                         convert(temp, kmerSize);
 108
 109                         temp = validParameter.validFile(parameters, "processors", false);       if (temp == "not found"){       temp = "1";                             }
 110                         convert(temp, processors);
 111
 112                         search = validParameter.validFile(parameters, "search", false);         if (search == "not found"){     search = "kmer";                }
 113
 114                         method = validParameter.validFile(parameters, "method", false);         if (method == "not found"){     method = "bayesian";    }
 115
 116                         temp = validParameter.validFile(parameters, "match", false);            if (temp == "not found"){       temp = "1.0";                   }
 117                         convert(temp, match);
 118
 119                         temp = validParameter.validFile(parameters, "mismatch", false);         if (temp == "not found"){       temp = "-1.0";                  }
 120                         convert(temp, misMatch);
 121
 122                         temp = validParameter.validFile(parameters, "gapopen", false);          if (temp == "not found"){       temp = "-2.0";                  }
 123                         convert(temp, gapOpen);
 124
 125                         temp = validParameter.validFile(parameters, "gapextend", false);        if (temp == "not found"){       temp = "-1.0";                  }
 126                         convert(temp, gapExtend);
 127
 128                         temp = validParameter.validFile(parameters, "numwanted", false);        if (temp == "not found"){       temp = "10";                    }
 129                         convert(temp, numWanted);
 130
 131                         temp = validParameter.validFile(parameters, "cutoff", false);           if (temp == "not found"){       temp = "0";                             }
 132                         convert(temp, cutoff);
 133
 134                         temp = validParameter.validFile(parameters, "probs", false);            if (temp == "not found"){       temp = "true";                  }
 135                         probs = isTrue(temp);
 136
 137                         temp = validParameter.validFile(parameters, "iters", false);            if (temp == "not found") { temp = "100";                        }
 138                         convert(temp, iters);
 139
 140
 141
 142                         if ((method == "bayesian") && (search != "kmer"))  {
 143                                 mothurOut("The bayesian method requires the kmer search." + search + "will be disregarded." ); mothurOutEndLine();
 144                                 search = "kmer";
 145                         }
 146                 }
 147
 148         }
 149         catch(exception& e) {
 150                 errorOut(e, "ClassifySeqsCommand", "ClassifySeqsCommand");
 151                 exit(1);
 152         }
 153 }
 154
 155 //**********************************************************************************************************************
 156
 157 ClassifySeqsCommand::~ClassifySeqsCommand(){
 158
 159         if (abort == false) {
 160                 for (int i = 0; i < lines.size(); i++) {  delete lines[i];  }  lines.clear();
 161         }
 162 }
 163
 164 //**********************************************************************************************************************
 165
 166 void ClassifySeqsCommand::help(){
 167         try {
 168                 mothurOut("The classify.seqs command reads a fasta file containing sequences and creates a .taxonomy file and a .tax.summary file.\n");
 169                 mothurOut("The classify.seqs command parameters are template, fasta, search, ksize, method, taxonomy, processors, match, mismatch, gapopen, gapextend, numwanted and probs.\n");
 170                 mothurOut("The template, fasta and taxonomy parameters are required. You may enter multiple fasta files by separating their names with dashes. ie. fasta=abrecovery.fasta-amzon.fasta \n");
 171                 mothurOut("The search parameter allows you to specify the method to find most similar template.  Your options are: suffix, kmer and blast. The default is kmer.\n");
 172                 mothurOut("The method parameter allows you to specify classification method to use.  Your options are: bayesian and knn. The default is bayesian.\n");
 173                 mothurOut("The ksize parameter allows you to specify the kmer size for finding most similar template to candidate.  The default is 8.\n");
 174                 mothurOut("The processors parameter allows you to specify the number of processors to use. The default is 1.\n");
 175                 mothurOut("The match parameter allows you to specify the bonus for having the same base. The default is 1.0.\n");
 176                 mothurOut("The mistmatch parameter allows you to specify the penalty for having different bases.  The default is -1.0.\n");
 177                 mothurOut("The gapopen parameter allows you to specify the penalty for opening a gap in an alignment. The default is -2.0.\n");
 178                 mothurOut("The gapextend parameter allows you to specify the penalty for extending a gap in an alignment.  The default is -1.0.\n");
 179                 mothurOut("The numwanted parameter allows you to specify the number of sequence matches you want with the knn method.  The default is 10.\n");
 180                 mothurOut("The cutoff parameter allows you to specify a bootstrap confidence threshold for your taxonomy.  The default is 0.\n");
 181                 mothurOut("The probs parameter shut off the bootstrapping results for the bayesian method. The default is true, meaning you want the bootstrapping to be run.\n");
 182                 mothurOut("The iters parameter allows you to specify how many iterations to do when calculating the bootstrap confidence score for your taxonomy with the bayesian method.  The default is 100.\n");
 183                 mothurOut("The classify.seqs command should be in the following format: \n");
 184                 mothurOut("classify.seqs(template=yourTemplateFile, fasta=yourFastaFile, method=yourClassificationMethod, search=yourSearchmethod, ksize=yourKmerSize, taxonomy=yourTaxonomyFile, processors=yourProcessors) \n");
 185                 mothurOut("Example classify.seqs(fasta=amazon.fasta, template=core.filtered, method=knn, search=gotoh, ksize=8, processors=2)\n");
 186                 mothurOut("The .taxonomy file consists of 2 columns: 1 = your sequence name, 2 = the taxonomy for your sequence. \n");
 187                 mothurOut("The .tax.summary is a summary of the different taxonomies represented in your fasta file. \n");
 188                 mothurOut("Note: No spaces between parameter labels (i.e. fasta), '=' and parameters (i.e.yourFastaFile).\n\n");
 189         }
 190         catch(exception& e) {
 191                 errorOut(e, "ClassifySeqsCommand", "help");
 192                 exit(1);
 193         }
 194 }
 195
 196
 197 //**********************************************************************************************************************
 198
 199 int ClassifySeqsCommand::execute(){
 200         try {
 201                 if (abort == true) {    return 0;       }
 202
 203                 if(method == "bayesian"){       classify = new Bayesian(taxonomyFileName, templateFileName, search, kmerSize, cutoff, iters);           }
 204                 else if(method == "knn"){       classify = new Knn(taxonomyFileName, templateFileName, search, kmerSize, gapOpen, gapExtend, match, misMatch, numWanted);                               }
 205                 else {
 206                         mothurOut(search + " is not a valid method option. I will run the command using bayesian.");
 207                         mothurOutEndLine();
 208                         classify = new Bayesian(taxonomyFileName, templateFileName, search, kmerSize, cutoff, iters);
 209                 }
 210
 211
 212                 for (int s = 0; s < fastaFileNames.size(); s++) {
 213
 214                         //read namefile
 215                         if(namefile != "") {
 216                                 nameMap.clear(); //remove old names
 217
 218                                 ifstream inNames;
 219                                 openInputFile(namefileNames[s], inNames);
 220
 221                                 string firstCol, secondCol;
 222                                 while(!inNames.eof()) {
 223                                         inNames >> firstCol >> secondCol; gobble(inNames);
 224                                         nameMap[firstCol] = getNumNames(secondCol);  //ex. seq1 seq1,seq3,seq5 -> seq1 = 3.
 225                                 }
 226                                 inNames.close();
 227                         }
 228
 229                         mothurOut("Classifying sequences from " + fastaFileNames[s] + " ..." ); mothurOutEndLine();
 230                         string newTaxonomyFile = getRootName(fastaFileNames[s]) + getRootName(taxonomyFileName) + "taxonomy";
 231                         string tempTaxonomyFile = getRootName(fastaFileNames[s]) + "taxonomy.temp";
 232                         string taxSummary = getRootName(fastaFileNames[s]) + getRootName(taxonomyFileName) + "tax.summary";
 233
 234                         int start = time(NULL);
 235                         int numFastaSeqs = 0;
 236                         for (int i = 0; i < lines.size(); i++) {  delete lines[i];  }  lines.clear();
 237
 238 #if defined (__APPLE__) || (__MACH__) || (linux) || (__linux)
 239                         if(processors == 1){
 240                                 ifstream inFASTA;
 241                                 openInputFile(fastaFileNames[s], inFASTA);
 242                                 numFastaSeqs=count(istreambuf_iterator<char>(inFASTA),istreambuf_iterator<char>(), '>');
 243                                 inFASTA.close();
 244
 245                                 lines.push_back(new linePair(0, numFastaSeqs));
 246
 247                                 driver(lines[0], newTaxonomyFile, tempTaxonomyFile, fastaFileNames[s]);
 248                         }
 249                         else{
 250                                 vector<int> positions;
 251                                 processIDS.resize(0);
 252
 253                                 ifstream inFASTA;
 254                                 openInputFile(fastaFileNames[s], inFASTA);
 255
 256                                 string input;
 257                                 while(!inFASTA.eof()){
 258                                         input = getline(inFASTA);
 259                                         if (input.length() != 0) {
 260                                                 if(input[0] == '>'){    int pos = inFASTA.tellg(); positions.push_back(pos - input.length() - 1);       }
 261                                         }
 262                                 }
 263                                 inFASTA.close();
 264
 265                                 numFastaSeqs = positions.size();
 266
 267                                 int numSeqsPerProcessor = numFastaSeqs / processors;
 268
 269                                 for (int i = 0; i < processors; i++) {
 270                                         int startPos = positions[ i * numSeqsPerProcessor ];
 271                                         if(i == processors - 1){
 272                                                 numSeqsPerProcessor = numFastaSeqs - i * numSeqsPerProcessor;
 273                                         }
 274                                         lines.push_back(new linePair(startPos, numSeqsPerProcessor));
 275                                 }
 276                                 createProcesses(newTaxonomyFile, tempTaxonomyFile, fastaFileNames[s]);
 277
 278                                 rename((newTaxonomyFile + toString(processIDS[0]) + ".temp").c_str(), newTaxonomyFile.c_str());
 279                                 rename((tempTaxonomyFile + toString(processIDS[0]) + ".temp").c_str(), tempTaxonomyFile.c_str());
 280
 281                                 for(int i=1;i<processors;i++){
 282                                         appendTaxFiles((newTaxonomyFile + toString(processIDS[i]) + ".temp"), newTaxonomyFile);
 283                                         appendTaxFiles((tempTaxonomyFile + toString(processIDS[i]) + ".temp"), tempTaxonomyFile);
 284                                         remove((newTaxonomyFile + toString(processIDS[i]) + ".temp").c_str());
 285                                         remove((tempTaxonomyFile + toString(processIDS[i]) + ".temp").c_str());
 286                                 }
 287
 288                         }
 289 #else
 290                         ifstream inFASTA;
 291                         openInputFile(fastaFileNames[s], inFASTA);
 292                         numFastaSeqs=count(istreambuf_iterator<char>(inFASTA),istreambuf_iterator<char>(), '>');
 293                         inFASTA.close();
 294
 295                         lines.push_back(new linePair(0, numFastaSeqs));
 296
 297                         driver(lines[0], newTaxonomyFile, tempTaxonomyFile, fastaFileNames[s]);
 298 #endif
 299                         //make taxonomy tree from new taxonomy file
 300                         PhyloTree taxaBrowser;
 301
 302                         ifstream in;
 303                         openInputFile(tempTaxonomyFile, in);
 304
 305                         //read in users taxonomy file and add sequences to tree
 306                         string name, taxon;
 307                         while(!in.eof()){
 308                                 in >> name >> taxon; gobble(in);
 309
 310                                 if (namefile != "") {
 311                                         itNames = nameMap.find(name);
 312
 313                                         if (itNames == nameMap.end()) {
 314                                                 mothurOut(name + " is not in your name file please correct."); mothurOutEndLine(); exit(1);
 315                                         }else{
 316                                                 for (int i = 0; i < itNames->second; i++) {
 317                                                         taxaBrowser.addSeqToTree(name+toString(i), taxon);  //add it as many times as there are identical seqs
 318                                                 }
 319                                         }
 320                                 }else {  taxaBrowser.addSeqToTree(name, taxon);  } //add it once
 321                         }
 322                         in.close();
 323
 324                         taxaBrowser.assignHeirarchyIDs(0);
 325
 326                         taxaBrowser.binUnclassified();
 327
 328                         remove(tempTaxonomyFile.c_str());
 329
 330                         //print summary file
 331                         ofstream outTaxTree;
 332                         openOutputFile(taxSummary, outTaxTree);
 333                         taxaBrowser.print(outTaxTree);
 334                         outTaxTree.close();
 335
 336                         //output taxonomy with the unclassified bins added
 337                         ifstream inTax;
 338                         openInputFile(newTaxonomyFile, inTax);
 339
 340                         ofstream outTax;
 341                         string unclass = newTaxonomyFile + ".unclass.temp";
 342                         openOutputFile(unclass, outTax);
 343
 344                         //get maxLevel from phylotree so you know how many 'unclassified's to add
 345                         int maxLevel = taxaBrowser.getMaxLevel();
 346
 347                         //read taxfile - this reading and rewriting is done to preserve the confidence sscores.
 348                         while (!inTax.eof()) {
 349                                 inTax >> name >> taxon; gobble(inTax);
 350
 351                                 string newTax = addUnclassifieds(taxon, maxLevel);
 352
 353                                 outTax << name << '\t' << newTax << endl;
 354                         }
 355                         inTax.close();
 356                         outTax.close();
 357
 358                         remove(newTaxonomyFile.c_str());
 359                         rename(unclass.c_str(), newTaxonomyFile.c_str());
 360
 361                         mothurOutEndLine();
 362                         mothurOut("It took " + toString(time(NULL) - start) + " secs to classify " + toString(numFastaSeqs) + " sequences."); mothurOutEndLine(); mothurOutEndLine();
 363                 }
 364
 365                 delete classify;
 366                 return 0;
 367         }
 368         catch(exception& e) {
 369                 errorOut(e, "ClassifySeqsCommand", "execute");
 370                 exit(1);
 371         }
 372 }
 373
 374 /**************************************************************************************************/
 375 string ClassifySeqsCommand::addUnclassifieds(string tax, int maxlevel) {
 376         try{
 377                 string newTax, taxon;
 378                 int level = 0;
 379
 380                 //keep what you have counting the levels
 381                 while (tax.find_first_of(';') != -1) {
 382                         //get taxon
 383                         taxon = tax.substr(0,tax.find_first_of(';'))+';';
 384                         tax = tax.substr(tax.find_first_of(';')+1, tax.length());
 385                         newTax += taxon;
 386                         level++;
 387                 }
 388
 389                 //add "unclassified" until you reach maxLevel
 390                 while (level < maxlevel) {
 391                         newTax += "unclassified;";
 392                         level++;
 393                 }
 394
 395                 return newTax;
 396         }
 397         catch(exception& e) {
 398                 errorOut(e, "ClassifySeqsCommand", "addUnclassifieds");
 399                 exit(1);
 400         }
 401 }
 402
 403 /**************************************************************************************************/
 404
 405 void ClassifySeqsCommand::createProcesses(string taxFileName, string tempTaxFile, string filename) {
 406         try {
 407 #if defined (__APPLE__) || (__MACH__) || (linux) || (__linux)
 408                 int process = 0;
 409                 //              processIDS.resize(0);
 410
 411                 //loop through and create all the processes you want
 412                 while (process != processors) {
 413                         int pid = fork();
 414
 415                         if (pid > 0) {
 416                                 processIDS.push_back(pid);  //create map from line number to pid so you can append files in correct order later
 417                                 process++;
 418                         }else if (pid == 0){
 419                                 driver(lines[process], taxFileName + toString(getpid()) + ".temp", tempTaxFile + toString(getpid()) + ".temp", filename);
 420                                 exit(0);
 421                         }else { mothurOut("unable to spawn the necessary processes."); mothurOutEndLine(); exit(0); }
 422                 }
 423
 424                 //force parent to wait until all the processes are done
 425                 for (int i=0;i<processors;i++) {
 426                         int temp = processIDS[i];
 427                         wait(&temp);
 428                 }
 429 #endif
 430         }
 431         catch(exception& e) {
 432                 errorOut(e, "ClassifySeqsCommand", "createProcesses");
 433                 exit(1);
 434         }
 435 }
 436 /**************************************************************************************************/
 437
 438 void ClassifySeqsCommand::appendTaxFiles(string temp, string filename) {
 439         try{
 440
 441                 ofstream output;
 442                 ifstream input;
 443                 openOutputFileAppend(filename, output);
 444                 openInputFile(temp, input);
 445
 446                 while(char c = input.get()){
 447                         if(input.eof())         {       break;                  }
 448                         else                            {       output << c;    }
 449                 }
 450
 451                 input.close();
 452                 output.close();
 453         }
 454         catch(exception& e) {
 455                 errorOut(e, "ClassifySeqsCommand", "appendTaxFiles");
 456                 exit(1);
 457         }
 458 }
 459
 460 //**********************************************************************************************************************
 461
 462 int ClassifySeqsCommand::driver(linePair* line, string taxFName, string tempTFName, string filename){
 463         try {
 464                 ofstream outTax;
 465                 openOutputFile(taxFName, outTax);
 466
 467                 ofstream outTaxSimple;
 468                 openOutputFile(tempTFName, outTaxSimple);
 469
 470                 ifstream inFASTA;
 471                 openInputFile(filename, inFASTA);
 472
 473                 inFASTA.seekg(line->start);
 474
 475                 string taxonomy;
 476
 477                 for(int i=0;i<line->numSeqs;i++){
 478
 479                         Sequence* candidateSeq = new Sequence(inFASTA);
 480
 481                         if (candidateSeq->getName() != "") {
 482                                 taxonomy = classify->getTaxonomy(candidateSeq);
 483
 484                                 if (taxonomy != "bad seq") {
 485                                         //output confidence scores or not
 486                                         if (probs) {
 487                                                 outTax << candidateSeq->getName() << '\t' << taxonomy << endl;
 488                                         }else{
 489                                                 outTax << candidateSeq->getName() << '\t' << classify->getSimpleTax() << endl;
 490                                                 cout << classify->getSimpleTax() << endl;
 491                                         }
 492
 493                                         outTaxSimple << candidateSeq->getName() << '\t' << classify->getSimpleTax() << endl;
 494                                 }
 495                         }
 496                         delete candidateSeq;
 497
 498                         if((i+1) % 100 == 0){
 499                                 mothurOut("Classifying sequence " + toString(i+1)); mothurOutEndLine();
 500                         }
 501                 }
 502
 503                 inFASTA.close();
 504                 outTax.close();
 505                 outTaxSimple.close();
 506
 507                 return 1;
 508         }
 509         catch(exception& e) {
 510                 errorOut(e, "ClassifySeqsCommand", "driver");
 511                 exit(1);
 512         }
 513 }
 514
 515 /**************************************************************************************************/