]> git.donarmstrong.com Git - mothur.git/blob - classifyseqscommand.cpp
added distance search method to classify.seqs
[mothur.git] / classifyseqscommand.cpp
1 /*
2  *  classifyseqscommand.cpp
3  *  Mothur
4  *
5  *  Created by westcott on 11/2/09.
6  *  Copyright 2009 Schloss Lab. All rights reserved.
7  *
8  */
9
10 #include "classifyseqscommand.h"
11 #include "sequence.hpp"
12 #include "bayesian.h"
13 #include "phylotree.h"
14 #include "knn.h"
15
16 //**********************************************************************************************************************
17
18 ClassifySeqsCommand::ClassifySeqsCommand(string option){
19         try {
20                 abort = false;
21                 
22                 //allow user to run help
23                 if(option == "help") { help(); abort = true; }
24                 
25                 else {
26                         
27                         //valid paramters for this command
28                         string AlignArray[] =  {"template","fasta","name","search","ksize","method","processors","taxonomy","match","mismatch","gapopen","gapextend","numwanted","cutoff","probs","iters", "outputdir","inputdir"};
29                         vector<string> myArray (AlignArray, AlignArray+(sizeof(AlignArray)/sizeof(string)));
30                         
31                         OptionParser parser(option);
32                         map<string, string> parameters = parser.getParameters(); 
33                         
34                         ValidParameters validParameter;
35                         map<string, string>::iterator it;
36                         
37                         //check to make sure all parameters are valid for command
38                         for (it = parameters.begin(); it != parameters.end(); it++) { 
39                                 if (validParameter.isValidParameter(it->first, myArray, it->second) != true) {  abort = true;  }
40                         }
41                         
42                         //if the user changes the output directory command factory will send this info to us in the output parameter 
43                         outputDir = validParameter.validFile(parameters, "outputdir", false);           if (outputDir == "not found"){  outputDir = "";         }
44                         
45                         //if the user changes the input directory command factory will send this info to us in the output parameter 
46                         string inputDir = validParameter.validFile(parameters, "inputdir", false);              
47                         if (inputDir == "not found"){   inputDir = "";          }
48                         else {
49                                 string path;
50                                 it = parameters.find("template");
51                                 //user has given a template file
52                                 if(it != parameters.end()){ 
53                                         path = hasPath(it->second);
54                                         //if the user has not given a path then, add inputdir. else leave path alone.
55                                         if (path == "") {       parameters["template"] = inputDir + it->second;         }
56                                 }
57                                 
58                                 it = parameters.find("taxonomy");
59                                 //user has given a template file
60                                 if(it != parameters.end()){ 
61                                         path = hasPath(it->second);
62                                         //if the user has not given a path then, add inputdir. else leave path alone.
63                                         if (path == "") {       parameters["taxonomy"] = inputDir + it->second;         }
64                                 }
65                         }
66
67                         //check for required parameters
68                         templateFileName = validParameter.validFile(parameters, "template", true);
69                         if (templateFileName == "not found") { 
70                                 mothurOut("template is a required parameter for the classify.seqs command."); 
71                                 mothurOutEndLine();
72                                 abort = true; 
73                         }
74                         else if (templateFileName == "not open") { abort = true; }      
75                         
76                         fastaFileName = validParameter.validFile(parameters, "fasta", false);
77                         if (fastaFileName == "not found") { mothurOut("fasta is a required parameter for the classify.seqs command."); mothurOutEndLine(); abort = true;  }
78                         else { 
79                                 splitAtDash(fastaFileName, fastaFileNames);
80                                 
81                                 //go through files and make sure they are good, if not, then disregard them
82                                 for (int i = 0; i < fastaFileNames.size(); i++) {
83                                         if (inputDir != "") {
84                                                 string path = hasPath(fastaFileNames[i]);
85                                                 //if the user has not given a path then, add inputdir. else leave path alone.
86                                                 if (path == "") {       fastaFileNames[i] = inputDir + fastaFileNames[i];               }
87                                         }
88                                         
89                                         int ableToOpen;
90                                         ifstream in;
91                                         ableToOpen = openInputFile(fastaFileNames[i], in);
92                                         if (ableToOpen == 1) { 
93                                                 mothurOut(fastaFileNames[i] + " will be disregarded."); mothurOutEndLine(); 
94                                                 //erase from file list
95                                                 fastaFileNames.erase(fastaFileNames.begin()+i);
96                                                 i--;
97                                         }
98                                         in.close();
99                                 }
100                                 
101                                 //make sure there is at least one valid file left
102                                 if (fastaFileNames.size() == 0) { mothurOut("no valid files."); mothurOutEndLine(); abort = true; }
103                         }
104
105                         
106                         taxonomyFileName = validParameter.validFile(parameters, "taxonomy", true);
107                         if (taxonomyFileName == "not found") { 
108                                 mothurOut("taxonomy is a required parameter for the classify.seqs command."); 
109                                 mothurOutEndLine();
110                                 abort = true; 
111                         }
112                         else if (taxonomyFileName == "not open") { abort = true; }      
113                         
114                         
115                         namefile = validParameter.validFile(parameters, "name", false);
116                         if (namefile == "not found") { namefile = "";  }
117
118                         else { 
119                                 splitAtDash(namefile, namefileNames);
120                                 
121                                 //go through files and make sure they are good, if not, then disregard them
122                                 for (int i = 0; i < namefileNames.size(); i++) {
123                                         if (inputDir != "") {
124                                                 string path = hasPath(namefileNames[i]);
125                                                 //if the user has not given a path then, add inputdir. else leave path alone.
126                                                 if (path == "") {       namefileNames[i] = inputDir + namefileNames[i];         }
127                                         }
128
129                                         int ableToOpen;
130                                         ifstream in;
131                                         ableToOpen = openInputFile(namefileNames[i], in);
132                                         if (ableToOpen == 1) {  mothurOut("Unable to match name file with fasta file."); mothurOutEndLine(); abort = true;      }
133                                         in.close();
134                                 }
135                         }
136
137                         if (namefile != "") {
138                                 if (namefileNames.size() != fastaFileNames.size()) { abort = true; mothurOut("If you provide a name file, you must have one for each fasta file."); mothurOutEndLine(); }
139                         }
140                         
141                         //check for optional parameter and set defaults
142                         // ...at some point should added some additional type checking...
143                         string temp;
144                         temp = validParameter.validFile(parameters, "ksize", false);            if (temp == "not found"){       temp = "8";                             }
145                         convert(temp, kmerSize); 
146                         
147                         temp = validParameter.validFile(parameters, "processors", false);       if (temp == "not found"){       temp = "1";                             }
148                         convert(temp, processors); 
149                         
150                         search = validParameter.validFile(parameters, "search", false);         if (search == "not found"){     search = "kmer";                }
151                         
152                         method = validParameter.validFile(parameters, "method", false);         if (method == "not found"){     method = "bayesian";    }
153                         
154                         temp = validParameter.validFile(parameters, "match", false);            if (temp == "not found"){       temp = "1.0";                   }
155                         convert(temp, match);  
156                         
157                         temp = validParameter.validFile(parameters, "mismatch", false);         if (temp == "not found"){       temp = "-1.0";                  }
158                         convert(temp, misMatch);  
159                         
160                         temp = validParameter.validFile(parameters, "gapopen", false);          if (temp == "not found"){       temp = "-2.0";                  }
161                         convert(temp, gapOpen);  
162                         
163                         temp = validParameter.validFile(parameters, "gapextend", false);        if (temp == "not found"){       temp = "-1.0";                  }
164                         convert(temp, gapExtend); 
165                         
166                         temp = validParameter.validFile(parameters, "numwanted", false);        if (temp == "not found"){       temp = "10";                    }
167                         convert(temp, numWanted);
168                         
169                         temp = validParameter.validFile(parameters, "cutoff", false);           if (temp == "not found"){       temp = "0";                             }
170                         convert(temp, cutoff);
171                         
172                         temp = validParameter.validFile(parameters, "probs", false);            if (temp == "not found"){       temp = "true";                  }
173                         probs = isTrue(temp);
174                         
175                         temp = validParameter.validFile(parameters, "iters", false);            if (temp == "not found") { temp = "100";                        }
176                         convert(temp, iters); 
177
178
179                         
180                         if ((method == "bayesian") && (search != "kmer"))  { 
181                                 mothurOut("The bayesian method requires the kmer search." + search + "will be disregarded." ); mothurOutEndLine();
182                                 search = "kmer";
183                         }
184                 }
185                 
186         }
187         catch(exception& e) {
188                 errorOut(e, "ClassifySeqsCommand", "ClassifySeqsCommand");
189                 exit(1);
190         }
191 }
192
193 //**********************************************************************************************************************
194
195 ClassifySeqsCommand::~ClassifySeqsCommand(){    
196
197         if (abort == false) {
198                 for (int i = 0; i < lines.size(); i++) {  delete lines[i];  }  lines.clear();
199         }
200 }
201
202 //**********************************************************************************************************************
203
204 void ClassifySeqsCommand::help(){
205         try {
206                 mothurOut("The classify.seqs command reads a fasta file containing sequences and creates a .taxonomy file and a .tax.summary file.\n");
207                 mothurOut("The classify.seqs command parameters are template, fasta, name, search, ksize, method, taxonomy, processors, match, mismatch, gapopen, gapextend, numwanted and probs.\n");
208                 mothurOut("The template, fasta and taxonomy parameters are required. You may enter multiple fasta files by separating their names with dashes. ie. fasta=abrecovery.fasta-amzon.fasta \n");
209                 mothurOut("The search parameter allows you to specify the method to find most similar template.  Your options are: suffix, kmer, blast and distance. The default is kmer.\n");
210                 mothurOut("The name parameter allows you add a names file with your fasta file, if you enter multiple fasta files, you must enter matching names files for them.\n");
211                 mothurOut("The method parameter allows you to specify classification method to use.  Your options are: bayesian and knn. The default is bayesian.\n");
212                 mothurOut("The ksize parameter allows you to specify the kmer size for finding most similar template to candidate.  The default is 8.\n");
213                 mothurOut("The processors parameter allows you to specify the number of processors to use. The default is 1.\n");
214                 mothurOut("The match parameter allows you to specify the bonus for having the same base. The default is 1.0.\n");
215                 mothurOut("The mistmatch parameter allows you to specify the penalty for having different bases.  The default is -1.0.\n");
216                 mothurOut("The gapopen parameter allows you to specify the penalty for opening a gap in an alignment. The default is -2.0.\n");
217                 mothurOut("The gapextend parameter allows you to specify the penalty for extending a gap in an alignment.  The default is -1.0.\n");
218                 mothurOut("The numwanted parameter allows you to specify the number of sequence matches you want with the knn method.  The default is 10.\n");
219                 mothurOut("The cutoff parameter allows you to specify a bootstrap confidence threshold for your taxonomy.  The default is 0.\n");
220                 mothurOut("The probs parameter shut off the bootstrapping results for the bayesian method. The default is true, meaning you want the bootstrapping to be run.\n");
221                 mothurOut("The iters parameter allows you to specify how many iterations to do when calculating the bootstrap confidence score for your taxonomy with the bayesian method.  The default is 100.\n");
222                 mothurOut("The classify.seqs command should be in the following format: \n");
223                 mothurOut("classify.seqs(template=yourTemplateFile, fasta=yourFastaFile, method=yourClassificationMethod, search=yourSearchmethod, ksize=yourKmerSize, taxonomy=yourTaxonomyFile, processors=yourProcessors) \n");
224                 mothurOut("Example classify.seqs(fasta=amazon.fasta, template=core.filtered, method=knn, search=gotoh, ksize=8, processors=2)\n");
225                 mothurOut("The .taxonomy file consists of 2 columns: 1 = your sequence name, 2 = the taxonomy for your sequence. \n");
226                 mothurOut("The .tax.summary is a summary of the different taxonomies represented in your fasta file. \n");
227                 mothurOut("Note: No spaces between parameter labels (i.e. fasta), '=' and parameters (i.e.yourFastaFile).\n\n");
228         }
229         catch(exception& e) {
230                 errorOut(e, "ClassifySeqsCommand", "help");
231                 exit(1);
232         }
233 }
234
235
236 //**********************************************************************************************************************
237
238 int ClassifySeqsCommand::execute(){
239         try {
240                 if (abort == true) {    return 0;       }
241                 
242                 if(method == "bayesian"){       classify = new Bayesian(taxonomyFileName, templateFileName, search, kmerSize, cutoff, iters);           }
243                 else if(method == "knn"){       classify = new Knn(taxonomyFileName, templateFileName, search, kmerSize, gapOpen, gapExtend, match, misMatch, numWanted);                               }
244                 else {
245                         mothurOut(search + " is not a valid method option. I will run the command using bayesian.");
246                         mothurOutEndLine();
247                         classify = new Bayesian(taxonomyFileName, templateFileName, search, kmerSize, cutoff, iters);   
248                 }
249
250                                 
251                 for (int s = 0; s < fastaFileNames.size(); s++) {
252                 
253                         //read namefile
254                         if(namefile != "") {
255                                 nameMap.clear(); //remove old names
256                                 
257                                 ifstream inNames;
258                                 openInputFile(namefileNames[s], inNames);
259                                 
260                                 string firstCol, secondCol;
261                                 while(!inNames.eof()) {
262                                         inNames >> firstCol >> secondCol; gobble(inNames);
263                                         nameMap[firstCol] = getNumNames(secondCol);  //ex. seq1 seq1,seq3,seq5 -> seq1 = 3.
264                                 }
265                                 inNames.close();
266                         }
267                 
268                         mothurOut("Classifying sequences from " + fastaFileNames[s] + " ..." ); mothurOutEndLine();
269                         
270                         if (outputDir == "") { outputDir += hasPath(fastaFileNames[s]); }
271                         string newTaxonomyFile = outputDir + getRootName(getSimpleName(fastaFileNames[s])) + getRootName(getSimpleName(taxonomyFileName)) + "taxonomy";
272                         string tempTaxonomyFile = outputDir + getRootName(getSimpleName(fastaFileNames[s])) + "taxonomy.temp";
273                         string taxSummary = outputDir + getRootName(getSimpleName(fastaFileNames[s])) + getRootName(getSimpleName(taxonomyFileName)) + "tax.summary";
274                         
275                         int start = time(NULL);
276                         int numFastaSeqs = 0;
277                         for (int i = 0; i < lines.size(); i++) {  delete lines[i];  }  lines.clear();
278                         
279 #if defined (__APPLE__) || (__MACH__) || (linux) || (__linux)
280                         if(processors == 1){
281                                 ifstream inFASTA;
282                                 openInputFile(fastaFileNames[s], inFASTA);
283                                 numFastaSeqs=count(istreambuf_iterator<char>(inFASTA),istreambuf_iterator<char>(), '>');
284                                 inFASTA.close();
285                                 
286                                 lines.push_back(new linePair(0, numFastaSeqs));
287                                 
288                                 driver(lines[0], newTaxonomyFile, tempTaxonomyFile, fastaFileNames[s]);
289                         }
290                         else{
291                                 vector<int> positions;
292                                 processIDS.resize(0);
293                                 
294                                 ifstream inFASTA;
295                                 openInputFile(fastaFileNames[s], inFASTA);
296                                 
297                                 string input;
298                                 while(!inFASTA.eof()){
299                                         input = getline(inFASTA);
300                                         if (input.length() != 0) {
301                                                 if(input[0] == '>'){    int pos = inFASTA.tellg(); positions.push_back(pos - input.length() - 1);       }
302                                         }
303                                 }
304                                 inFASTA.close();
305                                 
306                                 numFastaSeqs = positions.size();
307                                 
308                                 int numSeqsPerProcessor = numFastaSeqs / processors;
309                                 
310                                 for (int i = 0; i < processors; i++) {
311                                         int startPos = positions[ i * numSeqsPerProcessor ];
312                                         if(i == processors - 1){
313                                                 numSeqsPerProcessor = numFastaSeqs - i * numSeqsPerProcessor;
314                                         }
315                                         lines.push_back(new linePair(startPos, numSeqsPerProcessor));
316                                 }
317                                 createProcesses(newTaxonomyFile, tempTaxonomyFile, fastaFileNames[s]); 
318                                 
319                                 rename((newTaxonomyFile + toString(processIDS[0]) + ".temp").c_str(), newTaxonomyFile.c_str());
320                                 rename((tempTaxonomyFile + toString(processIDS[0]) + ".temp").c_str(), tempTaxonomyFile.c_str());
321                                 
322                                 for(int i=1;i<processors;i++){
323                                         appendTaxFiles((newTaxonomyFile + toString(processIDS[i]) + ".temp"), newTaxonomyFile);
324                                         appendTaxFiles((tempTaxonomyFile + toString(processIDS[i]) + ".temp"), tempTaxonomyFile);
325                                         remove((newTaxonomyFile + toString(processIDS[i]) + ".temp").c_str());
326                                         remove((tempTaxonomyFile + toString(processIDS[i]) + ".temp").c_str());
327                                 }
328                                 
329                         }
330 #else
331                         ifstream inFASTA;
332                         openInputFile(fastaFileNames[s], inFASTA);
333                         numFastaSeqs=count(istreambuf_iterator<char>(inFASTA),istreambuf_iterator<char>(), '>');
334                         inFASTA.close();
335                         
336                         lines.push_back(new linePair(0, numFastaSeqs));
337                         
338                         driver(lines[0], newTaxonomyFile, tempTaxonomyFile, fastaFileNames[s]);
339 #endif  
340                         //make taxonomy tree from new taxonomy file 
341                         PhyloTree taxaBrowser;
342                 
343                         ifstream in;
344                         openInputFile(tempTaxonomyFile, in);
345                 
346                         //read in users taxonomy file and add sequences to tree
347                         string name, taxon;
348                         while(!in.eof()){
349                                 in >> name >> taxon; gobble(in);
350                                 
351                                 if (namefile != "") {
352                                         itNames = nameMap.find(name);
353                 
354                                         if (itNames == nameMap.end()) { 
355                                                 mothurOut(name + " is not in your name file please correct."); mothurOutEndLine(); exit(1);
356                                         }else{
357                                                 for (int i = 0; i < itNames->second; i++) { 
358                                                         taxaBrowser.addSeqToTree(name+toString(i), taxon);  //add it as many times as there are identical seqs
359                                                 }
360                                         }
361                                 }else {  taxaBrowser.addSeqToTree(name, taxon);  } //add it once
362                         }
363                         in.close();
364         
365                         taxaBrowser.assignHeirarchyIDs(0);
366
367                         taxaBrowser.binUnclassified();
368                         
369                         remove(tempTaxonomyFile.c_str());
370                         
371                         //print summary file
372                         ofstream outTaxTree;
373                         openOutputFile(taxSummary, outTaxTree);
374                         taxaBrowser.print(outTaxTree);
375                         outTaxTree.close();
376                         
377                         //output taxonomy with the unclassified bins added
378                         ifstream inTax;
379                         openInputFile(newTaxonomyFile, inTax);
380                         
381                         ofstream outTax;
382                         string unclass = newTaxonomyFile + ".unclass.temp";
383                         openOutputFile(unclass, outTax);
384                         
385                         //get maxLevel from phylotree so you know how many 'unclassified's to add
386                         int maxLevel = taxaBrowser.getMaxLevel();
387                         
388                         //read taxfile - this reading and rewriting is done to preserve the confidence sscores.
389                         while (!inTax.eof()) {
390                                 inTax >> name >> taxon; gobble(inTax);
391                                 
392                                 string newTax = addUnclassifieds(taxon, maxLevel);
393                                 
394                                 outTax << name << '\t' << newTax << endl;
395                         }
396                         inTax.close();  
397                         outTax.close();
398                         
399                         remove(newTaxonomyFile.c_str());
400                         rename(unclass.c_str(), newTaxonomyFile.c_str());
401                         
402                         mothurOutEndLine();
403                         mothurOut("It took " + toString(time(NULL) - start) + " secs to classify " + toString(numFastaSeqs) + " sequences."); mothurOutEndLine(); mothurOutEndLine();
404                 }
405                 
406                 delete classify;
407                 return 0;
408         }
409         catch(exception& e) {
410                 errorOut(e, "ClassifySeqsCommand", "execute");
411                 exit(1);
412         }
413 }
414
415 /**************************************************************************************************/
416 string ClassifySeqsCommand::addUnclassifieds(string tax, int maxlevel) {
417         try{
418                 string newTax, taxon;
419                 int level = 0;
420                 
421                 //keep what you have counting the levels
422                 while (tax.find_first_of(';') != -1) {
423                         //get taxon
424                         taxon = tax.substr(0,tax.find_first_of(';'))+';';
425                         tax = tax.substr(tax.find_first_of(';')+1, tax.length());
426                         newTax += taxon;
427                         level++;
428                 }
429                 
430                 //add "unclassified" until you reach maxLevel
431                 while (level < maxlevel) {
432                         newTax += "unclassified;";
433                         level++;
434                 }
435                 
436                 return newTax;
437         }
438         catch(exception& e) {
439                 errorOut(e, "ClassifySeqsCommand", "addUnclassifieds");
440                 exit(1);
441         }
442 }
443
444 /**************************************************************************************************/
445
446 void ClassifySeqsCommand::createProcesses(string taxFileName, string tempTaxFile, string filename) {
447         try {
448 #if defined (__APPLE__) || (__MACH__) || (linux) || (__linux)
449                 int process = 0;
450                 //              processIDS.resize(0);
451                 
452                 //loop through and create all the processes you want
453                 while (process != processors) {
454                         int pid = fork();
455                         
456                         if (pid > 0) {
457                                 processIDS.push_back(pid);  //create map from line number to pid so you can append files in correct order later
458                                 process++;
459                         }else if (pid == 0){
460                                 driver(lines[process], taxFileName + toString(getpid()) + ".temp", tempTaxFile + toString(getpid()) + ".temp", filename);
461                                 exit(0);
462                         }else { mothurOut("unable to spawn the necessary processes."); mothurOutEndLine(); exit(0); }
463                 }
464                 
465                 //force parent to wait until all the processes are done
466                 for (int i=0;i<processors;i++) { 
467                         int temp = processIDS[i];
468                         wait(&temp);
469                 }
470 #endif          
471         }
472         catch(exception& e) {
473                 errorOut(e, "ClassifySeqsCommand", "createProcesses");
474                 exit(1);
475         }
476 }
477 /**************************************************************************************************/
478
479 void ClassifySeqsCommand::appendTaxFiles(string temp, string filename) {
480         try{
481                 
482                 ofstream output;
483                 ifstream input;
484                 openOutputFileAppend(filename, output);
485                 openInputFile(temp, input);
486                 
487                 while(char c = input.get()){
488                         if(input.eof())         {       break;                  }
489                         else                            {       output << c;    }
490                 }
491                 
492                 input.close();
493                 output.close();
494         }
495         catch(exception& e) {
496                 errorOut(e, "ClassifySeqsCommand", "appendTaxFiles");
497                 exit(1);
498         }
499 }
500
501 //**********************************************************************************************************************
502
503 int ClassifySeqsCommand::driver(linePair* line, string taxFName, string tempTFName, string filename){
504         try {
505                 ofstream outTax;
506                 openOutputFile(taxFName, outTax);
507                 
508                 ofstream outTaxSimple;
509                 openOutputFile(tempTFName, outTaxSimple);
510         
511                 ifstream inFASTA;
512                 openInputFile(filename, inFASTA);
513
514                 inFASTA.seekg(line->start);
515                 
516                 string taxonomy;
517
518                 for(int i=0;i<line->numSeqs;i++){
519                         
520                         Sequence* candidateSeq = new Sequence(inFASTA);
521                         
522                         if (candidateSeq->getName() != "") {
523                                 taxonomy = classify->getTaxonomy(candidateSeq);
524
525                                 if (taxonomy != "bad seq") {
526                                         //output confidence scores or not
527                                         if (probs) {
528                                                 outTax << candidateSeq->getName() << '\t' << taxonomy << endl;
529                                         }else{
530                                                 outTax << candidateSeq->getName() << '\t' << classify->getSimpleTax() << endl;
531                                                 cout << classify->getSimpleTax() << endl;
532                                         }
533                                         
534                                         outTaxSimple << candidateSeq->getName() << '\t' << classify->getSimpleTax() << endl;
535                                 }
536                         }                               
537                         delete candidateSeq;
538                         
539                         if((i+1) % 100 == 0){
540                                 mothurOut("Classifying sequence " + toString(i+1)); mothurOutEndLine();
541                         }
542                 }
543                 
544                 inFASTA.close();
545                 outTax.close();
546                 outTaxSimple.close();
547                 
548                 return 1;
549         }
550         catch(exception& e) {
551                 errorOut(e, "ClassifySeqsCommand", "driver");
552                 exit(1);
553         }
554 }
555
556 /**************************************************************************************************/