]> git.donarmstrong.com Git - mothur.git/blob - classifyseqscommand.cpp
added set.dir command and modified commands to redirect input and output, removed...
[mothur.git] / classifyseqscommand.cpp
1 /*
2  *  classifyseqscommand.cpp
3  *  Mothur
4  *
5  *  Created by westcott on 11/2/09.
6  *  Copyright 2009 Schloss Lab. All rights reserved.
7  *
8  */
9
10 #include "classifyseqscommand.h"
11 #include "sequence.hpp"
12 #include "bayesian.h"
13 #include "phylotree.h"
14 #include "knn.h"
15
16 //**********************************************************************************************************************
17
18 ClassifySeqsCommand::ClassifySeqsCommand(string option){
19         try {
20                 abort = false;
21                 
22                 //allow user to run help
23                 if(option == "help") { help(); abort = true; }
24                 
25                 else {
26                         
27                         //valid paramters for this command
28                         string AlignArray[] =  {"template","fasta","name","search","ksize","method","processors","taxonomy","match","mismatch","gapopen","gapextend","numwanted","cutoff","probs","iters", "outputdir","inputdir"};
29                         vector<string> myArray (AlignArray, AlignArray+(sizeof(AlignArray)/sizeof(string)));
30                         
31                         OptionParser parser(option);
32                         map<string, string> parameters = parser.getParameters(); 
33                         
34                         ValidParameters validParameter;
35                         map<string, string>::iterator it;
36                         
37                         //check to make sure all parameters are valid for command
38                         for (it = parameters.begin(); it != parameters.end(); it++) { 
39                                 if (validParameter.isValidParameter(it->first, myArray, it->second) != true) {  abort = true;  }
40                         }
41                         
42                         //if the user changes the output directory command factory will send this info to us in the output parameter 
43                         outputDir = validParameter.validFile(parameters, "outputdir", false);           if (outputDir == "not found"){  outputDir = "";         }
44                         
45                         //if the user changes the input directory command factory will send this info to us in the output parameter 
46                         string inputDir = validParameter.validFile(parameters, "inputdir", false);              
47                         if (inputDir == "not found"){   inputDir = "";          }
48                         else {
49                                 string path;
50                                 it = parameters.find("template");
51                                 //user has given a template file
52                                 if(it != parameters.end()){ 
53                                         path = hasPath(it->second);
54                                         //if the user has not given a path then, add inputdir. else leave path alone.
55                                         if (path == "") {       parameters["template"] = inputDir + it->second;         }
56                                 }
57                                 
58                                 it = parameters.find("taxonomy");
59                                 //user has given a template file
60                                 if(it != parameters.end()){ 
61                                         path = hasPath(it->second);
62                                         //if the user has not given a path then, add inputdir. else leave path alone.
63                                         if (path == "") {       parameters["taxonomy"] = inputDir + it->second;         }
64                                 }
65                         }
66
67                         //check for required parameters
68                         templateFileName = validParameter.validFile(parameters, "template", true);
69                         if (templateFileName == "not found") { 
70                                 mothurOut("template is a required parameter for the classify.seqs command."); 
71                                 mothurOutEndLine();
72                                 abort = true; 
73                         }
74                         else if (templateFileName == "not open") { abort = true; }      
75                         
76                         fastaFileName = validParameter.validFile(parameters, "fasta", false);
77                         if (fastaFileName == "not found") { mothurOut("fasta is a required parameter for the classify.seqs command."); mothurOutEndLine(); abort = true;  }
78                         else { 
79                                 splitAtDash(fastaFileName, fastaFileNames);
80                                 
81                                 //go through files and make sure they are good, if not, then disregard them
82                                 for (int i = 0; i < fastaFileNames.size(); i++) {
83                                         if (inputDir != "") {
84                                                 string path = hasPath(fastaFileNames[i]);
85                                                 //if the user has not given a path then, add inputdir. else leave path alone.
86                                                 if (path == "") {       fastaFileNames[i] = inputDir + fastaFileNames[i];               }
87                                         }
88                                         
89                                         int ableToOpen;
90                                         ifstream in;
91                                         ableToOpen = openInputFile(fastaFileNames[i], in);
92                                         if (ableToOpen == 1) { 
93                                                 mothurOut(fastaFileNames[i] + " will be disregarded."); mothurOutEndLine(); 
94                                                 //erase from file list
95                                                 fastaFileNames.erase(fastaFileNames.begin()+i);
96                                                 i--;
97                                         }
98                                         in.close();
99                                 }
100                                 
101                                 //make sure there is at least one valid file left
102                                 if (fastaFileNames.size() == 0) { mothurOut("no valid files."); mothurOutEndLine(); abort = true; }
103                         }
104
105                         
106                         taxonomyFileName = validParameter.validFile(parameters, "taxonomy", true);
107                         if (taxonomyFileName == "not found") { 
108                                 mothurOut("taxonomy is a required parameter for the classify.seqs command."); 
109                                 mothurOutEndLine();
110                                 abort = true; 
111                         }
112                         else if (taxonomyFileName == "not open") { abort = true; }      
113                         
114                         
115                         namefile = validParameter.validFile(parameters, "name", false);
116                         if (namefile == "not found") { namefile = "";  }
117
118                         else { 
119                                 splitAtDash(namefile, namefileNames);
120                                 
121                                 //go through files and make sure they are good, if not, then disregard them
122                                 for (int i = 0; i < namefileNames.size(); i++) {
123                                         if (inputDir != "") {
124                                                 string path = hasPath(namefileNames[i]);
125                                                 //if the user has not given a path then, add inputdir. else leave path alone.
126                                                 if (path == "") {       namefileNames[i] = inputDir + namefileNames[i];         }
127                                         }
128
129                                         int ableToOpen;
130                                         ifstream in;
131                                         ableToOpen = openInputFile(namefileNames[i], in);
132                                         if (ableToOpen == 1) {  mothurOut("Unable to match name file with fasta file."); mothurOutEndLine(); abort = true;      }
133                                         in.close();
134                                 }
135                         }
136
137                         if (namefile != "") {
138                                 if (namefileNames.size() != fastaFileNames.size()) { abort = true; mothurOut("If you provide a name file, you must have one for each fasta file."); mothurOutEndLine(); }
139                         }
140                         
141                         //check for optional parameter and set defaults
142                         // ...at some point should added some additional type checking...
143                         string temp;
144                         temp = validParameter.validFile(parameters, "ksize", false);            if (temp == "not found"){       temp = "8";                             }
145                         convert(temp, kmerSize); 
146                         
147                         temp = validParameter.validFile(parameters, "processors", false);       if (temp == "not found"){       temp = "1";                             }
148                         convert(temp, processors); 
149                         
150                         search = validParameter.validFile(parameters, "search", false);         if (search == "not found"){     search = "kmer";                }
151                         
152                         method = validParameter.validFile(parameters, "method", false);         if (method == "not found"){     method = "bayesian";    }
153                         
154                         temp = validParameter.validFile(parameters, "match", false);            if (temp == "not found"){       temp = "1.0";                   }
155                         convert(temp, match);  
156                         
157                         temp = validParameter.validFile(parameters, "mismatch", false);         if (temp == "not found"){       temp = "-1.0";                  }
158                         convert(temp, misMatch);  
159                         
160                         temp = validParameter.validFile(parameters, "gapopen", false);          if (temp == "not found"){       temp = "-2.0";                  }
161                         convert(temp, gapOpen);  
162                         
163                         temp = validParameter.validFile(parameters, "gapextend", false);        if (temp == "not found"){       temp = "-1.0";                  }
164                         convert(temp, gapExtend); 
165                         
166                         temp = validParameter.validFile(parameters, "numwanted", false);        if (temp == "not found"){       temp = "10";                    }
167                         convert(temp, numWanted);
168                         
169                         temp = validParameter.validFile(parameters, "cutoff", false);           if (temp == "not found"){       temp = "0";                             }
170                         convert(temp, cutoff);
171                         
172                         temp = validParameter.validFile(parameters, "probs", false);            if (temp == "not found"){       temp = "true";                  }
173                         probs = isTrue(temp);
174                         
175                         temp = validParameter.validFile(parameters, "iters", false);            if (temp == "not found") { temp = "100";                        }
176                         convert(temp, iters); 
177
178
179                         
180                         if ((method == "bayesian") && (search != "kmer"))  { 
181                                 mothurOut("The bayesian method requires the kmer search." + search + "will be disregarded." ); mothurOutEndLine();
182                                 search = "kmer";
183                         }
184                 }
185                 
186         }
187         catch(exception& e) {
188                 errorOut(e, "ClassifySeqsCommand", "ClassifySeqsCommand");
189                 exit(1);
190         }
191 }
192
193 //**********************************************************************************************************************
194
195 ClassifySeqsCommand::~ClassifySeqsCommand(){    
196
197         if (abort == false) {
198                 for (int i = 0; i < lines.size(); i++) {  delete lines[i];  }  lines.clear();
199         }
200 }
201
202 //**********************************************************************************************************************
203
204 void ClassifySeqsCommand::help(){
205         try {
206                 mothurOut("The classify.seqs command reads a fasta file containing sequences and creates a .taxonomy file and a .tax.summary file.\n");
207                 mothurOut("The classify.seqs command parameters are template, fasta, search, ksize, method, taxonomy, processors, match, mismatch, gapopen, gapextend, numwanted and probs.\n");
208                 mothurOut("The template, fasta and taxonomy parameters are required. You may enter multiple fasta files by separating their names with dashes. ie. fasta=abrecovery.fasta-amzon.fasta \n");
209                 mothurOut("The search parameter allows you to specify the method to find most similar template.  Your options are: suffix, kmer and blast. The default is kmer.\n");
210                 mothurOut("The method parameter allows you to specify classification method to use.  Your options are: bayesian and knn. The default is bayesian.\n");
211                 mothurOut("The ksize parameter allows you to specify the kmer size for finding most similar template to candidate.  The default is 8.\n");
212                 mothurOut("The processors parameter allows you to specify the number of processors to use. The default is 1.\n");
213                 mothurOut("The match parameter allows you to specify the bonus for having the same base. The default is 1.0.\n");
214                 mothurOut("The mistmatch parameter allows you to specify the penalty for having different bases.  The default is -1.0.\n");
215                 mothurOut("The gapopen parameter allows you to specify the penalty for opening a gap in an alignment. The default is -2.0.\n");
216                 mothurOut("The gapextend parameter allows you to specify the penalty for extending a gap in an alignment.  The default is -1.0.\n");
217                 mothurOut("The numwanted parameter allows you to specify the number of sequence matches you want with the knn method.  The default is 10.\n");
218                 mothurOut("The cutoff parameter allows you to specify a bootstrap confidence threshold for your taxonomy.  The default is 0.\n");
219                 mothurOut("The probs parameter shut off the bootstrapping results for the bayesian method. The default is true, meaning you want the bootstrapping to be run.\n");
220                 mothurOut("The iters parameter allows you to specify how many iterations to do when calculating the bootstrap confidence score for your taxonomy with the bayesian method.  The default is 100.\n");
221                 mothurOut("The classify.seqs command should be in the following format: \n");
222                 mothurOut("classify.seqs(template=yourTemplateFile, fasta=yourFastaFile, method=yourClassificationMethod, search=yourSearchmethod, ksize=yourKmerSize, taxonomy=yourTaxonomyFile, processors=yourProcessors) \n");
223                 mothurOut("Example classify.seqs(fasta=amazon.fasta, template=core.filtered, method=knn, search=gotoh, ksize=8, processors=2)\n");
224                 mothurOut("The .taxonomy file consists of 2 columns: 1 = your sequence name, 2 = the taxonomy for your sequence. \n");
225                 mothurOut("The .tax.summary is a summary of the different taxonomies represented in your fasta file. \n");
226                 mothurOut("Note: No spaces between parameter labels (i.e. fasta), '=' and parameters (i.e.yourFastaFile).\n\n");
227         }
228         catch(exception& e) {
229                 errorOut(e, "ClassifySeqsCommand", "help");
230                 exit(1);
231         }
232 }
233
234
235 //**********************************************************************************************************************
236
237 int ClassifySeqsCommand::execute(){
238         try {
239                 if (abort == true) {    return 0;       }
240                 
241                 if(method == "bayesian"){       classify = new Bayesian(taxonomyFileName, templateFileName, search, kmerSize, cutoff, iters);           }
242                 else if(method == "knn"){       classify = new Knn(taxonomyFileName, templateFileName, search, kmerSize, gapOpen, gapExtend, match, misMatch, numWanted);                               }
243                 else {
244                         mothurOut(search + " is not a valid method option. I will run the command using bayesian.");
245                         mothurOutEndLine();
246                         classify = new Bayesian(taxonomyFileName, templateFileName, search, kmerSize, cutoff, iters);   
247                 }
248
249                                 
250                 for (int s = 0; s < fastaFileNames.size(); s++) {
251                 
252                         //read namefile
253                         if(namefile != "") {
254                                 nameMap.clear(); //remove old names
255                                 
256                                 ifstream inNames;
257                                 openInputFile(namefileNames[s], inNames);
258                                 
259                                 string firstCol, secondCol;
260                                 while(!inNames.eof()) {
261                                         inNames >> firstCol >> secondCol; gobble(inNames);
262                                         nameMap[firstCol] = getNumNames(secondCol);  //ex. seq1 seq1,seq3,seq5 -> seq1 = 3.
263                                 }
264                                 inNames.close();
265                         }
266                 
267                         mothurOut("Classifying sequences from " + fastaFileNames[s] + " ..." ); mothurOutEndLine();
268                         
269                         if (outputDir == "") { outputDir += hasPath(fastaFileNames[s]); }
270                         string newTaxonomyFile = outputDir + getRootName(getSimpleName(fastaFileNames[s])) + getRootName(getSimpleName(taxonomyFileName)) + "taxonomy";
271                         string tempTaxonomyFile = outputDir + getRootName(getSimpleName(fastaFileNames[s])) + "taxonomy.temp";
272                         string taxSummary = outputDir + getRootName(getSimpleName(fastaFileNames[s])) + getRootName(getSimpleName(taxonomyFileName)) + "tax.summary";
273                         
274                         int start = time(NULL);
275                         int numFastaSeqs = 0;
276                         for (int i = 0; i < lines.size(); i++) {  delete lines[i];  }  lines.clear();
277                         
278 #if defined (__APPLE__) || (__MACH__) || (linux) || (__linux)
279                         if(processors == 1){
280                                 ifstream inFASTA;
281                                 openInputFile(fastaFileNames[s], inFASTA);
282                                 numFastaSeqs=count(istreambuf_iterator<char>(inFASTA),istreambuf_iterator<char>(), '>');
283                                 inFASTA.close();
284                                 
285                                 lines.push_back(new linePair(0, numFastaSeqs));
286                                 
287                                 driver(lines[0], newTaxonomyFile, tempTaxonomyFile, fastaFileNames[s]);
288                         }
289                         else{
290                                 vector<int> positions;
291                                 processIDS.resize(0);
292                                 
293                                 ifstream inFASTA;
294                                 openInputFile(fastaFileNames[s], inFASTA);
295                                 
296                                 string input;
297                                 while(!inFASTA.eof()){
298                                         input = getline(inFASTA);
299                                         if (input.length() != 0) {
300                                                 if(input[0] == '>'){    int pos = inFASTA.tellg(); positions.push_back(pos - input.length() - 1);       }
301                                         }
302                                 }
303                                 inFASTA.close();
304                                 
305                                 numFastaSeqs = positions.size();
306                                 
307                                 int numSeqsPerProcessor = numFastaSeqs / processors;
308                                 
309                                 for (int i = 0; i < processors; i++) {
310                                         int startPos = positions[ i * numSeqsPerProcessor ];
311                                         if(i == processors - 1){
312                                                 numSeqsPerProcessor = numFastaSeqs - i * numSeqsPerProcessor;
313                                         }
314                                         lines.push_back(new linePair(startPos, numSeqsPerProcessor));
315                                 }
316                                 createProcesses(newTaxonomyFile, tempTaxonomyFile, fastaFileNames[s]); 
317                                 
318                                 rename((newTaxonomyFile + toString(processIDS[0]) + ".temp").c_str(), newTaxonomyFile.c_str());
319                                 rename((tempTaxonomyFile + toString(processIDS[0]) + ".temp").c_str(), tempTaxonomyFile.c_str());
320                                 
321                                 for(int i=1;i<processors;i++){
322                                         appendTaxFiles((newTaxonomyFile + toString(processIDS[i]) + ".temp"), newTaxonomyFile);
323                                         appendTaxFiles((tempTaxonomyFile + toString(processIDS[i]) + ".temp"), tempTaxonomyFile);
324                                         remove((newTaxonomyFile + toString(processIDS[i]) + ".temp").c_str());
325                                         remove((tempTaxonomyFile + toString(processIDS[i]) + ".temp").c_str());
326                                 }
327                                 
328                         }
329 #else
330                         ifstream inFASTA;
331                         openInputFile(fastaFileNames[s], inFASTA);
332                         numFastaSeqs=count(istreambuf_iterator<char>(inFASTA),istreambuf_iterator<char>(), '>');
333                         inFASTA.close();
334                         
335                         lines.push_back(new linePair(0, numFastaSeqs));
336                         
337                         driver(lines[0], newTaxonomyFile, tempTaxonomyFile, fastaFileNames[s]);
338 #endif  
339                         //make taxonomy tree from new taxonomy file 
340                         PhyloTree taxaBrowser;
341                 
342                         ifstream in;
343                         openInputFile(tempTaxonomyFile, in);
344                 
345                         //read in users taxonomy file and add sequences to tree
346                         string name, taxon;
347                         while(!in.eof()){
348                                 in >> name >> taxon; gobble(in);
349                                 
350                                 if (namefile != "") {
351                                         itNames = nameMap.find(name);
352                 
353                                         if (itNames == nameMap.end()) { 
354                                                 mothurOut(name + " is not in your name file please correct."); mothurOutEndLine(); exit(1);
355                                         }else{
356                                                 for (int i = 0; i < itNames->second; i++) { 
357                                                         taxaBrowser.addSeqToTree(name+toString(i), taxon);  //add it as many times as there are identical seqs
358                                                 }
359                                         }
360                                 }else {  taxaBrowser.addSeqToTree(name, taxon);  } //add it once
361                         }
362                         in.close();
363         
364                         taxaBrowser.assignHeirarchyIDs(0);
365
366                         taxaBrowser.binUnclassified();
367                         
368                         remove(tempTaxonomyFile.c_str());
369                         
370                         //print summary file
371                         ofstream outTaxTree;
372                         openOutputFile(taxSummary, outTaxTree);
373                         taxaBrowser.print(outTaxTree);
374                         outTaxTree.close();
375                         
376                         //output taxonomy with the unclassified bins added
377                         ifstream inTax;
378                         openInputFile(newTaxonomyFile, inTax);
379                         
380                         ofstream outTax;
381                         string unclass = newTaxonomyFile + ".unclass.temp";
382                         openOutputFile(unclass, outTax);
383                         
384                         //get maxLevel from phylotree so you know how many 'unclassified's to add
385                         int maxLevel = taxaBrowser.getMaxLevel();
386                         
387                         //read taxfile - this reading and rewriting is done to preserve the confidence sscores.
388                         while (!inTax.eof()) {
389                                 inTax >> name >> taxon; gobble(inTax);
390                                 
391                                 string newTax = addUnclassifieds(taxon, maxLevel);
392                                 
393                                 outTax << name << '\t' << newTax << endl;
394                         }
395                         inTax.close();  
396                         outTax.close();
397                         
398                         remove(newTaxonomyFile.c_str());
399                         rename(unclass.c_str(), newTaxonomyFile.c_str());
400                         
401                         mothurOutEndLine();
402                         mothurOut("It took " + toString(time(NULL) - start) + " secs to classify " + toString(numFastaSeqs) + " sequences."); mothurOutEndLine(); mothurOutEndLine();
403                 }
404                 
405                 delete classify;
406                 return 0;
407         }
408         catch(exception& e) {
409                 errorOut(e, "ClassifySeqsCommand", "execute");
410                 exit(1);
411         }
412 }
413
414 /**************************************************************************************************/
415 string ClassifySeqsCommand::addUnclassifieds(string tax, int maxlevel) {
416         try{
417                 string newTax, taxon;
418                 int level = 0;
419                 
420                 //keep what you have counting the levels
421                 while (tax.find_first_of(';') != -1) {
422                         //get taxon
423                         taxon = tax.substr(0,tax.find_first_of(';'))+';';
424                         tax = tax.substr(tax.find_first_of(';')+1, tax.length());
425                         newTax += taxon;
426                         level++;
427                 }
428                 
429                 //add "unclassified" until you reach maxLevel
430                 while (level < maxlevel) {
431                         newTax += "unclassified;";
432                         level++;
433                 }
434                 
435                 return newTax;
436         }
437         catch(exception& e) {
438                 errorOut(e, "ClassifySeqsCommand", "addUnclassifieds");
439                 exit(1);
440         }
441 }
442
443 /**************************************************************************************************/
444
445 void ClassifySeqsCommand::createProcesses(string taxFileName, string tempTaxFile, string filename) {
446         try {
447 #if defined (__APPLE__) || (__MACH__) || (linux) || (__linux)
448                 int process = 0;
449                 //              processIDS.resize(0);
450                 
451                 //loop through and create all the processes you want
452                 while (process != processors) {
453                         int pid = fork();
454                         
455                         if (pid > 0) {
456                                 processIDS.push_back(pid);  //create map from line number to pid so you can append files in correct order later
457                                 process++;
458                         }else if (pid == 0){
459                                 driver(lines[process], taxFileName + toString(getpid()) + ".temp", tempTaxFile + toString(getpid()) + ".temp", filename);
460                                 exit(0);
461                         }else { mothurOut("unable to spawn the necessary processes."); mothurOutEndLine(); exit(0); }
462                 }
463                 
464                 //force parent to wait until all the processes are done
465                 for (int i=0;i<processors;i++) { 
466                         int temp = processIDS[i];
467                         wait(&temp);
468                 }
469 #endif          
470         }
471         catch(exception& e) {
472                 errorOut(e, "ClassifySeqsCommand", "createProcesses");
473                 exit(1);
474         }
475 }
476 /**************************************************************************************************/
477
478 void ClassifySeqsCommand::appendTaxFiles(string temp, string filename) {
479         try{
480                 
481                 ofstream output;
482                 ifstream input;
483                 openOutputFileAppend(filename, output);
484                 openInputFile(temp, input);
485                 
486                 while(char c = input.get()){
487                         if(input.eof())         {       break;                  }
488                         else                            {       output << c;    }
489                 }
490                 
491                 input.close();
492                 output.close();
493         }
494         catch(exception& e) {
495                 errorOut(e, "ClassifySeqsCommand", "appendTaxFiles");
496                 exit(1);
497         }
498 }
499
500 //**********************************************************************************************************************
501
502 int ClassifySeqsCommand::driver(linePair* line, string taxFName, string tempTFName, string filename){
503         try {
504                 ofstream outTax;
505                 openOutputFile(taxFName, outTax);
506                 
507                 ofstream outTaxSimple;
508                 openOutputFile(tempTFName, outTaxSimple);
509         
510                 ifstream inFASTA;
511                 openInputFile(filename, inFASTA);
512
513                 inFASTA.seekg(line->start);
514                 
515                 string taxonomy;
516
517                 for(int i=0;i<line->numSeqs;i++){
518                         
519                         Sequence* candidateSeq = new Sequence(inFASTA);
520                         
521                         if (candidateSeq->getName() != "") {
522                                 taxonomy = classify->getTaxonomy(candidateSeq);
523
524                                 if (taxonomy != "bad seq") {
525                                         //output confidence scores or not
526                                         if (probs) {
527                                                 outTax << candidateSeq->getName() << '\t' << taxonomy << endl;
528                                         }else{
529                                                 outTax << candidateSeq->getName() << '\t' << classify->getSimpleTax() << endl;
530                                                 cout << classify->getSimpleTax() << endl;
531                                         }
532                                         
533                                         outTaxSimple << candidateSeq->getName() << '\t' << classify->getSimpleTax() << endl;
534                                 }
535                         }                               
536                         delete candidateSeq;
537                         
538                         if((i+1) % 100 == 0){
539                                 mothurOut("Classifying sequence " + toString(i+1)); mothurOutEndLine();
540                         }
541                 }
542                 
543                 inFASTA.close();
544                 outTax.close();
545                 outTaxSimple.close();
546                 
547                 return 1;
548         }
549         catch(exception& e) {
550                 errorOut(e, "ClassifySeqsCommand", "driver");
551                 exit(1);
552         }
553 }
554
555 /**************************************************************************************************/