X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=classify.cpp;h=eb0865c2744acfe120505a5a533cd482af8b9b19;hb=ae57e166b2ed7b475ec3f466106bd76fabadd063;hp=147f499b87357f5782b98fc882db80f3233f2aee;hpb=cd9dbd8b53bbe32af3e9c6bead4aa6d796a278a5;p=mothur.git

diff --git a/classify.cpp b/classify.cpp
index 147f499..eb0865c 100644
--- a/classify.cpp
+++ b/classify.cpp
@@ -13,143 +13,223 @@
 #include "suffixdb.hpp"
 #include "blastdb.hpp"
 #include "distancedb.hpp"
+#include "referencedb.h"
 
 /**************************************************************************************************/
 void Classify::generateDatabaseAndNames(string tfile, string tempFile, string method, int kmerSize, float gapOpen, float gapExtend, float match, float misMatch)  {		
 	try {	
-		taxFile = tfile;
-		readTaxonomy(taxFile);	
+		ReferenceDB* rdb = ReferenceDB::getInstance();
 		
-		templateFile = tempFile;	
+		if (tfile == "saved") { tfile = rdb->getSavedTaxonomy(); }
 		
-		int start = time(NULL);
+		taxFile = tfile;
+		readTaxonomy(taxFile);	
 		int numSeqs = 0;
 		
-		m->mothurOut("Generating search database...    "); cout.flush();
-#ifdef USE_MPI	
-			int pid;
-			vector<long> positions;
-		
-			MPI_Status status; 
-			MPI_File inMPI;
-			MPI_Comm_rank(MPI_COMM_WORLD, &pid); //find out who we are
-
-			//char* inFileName = new char[tempFile.length()];
-			//memcpy(inFileName, tempFile.c_str(), tempFile.length());
+		if (tempFile == "saved") {
+			int start = time(NULL);
+			m->mothurOutEndLine();  m->mothurOut("Using sequences from " + rdb->getSavedReference() + " that are saved in memory.");	m->mothurOutEndLine();
+			
+			numSeqs = rdb->referenceSeqs.size();
+			templateFile = rdb->getSavedReference();
+			tempFile = rdb->getSavedReference();
+			
+			bool needToGenerate = true;
+			string kmerDBName;
+			if(method == "kmer")			{	
+				database = new KmerDB(tempFile, kmerSize);			
+				
+				kmerDBName = tempFile.substr(0,tempFile.find_last_of(".")+1) + char('0'+ kmerSize) + "mer";
+				ifstream kmerFileTest(kmerDBName.c_str());
+				if(kmerFileTest){	
+					bool GoodFile = m->checkReleaseVersion(kmerFileTest, m->getVersion());
+					if (GoodFile) {  needToGenerate = false;	}
+				}
+			}
+			else if(method == "suffix")		{	database = new SuffixDB(numSeqs);								}
+			else if(method == "blast")		{	database = new BlastDB(tempFile.substr(0,tempFile.find_last_of(".")+1), gapOpen, gapExtend, match, misMatch, "", threadID);	}
+			else if(method == "distance")	{	database = new DistanceDB();	}
+			else {
+				m->mothurOut(method + " is not a valid search option. I will run the command using kmer, ksize=8.");
+				m->mothurOutEndLine();
+				database = new KmerDB(tempFile, 8);
+			}
 			
-			char inFileName[1024];
-			strcpy(inFileName, tempFile.c_str());
+			if (needToGenerate) {
+				for (int k = 0; k < rdb->referenceSeqs.size(); k++) {
+					Sequence temp(rdb->referenceSeqs[k].getName(), rdb->referenceSeqs[k].getAligned());
+					names.push_back(temp.getName());
+					database->addSequence(temp);	
+				}
+				database->generateDB();
+			}else if ((method == "kmer") && (!needToGenerate)) {	
+				ifstream kmerFileTest(kmerDBName.c_str());
+				database->readKmerDB(kmerFileTest);	
+				
+				for (int k = 0; k < rdb->referenceSeqs.size(); k++) {
+					names.push_back(rdb->referenceSeqs[k].getName());
+				}			
+			}	
+			
+			database->setNumSeqs(numSeqs);
+			
+			//sanity check
+			bool okay = phyloTree->ErrorCheck(names);
+			
+			if (!okay) { m->control_pressed = true; }
+			
+			m->mothurOut("It took " + toString(time(NULL) - start) + " to load " + toString(rdb->referenceSeqs.size()) + " sequences and generate the search databases.");m->mothurOutEndLine();  
+			
+		}else {
+			
+			templateFile = tempFile;	
+			
+			int start = time(NULL);
+			
+			m->mothurOut("Generating search database...    "); cout.flush();
+	#ifdef USE_MPI	
+				int pid, processors;
+				vector<unsigned long long> positions;
+				int tag = 2001;
+			
+				MPI_Status status; 
+				MPI_File inMPI;
+				MPI_Comm_rank(MPI_COMM_WORLD, &pid); //find out who we are
+				MPI_Comm_size(MPI_COMM_WORLD, &processors);
+
+				//char* inFileName = new char[tempFile.length()];
+				//memcpy(inFileName, tempFile.c_str(), tempFile.length());
+				
+				char inFileName[1024];
+				strcpy(inFileName, tempFile.c_str());
 
-			MPI_File_open(MPI_COMM_WORLD, inFileName, MPI_MODE_RDONLY, MPI_INFO_NULL, &inMPI);  //comm, filename, mode, info, filepointer
-			//delete inFileName;
+				MPI_File_open(MPI_COMM_WORLD, inFileName, MPI_MODE_RDONLY, MPI_INFO_NULL, &inMPI);  //comm, filename, mode, info, filepointer
+				//delete inFileName;
 
-			if (pid == 0) { //only one process needs to scan file
-				positions = setFilePosFasta(tempFile, numSeqs); //fills MPIPos, returns numSeqs
+				if (pid == 0) { //only one process needs to scan file
+					positions = m->setFilePosFasta(tempFile, numSeqs); //fills MPIPos, returns numSeqs
 
-				//send file positions to all processes
-				MPI_Bcast(&numSeqs, 1, MPI_INT, 0, MPI_COMM_WORLD);  //send numSeqs
-				MPI_Bcast(&positions[0], (numSeqs+1), MPI_LONG, 0, MPI_COMM_WORLD); //send file pos	
-			}else{
-				MPI_Bcast(&numSeqs, 1, MPI_INT, 0, MPI_COMM_WORLD); //get numSeqs
-				positions.resize(numSeqs);
-				MPI_Bcast(&positions[0], (numSeqs+1), MPI_LONG, 0, MPI_COMM_WORLD); //get file positions
-			}
+					//send file positions to all processes
+					for(int i = 1; i < processors; i++) { 
+						MPI_Send(&numSeqs, 1, MPI_INT, i, tag, MPI_COMM_WORLD);
+						MPI_Send(&positions[0], (numSeqs+1), MPI_LONG, i, tag, MPI_COMM_WORLD);
+					}
+				}else{
+					MPI_Recv(&numSeqs, 1, MPI_INT, 0, tag, MPI_COMM_WORLD, &status);
+					positions.resize(numSeqs+1);
+					MPI_Recv(&positions[0], (numSeqs+1), MPI_LONG, 0, tag, MPI_COMM_WORLD, &status);
+				}
+				
+				//create database
+				if(method == "kmer")			{	database = new KmerDB(tempFile, kmerSize);			}
+				else if(method == "suffix")		{	database = new SuffixDB(numSeqs);								}
+				else if(method == "blast")		{	database = new BlastDB(tempFile.substr(0,tempFile.find_last_of(".")+1), gapOpen, gapExtend, match, misMatch, "");	}
+				else if(method == "distance")	{	database = new DistanceDB();	}
+				else {
+					m->mothurOut(method + " is not a valid search option. I will run the command using kmer, ksize=8."); m->mothurOutEndLine();
+					database = new KmerDB(tempFile, 8);
+				}
+
+				//read file 
+				for(int i=0;i<numSeqs;i++){
+					//read next sequence
+					int length = positions[i+1] - positions[i];
+					char* buf4 = new char[length];
+					MPI_File_read_at(inMPI, positions[i], buf4, length, MPI_CHAR, &status);
+					
+					string tempBuf = buf4;
+					if (tempBuf.length() > length) { tempBuf = tempBuf.substr(0, length); }
+					delete buf4;
+					istringstream iss (tempBuf,istringstream::in);
+					
+					Sequence temp(iss);  
+					if (temp.getName() != "") {
+						if (rdb->save) { rdb->referenceSeqs.push_back(temp); }
+						names.push_back(temp.getName());
+						database->addSequence(temp);	
+					}
+				}
+				
+				database->generateDB();
+				MPI_File_close(&inMPI);
+				MPI_Barrier(MPI_COMM_WORLD); //make everyone wait - just in case
+		#else
 			
-			//create database
-			if(method == "kmer")			{	database = new KmerDB(tempFile, kmerSize);			}
+			//need to know number of template seqs for suffixdb
+			if (method == "suffix") {
+				ifstream inFASTA;
+				m->openInputFile(tempFile, inFASTA);
+				m->getNumSeqs(inFASTA, numSeqs);
+				inFASTA.close();
+			}
+
+			bool needToGenerate = true;
+			string kmerDBName;
+			if(method == "kmer")			{	
+				database = new KmerDB(tempFile, kmerSize);			
+				
+				kmerDBName = tempFile.substr(0,tempFile.find_last_of(".")+1) + char('0'+ kmerSize) + "mer";
+				ifstream kmerFileTest(kmerDBName.c_str());
+				if(kmerFileTest){	
+					bool GoodFile = m->checkReleaseVersion(kmerFileTest, m->getVersion());
+					if (GoodFile) {  needToGenerate = false;	}
+				}
+			}
 			else if(method == "suffix")		{	database = new SuffixDB(numSeqs);								}
-			else if(method == "blast")		{	database = new BlastDB(gapOpen, gapExtend, match, misMatch);	}
+			else if(method == "blast")		{	database = new BlastDB(tempFile.substr(0,tempFile.find_last_of(".")+1), gapOpen, gapExtend, match, misMatch, "", threadID);	}
 			else if(method == "distance")	{	database = new DistanceDB();	}
 			else {
-				m->mothurOut(method + " is not a valid search option. I will run the command using kmer, ksize=8."); m->mothurOutEndLine();
+				m->mothurOut(method + " is not a valid search option. I will run the command using kmer, ksize=8.");
+				m->mothurOutEndLine();
 				database = new KmerDB(tempFile, 8);
 			}
-
-			//read file 
-			for(int i=0;i<numSeqs;i++){
-				//read next sequence
-				int length = positions[i+1] - positions[i];
-				char* buf4 = new char[length];
-				MPI_File_read_at(inMPI, positions[i], buf4, length, MPI_CHAR, &status);
-				
-				string tempBuf = buf4;
-				if (tempBuf.length() > length) { tempBuf = tempBuf.substr(0, length); }
-				delete buf4;
-				istringstream iss (tempBuf,istringstream::in);
+			
+			if (needToGenerate) {
+				ifstream fastaFile;
+				m->openInputFile(tempFile, fastaFile);
 				
-				Sequence temp(iss);  
-				if (temp.getName() != "") {
+				while (!fastaFile.eof()) {
+					Sequence temp(fastaFile);
+					m->gobble(fastaFile);
+					
+					if (rdb->save) { rdb->referenceSeqs.push_back(temp); }
+					
 					names.push_back(temp.getName());
+								
 					database->addSequence(temp);	
 				}
-			}
-			
-			database->generateDB();
-			MPI_File_close(&inMPI);
-	#else
-		
-		//need to know number of template seqs for suffixdb
-		if (method == "suffix") {
-			ifstream inFASTA;
-			openInputFile(tempFile, inFASTA);
-			numSeqs = count(istreambuf_iterator<char>(inFASTA),istreambuf_iterator<char>(), '>');
-			inFASTA.close();
-		}
+				fastaFile.close();
 
-		bool needToGenerate = true;
-		string kmerDBName;
-		if(method == "kmer")			{	
-			database = new KmerDB(tempFile, kmerSize);			
-			
-			kmerDBName = tempFile.substr(0,tempFile.find_last_of(".")+1) + char('0'+ kmerSize) + "mer";
-			ifstream kmerFileTest(kmerDBName.c_str());
-			if(kmerFileTest){	needToGenerate = false;		}
-		}
-		else if(method == "suffix")		{	database = new SuffixDB(numSeqs);								}
-		else if(method == "blast")		{	database = new BlastDB(gapOpen, gapExtend, match, misMatch);	}
-		else if(method == "distance")	{	database = new DistanceDB();	}
-		else {
-			m->mothurOut(method + " is not a valid search option. I will run the command using kmer, ksize=8.");
-			m->mothurOutEndLine();
-			database = new KmerDB(tempFile, 8);
-		}
-		
-		if (needToGenerate) {
-			ifstream fastaFile;
-			openInputFile(tempFile, fastaFile);
-			
-			while (!fastaFile.eof()) {
-				Sequence temp(fastaFile);
-				gobble(fastaFile);
+				database->generateDB();
+				
+			}else if ((method == "kmer") && (!needToGenerate)) {	
+				ifstream kmerFileTest(kmerDBName.c_str());
+				database->readKmerDB(kmerFileTest);	
 			
-				names.push_back(temp.getName());
-								
-				database->addSequence(temp);	
+				ifstream fastaFile;
+				m->openInputFile(tempFile, fastaFile);
+				
+				while (!fastaFile.eof()) {
+					Sequence temp(fastaFile);
+					m->gobble(fastaFile);
+					
+					if (rdb->save) { rdb->referenceSeqs.push_back(temp); }
+					names.push_back(temp.getName());
+				}
+				fastaFile.close();
 			}
-			fastaFile.close();
-
-			database->generateDB();
+	#endif	
+		
+			database->setNumSeqs(names.size());
 			
-		}else if ((method == "kmer") && (!needToGenerate)) {	
-			ifstream kmerFileTest(kmerDBName.c_str());
-			database->readKmerDB(kmerFileTest);	
+			//sanity check
+			bool okay = phyloTree->ErrorCheck(names);
 			
-			ifstream fastaFile;
-			openInputFile(tempFile, fastaFile);
+			if (!okay) { m->control_pressed = true; }
 			
-			while (!fastaFile.eof()) {
-				Sequence temp(fastaFile);
-				gobble(fastaFile);
-
-				names.push_back(temp.getName());
-			}
-			fastaFile.close();
+			m->mothurOut("DONE."); m->mothurOutEndLine();
+			m->mothurOut("It took " + toString(time(NULL) - start) + " seconds generate search database. "); m->mothurOutEndLine();
 		}
-#endif		
-		database->setNumSeqs(names.size());
-		
-		m->mothurOut("DONE."); m->mothurOutEndLine();
-		m->mothurOut("It took " + toString(time(NULL) - start) + " seconds generate search database. "); m->mothurOutEndLine();
 
 	}
 	catch(exception& e) {
@@ -171,12 +251,14 @@ int Classify::readTaxonomy(string file) {
 		m->mothurOut("Reading in the " + file + " taxonomy...\t");	cout.flush();
 
 #ifdef USE_MPI	
-		int pid, num;
-		vector<long> positions;
+		int pid, num, processors;
+		vector<unsigned long long> positions;
+		int tag = 2001;
 		
 		MPI_Status status; 
 		MPI_File inMPI;
 		MPI_Comm_rank(MPI_COMM_WORLD, &pid); //find out who we are
+		MPI_Comm_size(MPI_COMM_WORLD, &processors);
 
 		//char* inFileName = new char[file.length()];
 		//memcpy(inFileName, file.c_str(), file.length());
@@ -188,15 +270,17 @@ int Classify::readTaxonomy(string file) {
 		//delete inFileName;
 
 		if (pid == 0) {
-			positions = setFilePosEachLine(file, num);
+			positions = m->setFilePosEachLine(file, num);
 			
 			//send file positions to all processes
-			MPI_Bcast(&num, 1, MPI_INT, 0, MPI_COMM_WORLD);  //send numSeqs
-			MPI_Bcast(&positions[0], (num+1), MPI_LONG, 0, MPI_COMM_WORLD); //send file pos	
+			for(int i = 1; i < processors; i++) { 
+				MPI_Send(&num, 1, MPI_INT, i, tag, MPI_COMM_WORLD);
+				MPI_Send(&positions[0], (num+1), MPI_LONG, i, tag, MPI_COMM_WORLD);
+			}
 		}else{
-			MPI_Bcast(&num, 1, MPI_INT, 0, MPI_COMM_WORLD); //get numSeqs
-			positions.resize(num);
-			MPI_Bcast(&positions[0], (num+1), MPI_LONG, 0, MPI_COMM_WORLD); //get file positions
+			MPI_Recv(&num, 1, MPI_INT, 0, tag, MPI_COMM_WORLD, &status);
+			positions.resize(num+1);
+			MPI_Recv(&positions[0], (num+1), MPI_LONG, 0, tag, MPI_COMM_WORLD, &status);
 		}
 	
 		//read file 
@@ -218,9 +302,10 @@ int Classify::readTaxonomy(string file) {
 		}
 		
 		MPI_File_close(&inMPI);
+		MPI_Barrier(MPI_COMM_WORLD); //make everyone wait - just in case
 #else				
 		ifstream inTax;
-		openInputFile(file, inTax);
+		m->openInputFile(file, inTax);
 	
 		//read template seqs and save
 		while (!inTax.eof()) {
@@ -230,7 +315,7 @@ int Classify::readTaxonomy(string file) {
 			
 			phyloTree->addSeqToTree(name, taxInfo);
 		
-			gobble(inTax);
+			m->gobble(inTax);
 		}
 		inTax.close();
 #endif	
@@ -238,7 +323,7 @@ int Classify::readTaxonomy(string file) {
 		phyloTree->assignHeirarchyIDs(0);
 		
 		phyloTree->setUp(file);
-		
+	
 		m->mothurOut("DONE.");
 		m->mothurOutEndLine();	cout.flush();