X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=bayesian.cpp;h=e5ab89e6216426e2cd9ca1c8a7eca16db203cd00;hb=9013e13ecfb2fda3c2664a76f76cc99b8c7fa74c;hp=e59545a89a04031999885beb932dac5003ab6b8c;hpb=d5bf2c1354d0811a33394d918b15620606560d58;p=mothur.git diff --git a/bayesian.cpp b/bayesian.cpp index e59545a..e5ab89e 100644 --- a/bayesian.cpp +++ b/bayesian.cpp @@ -9,93 +9,197 @@ #include "bayesian.h" #include "kmer.hpp" +#include "phylosummary.h" /**************************************************************************************************/ Bayesian::Bayesian(string tfile, string tempFile, string method, int ksize, int cutoff, int i) : -Classify(tfile, tempFile, method, ksize, 0.0, 0.0, 0.0, 0.0), kmerSize(ksize), confidenceThreshold(cutoff), iters(i) { +Classify(), kmerSize(ksize), confidenceThreshold(cutoff), iters(i) { try { - numKmers = database->getMaxKmer() + 1; - - //initialze probabilities - wordGenusProb.resize(numKmers); - - genusNodes = phyloTree->getGenusNodes(); - - for (int j = 0; j < wordGenusProb.size(); j++) { wordGenusProb[j].resize(genusNodes.size()); } - - //reset counts because we are on a new word - for (int j = 0; j < genusNodes.size(); j++) { - TaxNode temp = phyloTree->get(genusNodes[j]); - genusTotals.push_back(temp.accessions.size()); - } - - /************calculate the probablity that each word will be in a specific taxonomy*************/ - ofstream out; - string probFileName = tempFile.substr(0,tempFile.find_last_of(".")+1) + char('0'+ kmerSize) + "mer.prob"; - ifstream probFileTest(probFileName.c_str()); + string tfileroot = tfile.substr(0,tfile.find_last_of(".")+1); + string tempfileroot = getRootName(getSimpleName(tempFile)); + string phyloTreeName = tfileroot + "tree.train"; + string probFileName = tfileroot + tempfileroot + char('0'+ kmerSize) + "mer.prob"; + string probFileName2 = tfileroot + tempfileroot + char('0'+ kmerSize) + "mer.numNonZero"; + ofstream out; ofstream out2; - string probFileName2 = tempFile.substr(0,tempFile.find_last_of(".")+1) + char('0'+ kmerSize) + "mer.numNonZero"; + + ifstream phyloTreeTest(phyloTreeName.c_str()); ifstream probFileTest2(probFileName2.c_str()); + ifstream probFileTest(probFileName.c_str()); int start = time(NULL); - if(probFileTest && probFileTest2){ - mothurOut("Reading template probabilities... "); cout.flush(); - readProbFile(probFileTest, probFileTest2); + if(probFileTest && probFileTest2 && phyloTreeTest){ + m->mothurOut("Reading template taxonomy... "); cout.flush(); + + phyloTree = new PhyloTree(phyloTreeTest, phyloTreeName); + + m->mothurOut("DONE."); m->mothurOutEndLine(); + + genusNodes = phyloTree->getGenusNodes(); + genusTotals = phyloTree->getGenusTotals(); + + m->mothurOut("Reading template probabilities... "); cout.flush(); + readProbFile(probFileTest, probFileTest2, probFileName, probFileName2); + }else{ - mothurOut("Calculating template probabilities... "); cout.flush(); - - ofstream out; - openOutputFile(probFileName, out); + + //create search database and names vector + generateDatabaseAndNames(tfile, tempFile, method, ksize, 0.0, 0.0, 0.0, 0.0); + + //prevents errors caused by creating shortcut files if you had an error in the sanity check. + if (m->control_pressed) { remove(phyloTreeName.c_str()); remove(probFileName.c_str()); remove(probFileName2.c_str()); } + else{ + genusNodes = phyloTree->getGenusNodes(); + genusTotals = phyloTree->getGenusTotals(); + + m->mothurOut("Calculating template taxonomy tree... "); cout.flush(); + + phyloTree->printTreeNodes(phyloTreeName); + + m->mothurOut("DONE."); m->mothurOutEndLine(); + + m->mothurOut("Calculating template probabilities... "); cout.flush(); + + numKmers = database->getMaxKmer() + 1; - ofstream out2; - openOutputFile(probFileName2, out2); + //initialze probabilities + wordGenusProb.resize(numKmers); - //for each word - for (int i = 0; i < numKmers; i++) { + for (int j = 0; j < wordGenusProb.size(); j++) { wordGenusProb[j].resize(genusNodes.size()); } + + ofstream out; + ofstream out2; - out << i << '\t'; + #ifdef USE_MPI + int pid; + MPI_Comm_rank(MPI_COMM_WORLD, &pid); //find out who we are + + if (pid == 0) { + #endif + + + openOutputFile(probFileName, out); + + out << numKmers << endl; - vector seqsWithWordi = database->getSequencesWithKmer(i); + openOutputFile(probFileName2, out2); - map count; - for (int k = 0; k < genusNodes.size(); k++) { count[genusNodes[k]] = 0; } + #ifdef USE_MPI + } + #endif + + + //for each word + for (int i = 0; i < numKmers; i++) { + if (m->control_pressed) { break; } + + #ifdef USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &pid); //find out who we are + + if (pid == 0) { + #endif + + out << i << '\t'; + + #ifdef USE_MPI + } + #endif + + vector seqsWithWordi = database->getSequencesWithKmer(i); + + map count; + for (int k = 0; k < genusNodes.size(); k++) { count[genusNodes[k]] = 0; } + + //for each sequence with that word + for (int j = 0; j < seqsWithWordi.size(); j++) { + int temp = phyloTree->getIndex(names[seqsWithWordi[j]]); + count[temp]++; //increment count of seq in this genus who have this word + } + + //probabilityInTemplate = (# of seqs with that word in template + 0.05) / (total number of seqs in template + 1); + float probabilityInTemplate = (seqsWithWordi.size() + 0.50) / (float) (names.size() + 1); + + int numNotZero = 0; + for (int k = 0; k < genusNodes.size(); k++) { + //probabilityInThisTaxonomy = (# of seqs with that word in this taxonomy + probabilityInTemplate) / (total number of seqs in this taxonomy + 1); + wordGenusProb[i][k] = log((count[genusNodes[k]] + probabilityInTemplate) / (float) (genusTotals[k] + 1)); + if (count[genusNodes[k]] != 0) { + #ifdef USE_MPI + int pid; + MPI_Comm_rank(MPI_COMM_WORLD, &pid); //find out who we are - //for each sequence with that word - for (int j = 0; j < seqsWithWordi.size(); j++) { - int temp = phyloTree->getIndex(names[seqsWithWordi[j]]); - count[temp]++; //increment count of seq in this genus who have this word + if (pid == 0) { + #endif + + out << k << '\t' << wordGenusProb[i][k] << '\t'; + + #ifdef USE_MPI + } + #endif + + numNotZero++; + } + } + + #ifdef USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &pid); //find out who we are + + if (pid == 0) { + #endif + + out << endl; + out2 << probabilityInTemplate << '\t' << numNotZero << endl; + + #ifdef USE_MPI + } + #endif } - //probabilityInTemplate = (# of seqs with that word in template + 0.05) / (total number of seqs in template + 1); - float probabilityInTemplate = (seqsWithWordi.size() + 0.50) / (float) (names.size() + 1); + #ifdef USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &pid); //find out who we are - int numNotZero = 0; - for (int k = 0; k < genusNodes.size(); k++) { - //probabilityInThisTaxonomy = (# of seqs with that word in this taxonomy + probabilityInTemplate) / (total number of seqs in this taxonomy + 1); - wordGenusProb[i][k] = log((count[genusNodes[k]] + probabilityInTemplate) / (float) (genusTotals[k] + 1)); - if (count[genusNodes[k]] != 0) { out << k << '\t' << wordGenusProb[i][k] << '\t'; numNotZero++; } - } - out << endl; - out2 << probabilityInTemplate << '\t' << numNotZero << endl; + if (pid == 0) { + #endif + + out.close(); + out2.close(); + + #ifdef USE_MPI + } + #endif + + //read in new phylotree with less info. - its faster + ifstream phyloTreeTest(phyloTreeName.c_str()); + delete phyloTree; + + phyloTree = new PhyloTree(phyloTreeTest, phyloTreeName); } - - out.close(); - out2.close(); } - - - mothurOut("DONE."); mothurOutEndLine(); - mothurOut("It took " + toString(time(NULL) - start) + " seconds get probabilities. "); mothurOutEndLine(); + + m->mothurOut("DONE."); m->mothurOutEndLine(); + m->mothurOut("It took " + toString(time(NULL) - start) + " seconds get probabilities. "); m->mothurOutEndLine(); + } + catch(exception& e) { + m->errorOut(e, "Bayesian", "Bayesian"); + exit(1); + } +} +/**************************************************************************************************/ +Bayesian::~Bayesian() { + try { + delete phyloTree; + if (database != NULL) { delete database; } } catch(exception& e) { - errorOut(e, "Bayesian", "getTaxonomy"); + m->errorOut(e, "Bayesian", "~Bayesian"); exit(1); } } + /**************************************************************************************************/ string Bayesian::getTaxonomy(Sequence* seq) { try { @@ -105,16 +209,22 @@ string Bayesian::getTaxonomy(Sequence* seq) { //get words contained in query //getKmerString returns a string where the index in the string is hte kmer number //and the character at that index can be converted to be the number of times that kmer was seen + string queryKmerString = kmer.getKmerString(seq->getUnaligned()); + vector queryKmers; for (int i = 0; i < queryKmerString.length(); i++) { if (queryKmerString[i] != '!') { //this kmer is in the query queryKmers.push_back(i); } } - + + if (queryKmers.size() == 0) { m->mothurOut(seq->getName() + "is bad."); m->mothurOutEndLine(); return "bad seq"; } + int index = getMostProbableTaxonomy(queryKmers); - + + if (m->control_pressed) { return tax; } +//cout << seq->getName() << '\t' << index << endl; //bootstrap - to set confidenceScore int numToSelect = queryKmers.size() / 8; tax = bootstrapResults(queryKmers, index, numToSelect); @@ -122,27 +232,23 @@ string Bayesian::getTaxonomy(Sequence* seq) { return tax; } catch(exception& e) { - errorOut(e, "Bayesian", "getTaxonomy"); + m->errorOut(e, "Bayesian", "getTaxonomy"); exit(1); } } /**************************************************************************************************/ string Bayesian::bootstrapResults(vector kmers, int tax, int numToSelect) { try { - - //taxConfidenceScore.clear(); //clear out previous seqs scores - vector< map > confidenceScores; //you need the added vector level of confusion to account for the level that name is seen since they can be the same - //map of classification to confidence for all areas seen - //ie. Bacteria;Alphaproteobacteria;Rhizobiales;Azorhizobium_et_rel.;Methylobacterium_et_rel.;Bosea; - //ie. Bacteria -> 100, Alphaproteobacteria -> 100, Rhizobiales -> 87, Azorhizobium_et_rel. -> 78, Methylobacterium_et_rel. -> 70, Bosea -> 50 - confidenceScores.resize(100); //if you have more than 100 levels of classification... - - map::iterator itBoot; - map::iterator itBoot2; + map confidenceScores; + + map::iterator itBoot; + map::iterator itBoot2; map::iterator itConvert; for (int i = 0; i < iters; i++) { + if (m->control_pressed) { return "control"; } + vector temp; for (int j = 0; j < numToSelect; j++) { @@ -154,61 +260,66 @@ string Bayesian::bootstrapResults(vector kmers, int tax, int numToSelect) { //get taxonomy int newTax = getMostProbableTaxonomy(temp); - TaxNode taxonomy = phyloTree->get(newTax); - + TaxNode taxonomyTemp = phyloTree->get(newTax); + //add to confidence results - while (taxonomy.level != 0) { //while you are not at the root + while (taxonomyTemp.level != 0) { //while you are not at the root - itBoot2 = confidenceScores[taxonomy.level].find(taxonomy.name); //is this a classification we already have a count on + itBoot2 = confidenceScores.find(newTax); //is this a classification we already have a count on - if (itBoot2 == confidenceScores[taxonomy.level].end()) { //not already in confidence scores - confidenceScores[taxonomy.level][taxonomy.name] = 1; + if (itBoot2 == confidenceScores.end()) { //not already in confidence scores + confidenceScores[newTax] = 1; }else{ - confidenceScores[taxonomy.level][taxonomy.name]++; + confidenceScores[newTax]++; } - - taxonomy = phyloTree->get(taxonomy.parent); + + newTax = taxonomyTemp.parent; + taxonomyTemp = phyloTree->get(newTax); } + } string confidenceTax = ""; simpleTax = ""; + + int seqTaxIndex = tax; TaxNode seqTax = phyloTree->get(tax); while (seqTax.level != 0) { //while you are not at the root - - itBoot2 = confidenceScores[seqTax.level].find(seqTax.name); //is this a classification we already have a count on + + itBoot2 = confidenceScores.find(seqTaxIndex); //is this a classification we already have a count on int confidence = 0; - if (itBoot2 != confidenceScores[seqTax.level].end()) { //not already in confidence scores - confidence = confidenceScores[seqTax.level][seqTax.name]; + if (itBoot2 != confidenceScores.end()) { //already in confidence scores + confidence = confidenceScores[seqTaxIndex]; } - if (confidence >= confidenceThreshold) { + if (((confidence/(float)iters) * 100) >= confidenceThreshold) { confidenceTax = seqTax.name + "(" + toString(((confidence/(float)iters) * 100)) + ");" + confidenceTax; simpleTax = seqTax.name + ";" + simpleTax; } + seqTaxIndex = seqTax.parent; seqTax = phyloTree->get(seqTax.parent); } + if (confidenceTax == "") { confidenceTax = "unclassified;"; simpleTax = "unclassified;"; } return confidenceTax; } catch(exception& e) { - errorOut(e, "Bayesian", "bootstrapResults"); + m->errorOut(e, "Bayesian", "bootstrapResults"); exit(1); } } /**************************************************************************************************/ int Bayesian::getMostProbableTaxonomy(vector queryKmer) { try { - int indexofGenus; + int indexofGenus = 0; double maxProbability = -1000000.0; //find taxonomy with highest probability that this sequence is from it for (int k = 0; k < genusNodes.size(); k++) { - //for each taxonomy calc its probability double prob = 1.0; for (int i = 0; i < queryKmer.size(); i++) { @@ -225,7 +336,7 @@ int Bayesian::getMostProbableTaxonomy(vector queryKmer) { return indexofGenus; } catch(exception& e) { - errorOut(e, "Bayesian", "getMostProbableTaxonomy"); + m->errorOut(e, "Bayesian", "getMostProbableTaxonomy"); exit(1); } } @@ -252,46 +363,171 @@ map Bayesian::parseTaxMap(string newTax) { } catch(exception& e) { - errorOut(e, "Bayesian", "parseTax"); + m->errorOut(e, "Bayesian", "parseTax"); exit(1); } } /**************************************************************************************************/ -void Bayesian::readProbFile(ifstream& in, ifstream& inNum) { +void Bayesian::readProbFile(ifstream& in, ifstream& inNum, string inName, string inNumName) { try{ - int kmer, name, count; count = 0; - vector num; num.resize(numKmers); - float prob; - vector zeroCountProb; zeroCountProb.resize(numKmers); - - while (inNum) { - inNum >> zeroCountProb[count] >> num[count]; - count++; - gobble(inNum); - } - inNum.close(); + #ifdef USE_MPI + + int pid, num, num2, processors; + vector positions; + vector positions2; + + MPI_Status status; + MPI_File inMPI; + MPI_File inMPI2; + MPI_Comm_rank(MPI_COMM_WORLD, &pid); //find out who we are + MPI_Comm_size(MPI_COMM_WORLD, &processors); + int tag = 2001; + + char inFileName[1024]; + strcpy(inFileName, inNumName.c_str()); + + char inFileName2[1024]; + strcpy(inFileName2, inName.c_str()); + + MPI_File_open(MPI_COMM_WORLD, inFileName, MPI_MODE_RDONLY, MPI_INFO_NULL, &inMPI); //comm, filename, mode, info, filepointer + MPI_File_open(MPI_COMM_WORLD, inFileName2, MPI_MODE_RDONLY, MPI_INFO_NULL, &inMPI2); //comm, filename, mode, info, filepointer + + if (pid == 0) { + positions = setFilePosEachLine(inNumName, num); + positions2 = setFilePosEachLine(inName, num2); + + for(int i = 1; i < processors; i++) { + MPI_Send(&num, 1, MPI_INT, i, tag, MPI_COMM_WORLD); + MPI_Send(&positions[0], (num+1), MPI_LONG, i, tag, MPI_COMM_WORLD); + + MPI_Send(&num2, 1, MPI_INT, i, tag, MPI_COMM_WORLD); + MPI_Send(&positions2[0], (num2+1), MPI_LONG, i, tag, MPI_COMM_WORLD); + } + + }else{ + MPI_Recv(&num, 1, MPI_INT, 0, tag, MPI_COMM_WORLD, &status); + positions.resize(num+1); + MPI_Recv(&positions[0], (num+1), MPI_LONG, 0, tag, MPI_COMM_WORLD, &status); + + MPI_Recv(&num2, 1, MPI_INT, 0, tag, MPI_COMM_WORLD, &status); + positions2.resize(num2+1); + MPI_Recv(&positions2[0], (num2+1), MPI_LONG, 0, tag, MPI_COMM_WORLD, &status); + } - while(in) { - in >> kmer; + //read numKmers + int length = positions2[1] - positions2[0]; + char* buf = new char[length]; + + MPI_File_read_at(inMPI2, positions2[0], buf, length, MPI_CHAR, &status); + + string tempBuf = buf; + if (tempBuf.length() > length) { tempBuf = tempBuf.substr(0, length); } + delete buf; + + istringstream iss (tempBuf,istringstream::in); + iss >> numKmers; + + //initialze probabilities + wordGenusProb.resize(numKmers); + + for (int j = 0; j < wordGenusProb.size(); j++) { wordGenusProb[j].resize(genusNodes.size()); } + + int kmer, name; + vector numbers; numbers.resize(numKmers); + float prob; + vector zeroCountProb; zeroCountProb.resize(numKmers); + + //read file + for(int i=0;i length) { tempBuf = tempBuf.substr(0, length); } + delete buf4; + + istringstream iss (tempBuf,istringstream::in); + iss >> zeroCountProb[i] >> numbers[i]; + } - //set them all to zero value - for (int i = 0; i < genusNodes.size(); i++) { - wordGenusProb[kmer][i] = log(zeroCountProb[kmer] / (float) (genusTotals[i]+1)); + MPI_File_close(&inMPI); + + for(int i=1;i length) { tempBuf = tempBuf.substr(0, length); } + delete buf4; + + istringstream iss (tempBuf,istringstream::in); + + iss >> kmer; + + //set them all to zero value + for (int i = 0; i < genusNodes.size(); i++) { + wordGenusProb[kmer][i] = log(zeroCountProb[kmer] / (float) (genusTotals[i]+1)); + } + + //get probs for nonzero values + for (int i = 0; i < numbers[kmer]; i++) { + iss >> name >> prob; + wordGenusProb[kmer][name] = prob; + } + } + MPI_File_close(&inMPI2); + MPI_Barrier(MPI_COMM_WORLD); //make everyone wait - just in case + #else + + in >> numKmers; gobble(in); + + //initialze probabilities + wordGenusProb.resize(numKmers); - //get probs for nonzero values - for (int i = 0; i < num[kmer]; i++) { - in >> name >> prob; - wordGenusProb[kmer][name] = prob; + for (int j = 0; j < wordGenusProb.size(); j++) { wordGenusProb[j].resize(genusNodes.size()); } + + int kmer, name, count; count = 0; + vector num; num.resize(numKmers); + float prob; + vector zeroCountProb; zeroCountProb.resize(numKmers); + + while (inNum) { + inNum >> zeroCountProb[count] >> num[count]; + count++; + gobble(inNum); } + inNum.close(); + + while(in) { + in >> kmer; + + //set them all to zero value + for (int i = 0; i < genusNodes.size(); i++) { + wordGenusProb[kmer][i] = log(zeroCountProb[kmer] / (float) (genusTotals[i]+1)); + } + + //get probs for nonzero values + for (int i = 0; i < num[kmer]; i++) { + in >> name >> prob; + wordGenusProb[kmer][name] = prob; + } + + gobble(in); + } + in.close(); - gobble(in); - } - in.close(); + #endif } catch(exception& e) { - errorOut(e, "Bayesian", "readProbFile"); + m->errorOut(e, "Bayesian", "readProbFile"); exit(1); } }