From fe08eacea5a2d34cd4bd27e8451fd2872102a80a Mon Sep 17 00:00:00 2001 From: Sarah Westcott Date: Tue, 22 Oct 2013 08:25:55 -0400 Subject: [PATCH] added picrust and ref taxonomy parameters to make.biom --- makebiomcommand.cpp | 244 ++++++++++++++++++++++++++++++++++++++------ makebiomcommand.h | 7 +- phylotree.cpp | 42 ++++++++ phylotree.h | 1 + qualityscores.cpp | 2 +- 5 files changed, 264 insertions(+), 32 deletions(-) diff --git a/makebiomcommand.cpp b/makebiomcommand.cpp index 0d289d7..826cf37 100644 --- a/makebiomcommand.cpp +++ b/makebiomcommand.cpp @@ -10,6 +10,7 @@ #include "sharedrabundvector.h" #include "inputdata.h" #include "sharedutilities.h" +#include "phylotree.h" //taken from http://biom-format.org/documentation/biom_format.html /* Minimal Sparse @@ -94,11 +95,11 @@ vector MakeBiomCommand::setParameters(){ try { CommandParameter pshared("shared", "InputTypes", "", "", "none", "none", "none","biom",false,true,true); parameters.push_back(pshared); CommandParameter pcontaxonomy("constaxonomy", "InputTypes", "", "", "none", "none", "none","",false,false); parameters.push_back(pcontaxonomy); - //CommandParameter preference("referencetax", "InputTypes", "", "", "none", "none", "none","",false,false); parameters.push_back(preference); + CommandParameter preference("reftaxonomy", "InputTypes", "", "", "none", "none", "refPi","",false,false); parameters.push_back(preference); CommandParameter pmetadata("metadata", "InputTypes", "", "", "none", "none", "none","",false,false); parameters.push_back(pmetadata); CommandParameter pgroups("groups", "String", "", "", "", "", "","",false,false); parameters.push_back(pgroups); CommandParameter plabel("label", "String", "", "", "", "", "","",false,false); parameters.push_back(plabel); - //CommandParameter ppicrust("picrust", "Boolean", "", "F", "", "", "","",false,false); parameters.push_back(ppicrust); + CommandParameter ppicrust("picrust", "InputTypes", "", "", "none", "none", "refPi","shared",false,false); parameters.push_back(ppicrust); CommandParameter pinputdir("inputdir", "String", "", "", "", "", "","",false,false); parameters.push_back(pinputdir); CommandParameter poutputdir("outputdir", "String", "", "", "", "", "","",false,false); parameters.push_back(poutputdir); CommandParameter pmatrixtype("matrixtype", "Multiple", "sparse-dense", "sparse", "", "", "","",false,false); parameters.push_back(pmatrixtype); @@ -116,14 +117,14 @@ vector MakeBiomCommand::setParameters(){ string MakeBiomCommand::getHelpString(){ try { string helpString = ""; - helpString += "The make.biom command parameters are shared, contaxonomy, metadata, groups, matrixtype and label. shared is required, unless you have a valid current file.\n"; //, picrust and referencetax + helpString += "The make.biom command parameters are shared, contaxonomy, metadata, groups, matrixtype, picrust, reftaxonomy and label. shared is required, unless you have a valid current file.\n"; // helpString += "The groups parameter allows you to specify which of the groups in your groupfile you would like included. The group names are separated by dashes.\n"; helpString += "The label parameter allows you to select what distance levels you would like, and are also separated by dashes.\n"; helpString += "The matrixtype parameter allows you to select what type you would like to make. Choices are sparse and dense, default is sparse.\n"; helpString += "The contaxonomy file is the taxonomy file outputted by classify.otu(list=yourListfile, taxonomy=yourTaxonomyFile). Be SURE that the you are the constaxonomy file distance matches the shared file distance. ie, for *.0.03.cons.taxonomy set label=0.03. Mothur is smart enough to handle shared files that have been subsampled. It is used to assign taxonomy information to the metadata of rows.\n"; helpString += "The metadata parameter is used to provide experimental parameters to the columns. Things like 'sample1 gut human_gut'. \n"; - //helpString += "The picrust parameter is used to indicate the biom file is for input to picrust. NOTE: Picrust requires a greengenes taxonomy. \n"; - //helpString += "The referencetax parameter is used with the picrust parameter. Picrust requires the name of the reference taxonomy sequence to be in the biom file. \n"; + helpString += "The picrust parameter is used to provide the greengenes OTU IDs map table. NOTE: Picrust requires a greengenes taxonomy. \n"; + helpString += "The referencetax parameter is used with the picrust parameter. Picrust requires the greengenes OTU IDs to be in the biom file. \n"; helpString += "The make.biom command should be in the following format: make.biom(shared=yourShared, groups=yourGroups, label=yourLabels).\n"; helpString += "Example make.biom(shared=abrecovery.an.shared, groups=A-B-C).\n"; helpString += "The default value for groups is all the groups in your groupfile, and all labels in your inputfile will be used.\n"; @@ -141,7 +142,8 @@ string MakeBiomCommand::getOutputPattern(string type) { try { string pattern = ""; - if (type == "biom") { pattern = "[filename],[distance],biom"; } + if (type == "biom") { pattern = "[filename],[distance],biom"; } + else if (type == "shared") { pattern = "[filename],[distance],biom_shared"; } else { m->mothurOut("[ERROR]: No definition for type " + type + " output pattern.\n"); m->control_pressed = true; } return pattern; @@ -159,6 +161,7 @@ MakeBiomCommand::MakeBiomCommand(){ setParameters(); vector tempOutNames; outputTypes["biom"] = tempOutNames; + outputTypes["shared"] = tempOutNames; } catch(exception& e) { m->errorOut(e, "MakeBiomCommand", "MakeBiomCommand"); @@ -192,6 +195,7 @@ MakeBiomCommand::MakeBiomCommand(string option) { //initialize outputTypes vector tempOutNames; outputTypes["biom"] = tempOutNames; + outputTypes["shared"] = tempOutNames; //if the user changes the input directory command factory will send this info to us in the output parameter string inputDir = validParameter.validFile(parameters, "inputdir", false); @@ -214,12 +218,20 @@ MakeBiomCommand::MakeBiomCommand(string option) { if (path == "") { parameters["constaxonomy"] = inputDir + it->second; } } - it = parameters.find("referencetax"); + it = parameters.find("reftaxonomy"); //user has given a template file if(it != parameters.end()){ path = m->hasPath(it->second); //if the user has not given a path then, add inputdir. else leave path alone. - if (path == "") { parameters["referencetax"] = inputDir + it->second; } + if (path == "") { parameters["reftaxonomy"] = inputDir + it->second; } + } + + it = parameters.find("picrust"); + //user has given a template file + if(it != parameters.end()){ + path = m->hasPath(it->second); + //if the user has not given a path then, add inputdir. else leave path alone. + if (path == "") { parameters["picrust"] = inputDir + it->second; } } it = parameters.find("metadata"); @@ -249,9 +261,13 @@ MakeBiomCommand::MakeBiomCommand(string option) { if (contaxonomyfile == "not found") { contaxonomyfile = ""; } else if (contaxonomyfile == "not open") { contaxonomyfile = ""; abort = true; } - //referenceTax = validParameter.validFile(parameters, "referencetax", true); - //if (referenceTax == "not found") { referenceTax = ""; } - //else if (referenceTax == "not open") { referenceTax = ""; abort = true; } + referenceTax = validParameter.validFile(parameters, "reftaxonomy", true); + if (referenceTax == "not found") { referenceTax = ""; } + else if (referenceTax == "not open") { referenceTax = ""; abort = true; } + + picrustOtuFile = validParameter.validFile(parameters, "picrust", true); + if (picrustOtuFile == "not found") { picrustOtuFile = ""; } + else if (picrustOtuFile == "not open") { picrustOtuFile = ""; abort = true; } metadatafile = validParameter.validFile(parameters, "metadata", true); if (metadatafile == "not found") { metadatafile = ""; } @@ -265,13 +281,6 @@ MakeBiomCommand::MakeBiomCommand(string option) { if(label != "all") { m->splitAtDash(label, labels); allLines = 0; } else { allLines = 1; } } - - //string temp = validParameter.validFile(parameters, "picrust", false); if (temp == "not found"){ temp = "f"; } - //picrust = m->isTrue(temp); - //if (picrust && ((contaxonomyfile == "") || (referenceTax == ""))) { - //m->mothurOut("[ERROR]: the picrust parameter requires a consensus taxonomy with greengenes taxonomy the reference."); m->mothurOutEndLine(); abort = true; - //} - picrust=false; groups = validParameter.validFile(parameters, "groups", false); if (groups == "not found") { groups = ""; } @@ -280,6 +289,12 @@ MakeBiomCommand::MakeBiomCommand(string option) { m->setGroups(Groups); } + if (picrustOtuFile != "") { + picrust=true; + if (contaxonomyfile == "") { m->mothurOut("[ERROR]: the constaxonomy parameter is required with the picrust parameter, aborting."); m->mothurOutEndLine(); abort = true; } + if (referenceTax == "") { m->mothurOut("[ERROR]: the reftaxonomy parameter is required with the picrust parameter, aborting."); m->mothurOutEndLine(); abort = true; } + }else { picrust=false; } + if ((contaxonomyfile != "") && (labels.size() > 1)) { m->mothurOut("[ERROR]: the contaxonomy parameter cannot be used with multiple labels."); m->mothurOutEndLine(); abort = true; } format = validParameter.validFile(parameters, "matrixtype", false); if (format == "not found") { format = "sparse"; } @@ -428,9 +443,9 @@ int MakeBiomCommand::getBiom(vector& lookup){ out << "{\n" + spaces + "\"id\":\"" + sharedfile + "-" + lookup[0]->getLabel() + "\",\n" + spaces + "\"format\": \"Biological Observation Matrix 0.9.1\",\n" + spaces + "\"format_url\": \"http://biom-format.org\",\n"; out << spaces + "\"type\": \"OTU table\",\n" + spaces + "\"generated_by\": \"" << mothurString << "\",\n" + spaces + "\"date\": \"" << dateString << "\",\n"; + + vector metadata = getMetaData(lookup); int numBins = lookup[0]->getNumBins(); - vector picrustLabels; - vector metadata = getMetaData(lookup, picrustLabels); if (m->control_pressed) { out.close(); return 0; } @@ -447,11 +462,10 @@ int MakeBiomCommand::getBiom(vector& lookup){ string rowBack = "\", \"metadata\":"; for (int i = 0; i < numBins-1; i++) { if (m->control_pressed) { out.close(); return 0; } - if (!picrust) { out << rowFront << m->currentSharedBinLabels[i] << rowBack << metadata[i] << "},\n"; } - else { out << rowFront << picrustLabels[i] << rowBack << metadata[i] << "},\n"; } + out << rowFront << m->currentSharedBinLabels[i] << rowBack << metadata[i] << "},\n"; } - if (!picrust) { out << rowFront << m->currentSharedBinLabels[(numBins-1)] << rowBack << metadata[(numBins-1)] << "}\n" + spaces + "],\n"; } - else { out << rowFront << picrustLabels[(numBins-1)] << rowBack << metadata[(numBins-1)] << "}\n" + spaces + "],\n"; } + out << rowFront << m->currentSharedBinLabels[(numBins-1)] << rowBack << metadata[(numBins-1)] << "}\n" + spaces + "],\n"; + //get column info /*"columns": [ {"id":"Sample1", "metadata":null}, @@ -543,7 +557,7 @@ int MakeBiomCommand::getBiom(vector& lookup){ } } //********************************************************************************************************************** -vector MakeBiomCommand::getMetaData(vector& lookup, vector& picrustLabels){ +vector MakeBiomCommand::getMetaData(vector& lookup){ try { vector metadata; @@ -601,6 +615,8 @@ vector MakeBiomCommand::getMetaData(vector& lookup, }else { labelTaxMap[m->getSimpleLabel(otuLabels[i])] = taxs[i]; } } + //merges OTUs classified to same gg otuid, sets otulabels to gg otuids, averages confidence scores of merged otus. overwritting of otulabels is fine because constaxonomy only allows for one label to be processed. If this assumption changes, could cause bug. + if (picrust) { getGreenGenesOTUIDs(lookup, labelTaxMap); } //{"taxonomy":["k__Bacteria", "p__Proteobacteria", "c__Gammaproteobacteria", "o__Enterobacteriales", "f__Enterobacteriaceae", "g__Escherichia", "s__"]} @@ -615,10 +631,6 @@ vector MakeBiomCommand::getMetaData(vector& lookup, if (it == labelTaxMap.end()) { m->mothurOut("[ERROR]: can't find taxonomy information for " + m->currentSharedBinLabels[i] + ".\n"); m->control_pressed = true; } else { - if (picrust) { - string temp = it->second; m->removeConfidences(temp); - picrustLabels.push_back(temp); - } vector bootstrapValues; string data = "{\"taxonomy\":["; @@ -651,6 +663,180 @@ vector MakeBiomCommand::getMetaData(vector& lookup, exit(1); } +} +//********************************************************************************************************************** +int MakeBiomCommand::getGreenGenesOTUIDs(vector& lookup, map& labelTaxMap){ + try { + //read reftaxonomy + PhyloTree phylo(referenceTax); + + //read otu map file + map otuMap = readGGOtuMap(); //maps reference ID -> OTU ID + + if (m->control_pressed) { return 0; } + + map > ggOTUIDs; + //loop through otu taxonomies + for (map::iterator it = labelTaxMap.begin(); it != labelTaxMap.end(); it++) { //maps label -> consensus taxonomy + if (m->control_pressed) { break; } + + //get list of reference ids that map to this taxonomy + vector referenceIds = phylo.getSeqs(it->second); + + if (m->control_pressed) { break; } + + //look for each one in otu map to find match + string otuID = "not found"; + string referenceString = ""; + for (int i = 0; i < referenceIds.size(); i++) { + referenceString += referenceIds[i] + " "; + map::iterator itMap = otuMap.find(referenceIds[i]); + if (itMap != otuMap.end()) { //found it + otuID = itMap->second; + i += referenceIds.size(); //stop looking + } + } + + //if found, add otu to ggOTUID list + if (otuID != "not found") { + map >::iterator itGG = ggOTUIDs.find(otuID); + if (itGG == ggOTUIDs.end()) { + vector temp; temp.push_back(it->first); //save mothur OTU label + ggOTUIDs[otuID] = temp; + }else { ggOTUIDs[otuID].push_back(it->first); } //add mothur OTU label to list + }else { m->mothurOut("[ERROR]: could not find OTUId for " + it->second + ". Its reference sequences are " + referenceString + ".\n"); m->control_pressed = true; } + + } + + + vector newLookup; + for (int i = 0; i < lookup.size(); i++) { + SharedRAbundVector* temp = new SharedRAbundVector(); + temp->setLabel(lookup[i]->getLabel()); + temp->setGroup(lookup[i]->getGroup()); + newLookup.push_back(temp); + } + + map labelIndex; + for (int i = 0; i < m->currentSharedBinLabels.size(); i++) { labelIndex[m->getSimpleLabel(m->currentSharedBinLabels[i])] = i; } + + vector newBinLabels; + map newLabelTaxMap; + //loop through ggOTUID list combining mothur otus and adjusting labels + //ggOTUIDs = 16097 -> + for (map >::iterator itMap = ggOTUIDs.begin(); itMap != ggOTUIDs.end(); itMap++) { + if (m->control_pressed) { for (int j = 0; j < newLookup.size(); j++) { delete newLookup[j]; } return 0; } + + //set new gg otu id to taxonomy. OTU01 -> k__Bacteria becomes 16097 -> k__Bacteria + //find taxonomy of this otu + map::iterator it = labelTaxMap.find(m->getSimpleLabel(itMap->second[0])); + vector scores; + vector taxonomies = parseTax(it->second, scores); + + //merge/set OTU abundances + vector abunds; abunds.resize(lookup.size(), 0); + string mergeString = ""; + vector boots; boots.resize(scores.size(), 0); + for (int j = 0; j < itMap->second.size(); j++) { // + //merge bootstrap scores + vector scores; + vector taxonomies = parseTax(it->second, scores); + for (int i = 0; i < boots.size(); i++) { + float tempScore; m->mothurConvert(scores[i], tempScore); + boots[i] += tempScore; + } + + //merge abunds + mergeString += (itMap->second)[j] + " "; + for (int i = 0; i < lookup.size(); i++) { + abunds[i] += lookup[i]->getAbundance(labelIndex[m->getSimpleLabel((itMap->second)[j])]); + } + } + + if (m->debug) { m->mothurOut("[DEBUG]: merging " + mergeString + " for ggOTUid = " + itMap->first + ".\n"); } + + //average scores + //add merged otu to new lookup + for (int j = 0; j < boots.size(); j++) { boots[j] /= (float) itMap->second.size(); } + + //assemble new taxomoy + string newTaxString = ""; + for (int j = 0; j < boots.size(); j++) { + newTaxString += taxonomies[j] + "(" + toString(boots[j]) + ");"; + } + + //set new gg otu id to taxonomy. OTU01 -> k__Bacteria becomes 16097 -> k__Bacteria + //find taxonomy of this otu + newLabelTaxMap[itMap->first] = newTaxString; + + //add merged otu to new lookup + for (int j = 0; j < abunds.size(); j++) { newLookup[j]->push_back(abunds[j], newLookup[j]->getGroup()); } + + //saved otu label + newBinLabels.push_back(itMap->first); + } + + for (int j = 0; j < lookup.size(); j++) { delete lookup[j]; } + + lookup = newLookup; + m->currentSharedBinLabels = newBinLabels; + labelTaxMap = newLabelTaxMap; + + map variables; + variables["[filename]"] = outputDir + m->getRootName(m->getSimpleName(sharedfile)); + variables["[distance]"] = lookup[0]->getLabel(); + string outputFileName = getOutputFileName("shared",variables); + ofstream out; + m->openOutputFile(outputFileName, out); + outputNames.push_back(outputFileName); outputTypes["shared"].push_back(outputFileName); + + lookup[0]->printHeaders(out); + + for (int i = 0; i < lookup.size(); i++) { + out << lookup[i]->getLabel() << '\t' << lookup[i]->getGroup() << '\t'; + lookup[i]->print(out); + } + out.close(); + + return 0; + } + catch(exception& e) { + m->errorOut(e, "MakeBiomCommand", "getGreenGenesOTUIDs"); + exit(1); + } + +} +//********************************************************************************************************************** +map MakeBiomCommand::readGGOtuMap(){ + try { + map otuMap; + + ifstream in; + m->openInputFile(picrustOtuFile, in); + + //map referenceIDs -> otuIDs + //lines look like: + //16097 671376 616121 533566 683683 4332909 4434717 772666 611808 695209 + while(!in.eof()) { + if (m->control_pressed) { break; } + + string line = m->getline(in); m->gobble(in); + vector pieces = m->splitWhiteSpace(line); + + if (pieces.size() != 0) { + string otuID = pieces[0]; + for (int i = 1; i < pieces.size(); i++) { otuMap[pieces[i]] = otuID; } + } + } + in.close(); + + return otuMap; + } + catch(exception& e) { + m->errorOut(e, "MakeBiomCommand", "readGGOtuMap"); + exit(1); + } + } //********************************************************************************************************************** int MakeBiomCommand::getSampleMetaData(vector& lookup){ diff --git a/makebiomcommand.h b/makebiomcommand.h index 1ff3985..8f07c40 100644 --- a/makebiomcommand.h +++ b/makebiomcommand.h @@ -36,16 +36,19 @@ public: private: - string sharedfile, contaxonomyfile, metadatafile, groups, outputDir, format, label, referenceTax; + string sharedfile, contaxonomyfile, metadatafile, groups, outputDir, format, label, referenceTax, picrustOtuFile; vector outputNames, Groups, sampleMetadata; set labels; bool abort, allLines, picrust; int getBiom(vector&); - vector getMetaData(vector&, vector&); + vector getMetaData(vector&); vector parseTax(string tax, vector& scores); int getSampleMetaData(vector&); + //for picrust + int getGreenGenesOTUIDs(vector&, map&); + map readGGOtuMap(); }; diff --git a/phylotree.cpp b/phylotree.cpp index b9bab4e..2e3136e 100644 --- a/phylotree.cpp +++ b/phylotree.cpp @@ -233,6 +233,48 @@ string PhyloTree::getNextTaxon(string& heirarchy, string seqname){ exit(1); } } +/**************************************************************************************************/ + +vector PhyloTree::getSeqs(string seqTaxonomy){ + try { + string taxCopy = seqTaxonomy; + vector names; + map::iterator childPointer; + + int currentNode = 0; + + m->removeConfidences(seqTaxonomy); + + string taxon; + while(seqTaxonomy != ""){ + + if (m->control_pressed) { return names; } + + taxon = getNextTaxon(seqTaxonomy, ""); + + if (m->debug) { m->mothurOut(taxon +'\n'); } + + if (taxon == "") { m->mothurOut(taxCopy + " has an error in the taxonomy. This may be due to a ;;"); m->mothurOutEndLine(); break; } + + childPointer = tree[currentNode].children.find(taxon); + + if(childPointer != tree[currentNode].children.end()){ //if the node already exists, move on + currentNode = childPointer->second; + } + else{ //otherwise, error this taxonomy is not in tree + m->mothurOut("[ERROR]: " + taxCopy + " is not in taxonomy tree, please correct."); m->mothurOutEndLine(); m->control_pressed = true; return names; + } + + if (seqTaxonomy == "") { names = tree[currentNode].accessions; } + } + + return names; + } + catch(exception& e) { + m->errorOut(e, "PhyloTree", "getSeqs"); + exit(1); + } +} /**************************************************************************************************/ diff --git a/phylotree.h b/phylotree.h index e000220..4b7a35a 100644 --- a/phylotree.h +++ b/phylotree.h @@ -46,6 +46,7 @@ public: string getName(int i); int getGenusIndex(string seqName); string getFullTaxonomy(string); //pass a sequence name return taxonomy + vector getSeqs(string); //returns names of sequences in given taxonomy int getMaxLevel() { return maxLevel; } int getNumSeqs() { return numSeqs; } diff --git a/qualityscores.cpp b/qualityscores.cpp index 1f86efc..0c55650 100644 --- a/qualityscores.cpp +++ b/qualityscores.cpp @@ -200,7 +200,7 @@ bool QualityScores::stripQualThreshold(Sequence& sequence, double qThreshold){ if(seqName != sequence.getName()){ m->mothurOut("sequence name mismatch btwn fasta: " + sequence.getName() + " and qual file: " + seqName); - m->mothurOutEndLine(); + m->mothurOutEndLine(); m->control_pressed = true; } int end; -- 2.39.2