#include "sharedrabundvector.h"
#include "inputdata.h"
#include "sharedutilities.h"
+#include "phylotree.h"
//taken from http://biom-format.org/documentation/biom_format.html
/* Minimal Sparse
try {
CommandParameter pshared("shared", "InputTypes", "", "", "none", "none", "none","biom",false,true,true); parameters.push_back(pshared);
CommandParameter pcontaxonomy("constaxonomy", "InputTypes", "", "", "none", "none", "none","",false,false); parameters.push_back(pcontaxonomy);
- //CommandParameter preference("referencetax", "InputTypes", "", "", "none", "none", "none","",false,false); parameters.push_back(preference);
+ CommandParameter preference("reftaxonomy", "InputTypes", "", "", "none", "none", "refPi","",false,false); parameters.push_back(preference);
CommandParameter pmetadata("metadata", "InputTypes", "", "", "none", "none", "none","",false,false); parameters.push_back(pmetadata);
CommandParameter pgroups("groups", "String", "", "", "", "", "","",false,false); parameters.push_back(pgroups);
CommandParameter plabel("label", "String", "", "", "", "", "","",false,false); parameters.push_back(plabel);
- //CommandParameter ppicrust("picrust", "Boolean", "", "F", "", "", "","",false,false); parameters.push_back(ppicrust);
+ CommandParameter ppicrust("picrust", "InputTypes", "", "", "none", "none", "refPi","shared",false,false); parameters.push_back(ppicrust);
CommandParameter pinputdir("inputdir", "String", "", "", "", "", "","",false,false); parameters.push_back(pinputdir);
CommandParameter poutputdir("outputdir", "String", "", "", "", "", "","",false,false); parameters.push_back(poutputdir);
CommandParameter pmatrixtype("matrixtype", "Multiple", "sparse-dense", "sparse", "", "", "","",false,false); parameters.push_back(pmatrixtype);
string MakeBiomCommand::getHelpString(){
try {
string helpString = "";
- helpString += "The make.biom command parameters are shared, contaxonomy, metadata, groups, matrixtype and label. shared is required, unless you have a valid current file.\n"; //, picrust and referencetax
+ helpString += "The make.biom command parameters are shared, contaxonomy, metadata, groups, matrixtype, picrust, reftaxonomy and label. shared is required, unless you have a valid current file.\n"; //
helpString += "The groups parameter allows you to specify which of the groups in your groupfile you would like included. The group names are separated by dashes.\n";
helpString += "The label parameter allows you to select what distance levels you would like, and are also separated by dashes.\n";
helpString += "The matrixtype parameter allows you to select what type you would like to make. Choices are sparse and dense, default is sparse.\n";
helpString += "The contaxonomy file is the taxonomy file outputted by classify.otu(list=yourListfile, taxonomy=yourTaxonomyFile). Be SURE that the you are the constaxonomy file distance matches the shared file distance. ie, for *.0.03.cons.taxonomy set label=0.03. Mothur is smart enough to handle shared files that have been subsampled. It is used to assign taxonomy information to the metadata of rows.\n";
helpString += "The metadata parameter is used to provide experimental parameters to the columns. Things like 'sample1 gut human_gut'. \n";
- //helpString += "The picrust parameter is used to indicate the biom file is for input to picrust. NOTE: Picrust requires a greengenes taxonomy. \n";
- //helpString += "The referencetax parameter is used with the picrust parameter. Picrust requires the name of the reference taxonomy sequence to be in the biom file. \n";
+ helpString += "The picrust parameter is used to provide the greengenes OTU IDs map table. NOTE: Picrust requires a greengenes taxonomy. \n";
+ helpString += "The referencetax parameter is used with the picrust parameter. Picrust requires the greengenes OTU IDs to be in the biom file. \n";
helpString += "The make.biom command should be in the following format: make.biom(shared=yourShared, groups=yourGroups, label=yourLabels).\n";
helpString += "Example make.biom(shared=abrecovery.an.shared, groups=A-B-C).\n";
helpString += "The default value for groups is all the groups in your groupfile, and all labels in your inputfile will be used.\n";
try {
string pattern = "";
- if (type == "biom") { pattern = "[filename],[distance],biom"; }
+ if (type == "biom") { pattern = "[filename],[distance],biom"; }
+ else if (type == "shared") { pattern = "[filename],[distance],biom_shared"; }
else { m->mothurOut("[ERROR]: No definition for type " + type + " output pattern.\n"); m->control_pressed = true; }
return pattern;
setParameters();
vector<string> tempOutNames;
outputTypes["biom"] = tempOutNames;
+ outputTypes["shared"] = tempOutNames;
}
catch(exception& e) {
m->errorOut(e, "MakeBiomCommand", "MakeBiomCommand");
//initialize outputTypes
vector<string> tempOutNames;
outputTypes["biom"] = tempOutNames;
+ outputTypes["shared"] = tempOutNames;
//if the user changes the input directory command factory will send this info to us in the output parameter
string inputDir = validParameter.validFile(parameters, "inputdir", false);
if (path == "") { parameters["constaxonomy"] = inputDir + it->second; }
}
- it = parameters.find("referencetax");
+ it = parameters.find("reftaxonomy");
//user has given a template file
if(it != parameters.end()){
path = m->hasPath(it->second);
//if the user has not given a path then, add inputdir. else leave path alone.
- if (path == "") { parameters["referencetax"] = inputDir + it->second; }
+ if (path == "") { parameters["reftaxonomy"] = inputDir + it->second; }
+ }
+
+ it = parameters.find("picrust");
+ //user has given a template file
+ if(it != parameters.end()){
+ path = m->hasPath(it->second);
+ //if the user has not given a path then, add inputdir. else leave path alone.
+ if (path == "") { parameters["picrust"] = inputDir + it->second; }
}
it = parameters.find("metadata");
if (contaxonomyfile == "not found") { contaxonomyfile = ""; }
else if (contaxonomyfile == "not open") { contaxonomyfile = ""; abort = true; }
- //referenceTax = validParameter.validFile(parameters, "referencetax", true);
- //if (referenceTax == "not found") { referenceTax = ""; }
- //else if (referenceTax == "not open") { referenceTax = ""; abort = true; }
+ referenceTax = validParameter.validFile(parameters, "reftaxonomy", true);
+ if (referenceTax == "not found") { referenceTax = ""; }
+ else if (referenceTax == "not open") { referenceTax = ""; abort = true; }
+
+ picrustOtuFile = validParameter.validFile(parameters, "picrust", true);
+ if (picrustOtuFile == "not found") { picrustOtuFile = ""; }
+ else if (picrustOtuFile == "not open") { picrustOtuFile = ""; abort = true; }
metadatafile = validParameter.validFile(parameters, "metadata", true);
if (metadatafile == "not found") { metadatafile = ""; }
if(label != "all") { m->splitAtDash(label, labels); allLines = 0; }
else { allLines = 1; }
}
-
- //string temp = validParameter.validFile(parameters, "picrust", false); if (temp == "not found"){ temp = "f"; }
- //picrust = m->isTrue(temp);
- //if (picrust && ((contaxonomyfile == "") || (referenceTax == ""))) {
- //m->mothurOut("[ERROR]: the picrust parameter requires a consensus taxonomy with greengenes taxonomy the reference."); m->mothurOutEndLine(); abort = true;
- //}
- picrust=false;
groups = validParameter.validFile(parameters, "groups", false);
if (groups == "not found") { groups = ""; }
m->setGroups(Groups);
}
+ if (picrustOtuFile != "") {
+ picrust=true;
+ if (contaxonomyfile == "") { m->mothurOut("[ERROR]: the constaxonomy parameter is required with the picrust parameter, aborting."); m->mothurOutEndLine(); abort = true; }
+ if (referenceTax == "") { m->mothurOut("[ERROR]: the reftaxonomy parameter is required with the picrust parameter, aborting."); m->mothurOutEndLine(); abort = true; }
+ }else { picrust=false; }
+
if ((contaxonomyfile != "") && (labels.size() > 1)) { m->mothurOut("[ERROR]: the contaxonomy parameter cannot be used with multiple labels."); m->mothurOutEndLine(); abort = true; }
format = validParameter.validFile(parameters, "matrixtype", false); if (format == "not found") { format = "sparse"; }
out << "{\n" + spaces + "\"id\":\"" + sharedfile + "-" + lookup[0]->getLabel() + "\",\n" + spaces + "\"format\": \"Biological Observation Matrix 0.9.1\",\n" + spaces + "\"format_url\": \"http://biom-format.org\",\n";
out << spaces + "\"type\": \"OTU table\",\n" + spaces + "\"generated_by\": \"" << mothurString << "\",\n" + spaces + "\"date\": \"" << dateString << "\",\n";
+
+ vector<string> metadata = getMetaData(lookup);
int numBins = lookup[0]->getNumBins();
- vector<string> picrustLabels;
- vector<string> metadata = getMetaData(lookup, picrustLabels);
if (m->control_pressed) { out.close(); return 0; }
string rowBack = "\", \"metadata\":";
for (int i = 0; i < numBins-1; i++) {
if (m->control_pressed) { out.close(); return 0; }
- if (!picrust) { out << rowFront << m->currentSharedBinLabels[i] << rowBack << metadata[i] << "},\n"; }
- else { out << rowFront << picrustLabels[i] << rowBack << metadata[i] << "},\n"; }
+ out << rowFront << m->currentSharedBinLabels[i] << rowBack << metadata[i] << "},\n";
}
- if (!picrust) { out << rowFront << m->currentSharedBinLabels[(numBins-1)] << rowBack << metadata[(numBins-1)] << "}\n" + spaces + "],\n"; }
- else { out << rowFront << picrustLabels[(numBins-1)] << rowBack << metadata[(numBins-1)] << "}\n" + spaces + "],\n"; }
+ out << rowFront << m->currentSharedBinLabels[(numBins-1)] << rowBack << metadata[(numBins-1)] << "}\n" + spaces + "],\n";
+
//get column info
/*"columns": [
{"id":"Sample1", "metadata":null},
}
}
//**********************************************************************************************************************
-vector<string> MakeBiomCommand::getMetaData(vector<SharedRAbundVector*>& lookup, vector<string>& picrustLabels){
+vector<string> MakeBiomCommand::getMetaData(vector<SharedRAbundVector*>& lookup){
try {
vector<string> metadata;
}else { labelTaxMap[m->getSimpleLabel(otuLabels[i])] = taxs[i]; }
}
+ //merges OTUs classified to same gg otuid, sets otulabels to gg otuids, averages confidence scores of merged otus. overwritting of otulabels is fine because constaxonomy only allows for one label to be processed. If this assumption changes, could cause bug.
+ if (picrust) { getGreenGenesOTUIDs(lookup, labelTaxMap); }
//{"taxonomy":["k__Bacteria", "p__Proteobacteria", "c__Gammaproteobacteria", "o__Enterobacteriales", "f__Enterobacteriaceae", "g__Escherichia", "s__"]}
if (it == labelTaxMap.end()) { m->mothurOut("[ERROR]: can't find taxonomy information for " + m->currentSharedBinLabels[i] + ".\n"); m->control_pressed = true; }
else {
- if (picrust) {
- string temp = it->second; m->removeConfidences(temp);
- picrustLabels.push_back(temp);
- }
vector<string> bootstrapValues;
string data = "{\"taxonomy\":[";
exit(1);
}
+}
+//**********************************************************************************************************************
+int MakeBiomCommand::getGreenGenesOTUIDs(vector<SharedRAbundVector*>& lookup, map<string, string>& labelTaxMap){
+ try {
+ //read reftaxonomy
+ PhyloTree phylo(referenceTax);
+
+ //read otu map file
+ map<string, string> otuMap = readGGOtuMap(); //maps reference ID -> OTU ID
+
+ if (m->control_pressed) { return 0; }
+
+ map<string, vector<string> > ggOTUIDs;
+ //loop through otu taxonomies
+ for (map<string, string>::iterator it = labelTaxMap.begin(); it != labelTaxMap.end(); it++) { //maps label -> consensus taxonomy
+ if (m->control_pressed) { break; }
+
+ //get list of reference ids that map to this taxonomy
+ vector<string> referenceIds = phylo.getSeqs(it->second);
+
+ if (m->control_pressed) { break; }
+
+ //look for each one in otu map to find match
+ string otuID = "not found";
+ string referenceString = "";
+ for (int i = 0; i < referenceIds.size(); i++) {
+ referenceString += referenceIds[i] + " ";
+ map<string, string>::iterator itMap = otuMap.find(referenceIds[i]);
+ if (itMap != otuMap.end()) { //found it
+ otuID = itMap->second;
+ i += referenceIds.size(); //stop looking
+ }
+ }
+
+ //if found, add otu to ggOTUID list
+ if (otuID != "not found") {
+ map<string, vector<string> >::iterator itGG = ggOTUIDs.find(otuID);
+ if (itGG == ggOTUIDs.end()) {
+ vector<string> temp; temp.push_back(it->first); //save mothur OTU label
+ ggOTUIDs[otuID] = temp;
+ }else { ggOTUIDs[otuID].push_back(it->first); } //add mothur OTU label to list
+ }else { m->mothurOut("[ERROR]: could not find OTUId for " + it->second + ". Its reference sequences are " + referenceString + ".\n"); m->control_pressed = true; }
+
+ }
+
+
+ vector<SharedRAbundVector*> newLookup;
+ for (int i = 0; i < lookup.size(); i++) {
+ SharedRAbundVector* temp = new SharedRAbundVector();
+ temp->setLabel(lookup[i]->getLabel());
+ temp->setGroup(lookup[i]->getGroup());
+ newLookup.push_back(temp);
+ }
+
+ map<string, int> labelIndex;
+ for (int i = 0; i < m->currentSharedBinLabels.size(); i++) { labelIndex[m->getSimpleLabel(m->currentSharedBinLabels[i])] = i; }
+
+ vector<string> newBinLabels;
+ map<string, string> newLabelTaxMap;
+ //loop through ggOTUID list combining mothur otus and adjusting labels
+ //ggOTUIDs = 16097 -> <OTU01, OTU10, OTU22>
+ for (map<string, vector<string> >::iterator itMap = ggOTUIDs.begin(); itMap != ggOTUIDs.end(); itMap++) {
+ if (m->control_pressed) { for (int j = 0; j < newLookup.size(); j++) { delete newLookup[j]; } return 0; }
+
+ //set new gg otu id to taxonomy. OTU01 -> k__Bacteria becomes 16097 -> k__Bacteria
+ //find taxonomy of this otu
+ map<string, string>::iterator it = labelTaxMap.find(m->getSimpleLabel(itMap->second[0]));
+ vector<string> scores;
+ vector<string> taxonomies = parseTax(it->second, scores);
+
+ //merge/set OTU abundances
+ vector<int> abunds; abunds.resize(lookup.size(), 0);
+ string mergeString = "";
+ vector<float> boots; boots.resize(scores.size(), 0);
+ for (int j = 0; j < itMap->second.size(); j++) { //<OTU01, OTU10, OTU22>
+ //merge bootstrap scores
+ vector<string> scores;
+ vector<string> taxonomies = parseTax(it->second, scores);
+ for (int i = 0; i < boots.size(); i++) {
+ float tempScore; m->mothurConvert(scores[i], tempScore);
+ boots[i] += tempScore;
+ }
+
+ //merge abunds
+ mergeString += (itMap->second)[j] + " ";
+ for (int i = 0; i < lookup.size(); i++) {
+ abunds[i] += lookup[i]->getAbundance(labelIndex[m->getSimpleLabel((itMap->second)[j])]);
+ }
+ }
+
+ if (m->debug) { m->mothurOut("[DEBUG]: merging " + mergeString + " for ggOTUid = " + itMap->first + ".\n"); }
+
+ //average scores
+ //add merged otu to new lookup
+ for (int j = 0; j < boots.size(); j++) { boots[j] /= (float) itMap->second.size(); }
+
+ //assemble new taxomoy
+ string newTaxString = "";
+ for (int j = 0; j < boots.size(); j++) {
+ newTaxString += taxonomies[j] + "(" + toString(boots[j]) + ");";
+ }
+
+ //set new gg otu id to taxonomy. OTU01 -> k__Bacteria becomes 16097 -> k__Bacteria
+ //find taxonomy of this otu
+ newLabelTaxMap[itMap->first] = newTaxString;
+
+ //add merged otu to new lookup
+ for (int j = 0; j < abunds.size(); j++) { newLookup[j]->push_back(abunds[j], newLookup[j]->getGroup()); }
+
+ //saved otu label
+ newBinLabels.push_back(itMap->first);
+ }
+
+ for (int j = 0; j < lookup.size(); j++) { delete lookup[j]; }
+
+ lookup = newLookup;
+ m->currentSharedBinLabels = newBinLabels;
+ labelTaxMap = newLabelTaxMap;
+
+ map<string, string> variables;
+ variables["[filename]"] = outputDir + m->getRootName(m->getSimpleName(sharedfile));
+ variables["[distance]"] = lookup[0]->getLabel();
+ string outputFileName = getOutputFileName("shared",variables);
+ ofstream out;
+ m->openOutputFile(outputFileName, out);
+ outputNames.push_back(outputFileName); outputTypes["shared"].push_back(outputFileName);
+
+ lookup[0]->printHeaders(out);
+
+ for (int i = 0; i < lookup.size(); i++) {
+ out << lookup[i]->getLabel() << '\t' << lookup[i]->getGroup() << '\t';
+ lookup[i]->print(out);
+ }
+ out.close();
+
+ return 0;
+ }
+ catch(exception& e) {
+ m->errorOut(e, "MakeBiomCommand", "getGreenGenesOTUIDs");
+ exit(1);
+ }
+
+}
+//**********************************************************************************************************************
+map<string, string> MakeBiomCommand::readGGOtuMap(){
+ try {
+ map<string, string> otuMap;
+
+ ifstream in;
+ m->openInputFile(picrustOtuFile, in);
+
+ //map referenceIDs -> otuIDs
+ //lines look like:
+ //16097 671376 616121 533566 683683 4332909 4434717 772666 611808 695209
+ while(!in.eof()) {
+ if (m->control_pressed) { break; }
+
+ string line = m->getline(in); m->gobble(in);
+ vector<string> pieces = m->splitWhiteSpace(line);
+
+ if (pieces.size() != 0) {
+ string otuID = pieces[0];
+ for (int i = 1; i < pieces.size(); i++) { otuMap[pieces[i]] = otuID; }
+ }
+ }
+ in.close();
+
+ return otuMap;
+ }
+ catch(exception& e) {
+ m->errorOut(e, "MakeBiomCommand", "readGGOtuMap");
+ exit(1);
+ }
+
}
//**********************************************************************************************************************
int MakeBiomCommand::getSampleMetaData(vector<SharedRAbundVector*>& lookup){
projects / mothur.git / commitdiff