5 // Created by Sarah Westcott on 4/16/12.
6 // Copyright (c) 2012 Schloss Lab. All rights reserved.
9 #include "makebiomcommand.h"
10 #include "sharedrabundvector.h"
11 #include "inputdata.h"
13 //taken from http://biom-format.org/documentation/biom_format.html
17 "format": "Biological Observation Matrix 0.9.1",
18 "format_url": "http://biom-format.org",
20 "generated_by": "QIIME revision 1.4.0-dev",
21 "date": "2011-12-19T19:00:00",
23 {"id":"GG_OTU_1", "metadata":null},
24 {"id":"GG_OTU_2", "metadata":null},
25 {"id":"GG_OTU_3", "metadata":null},
26 {"id":"GG_OTU_4", "metadata":null},
27 {"id":"GG_OTU_5", "metadata":null}
30 {"id":"Sample1", "metadata":null},
31 {"id":"Sample2", "metadata":null},
32 {"id":"Sample3", "metadata":null},
33 {"id":"Sample4", "metadata":null},
34 {"id":"Sample5", "metadata":null},
35 {"id":"Sample6", "metadata":null}
37 "matrix_type": "sparse",
38 "matrix_element_type": "int",
61 "format": "Biological Observation Matrix 0.9.1",
62 "format_url": "http://biom-format.org",
64 "generated_by": "QIIME revision 1.4.0-dev",
65 "date": "2011-12-19T19:00:00",
67 {"id":"GG_OTU_1", "metadata":null},
68 {"id":"GG_OTU_2", "metadata":null},
69 {"id":"GG_OTU_3", "metadata":null},
70 {"id":"GG_OTU_4", "metadata":null},
71 {"id":"GG_OTU_5", "metadata":null}
74 {"id":"Sample1", "metadata":null},
75 {"id":"Sample2", "metadata":null},
76 {"id":"Sample3", "metadata":null},
77 {"id":"Sample4", "metadata":null},
78 {"id":"Sample5", "metadata":null},
79 {"id":"Sample6", "metadata":null}
81 "matrix_type": "dense",
82 "matrix_element_type": "int",
84 "data": [[0,0,1,0,0,0],
91 //**********************************************************************************************************************
92 vector<string> MakeBiomCommand::setParameters(){
94 CommandParameter pshared("shared", "InputTypes", "", "", "none", "none", "none",false,true); parameters.push_back(pshared);
95 CommandParameter pcontaxonomy("contaxonomy", "InputTypes", "", "", "none", "none", "none",false,false); parameters.push_back(pcontaxonomy);
96 CommandParameter pgroups("groups", "String", "", "", "", "", "",false,false); parameters.push_back(pgroups);
97 CommandParameter plabel("label", "String", "", "", "", "", "",false,false); parameters.push_back(plabel);
98 CommandParameter pinputdir("inputdir", "String", "", "", "", "", "",false,false); parameters.push_back(pinputdir);
99 CommandParameter poutputdir("outputdir", "String", "", "", "", "", "",false,false); parameters.push_back(poutputdir);
100 CommandParameter pmatrixtype("matrixtype", "Multiple", "sparse-dense", "sparse", "", "", "",false,false); parameters.push_back(pmatrixtype);
102 vector<string> myArray;
103 for (int i = 0; i < parameters.size(); i++) { myArray.push_back(parameters[i].name); }
106 catch(exception& e) {
107 m->errorOut(e, "MakeBiomCommand", "setParameters");
111 //**********************************************************************************************************************
112 string MakeBiomCommand::getHelpString(){
114 string helpString = "";
115 helpString += "The make.biom command parameters are shared, contaxonomy, groups, matrixtype and label. shared is required, unless you have a valid current file.\n";
116 helpString += "The groups parameter allows you to specify which of the groups in your groupfile you would like included. The group names are separated by dashes.\n";
117 helpString += "The label parameter allows you to select what distance levels you would like, and are also separated by dashes.\n";
118 helpString += "The matrixtype parameter allows you to select what type you would like to make. Choices are sparse and dense, default is sparse.\n";
119 helpString += "The contaxonomy file is the taxonomy file outputted by classify.otu(list=yourListfile, taxonomy=yourTaxonomyFile). Be SURE that the you are the constaxonomy file distance matches the shared file distance. ie, for *.0.03.cons.taxonomy set label=0.03. Mothur is smart enough to handle shared files that have been subsampled.\n";
120 helpString += "The make.biom command should be in the following format: make.biom(shared=yourShared, groups=yourGroups, label=yourLabels).\n";
121 helpString += "Example make.biom(shared=abrecovery.an.shared, groups=A-B-C).\n";
122 helpString += "The default value for groups is all the groups in your groupfile, and all labels in your inputfile will be used.\n";
123 helpString += "The make.biom command outputs a .biom file.\n";
124 helpString += "Note: No spaces between parameter labels (i.e. groups), '=' and parameters (i.e.yourGroups).\n";
127 catch(exception& e) {
128 m->errorOut(e, "MakeBiomCommand", "getHelpString");
132 //**********************************************************************************************************************
133 MakeBiomCommand::MakeBiomCommand(){
135 abort = true; calledHelp = true;
137 vector<string> tempOutNames;
138 outputTypes["biom"] = tempOutNames;
140 catch(exception& e) {
141 m->errorOut(e, "MakeBiomCommand", "MakeBiomCommand");
145 //**********************************************************************************************************************
146 MakeBiomCommand::MakeBiomCommand(string option) {
148 abort = false; calledHelp = false;
151 //allow user to run help
152 if(option == "help") { help(); abort = true; calledHelp = true; }
153 else if(option == "citation") { citation(); abort = true; calledHelp = true;}
156 vector<string> myArray = setParameters();
158 OptionParser parser(option);
159 map<string,string> parameters = parser.getParameters();
160 map<string,string>::iterator it;
162 ValidParameters validParameter;
164 //check to make sure all parameters are valid for command
165 for (it = parameters.begin(); it != parameters.end(); it++) {
166 if (validParameter.isValidParameter(it->first, myArray, it->second) != true) { abort = true; }
169 //initialize outputTypes
170 vector<string> tempOutNames;
171 outputTypes["biom"] = tempOutNames;
173 //if the user changes the input directory command factory will send this info to us in the output parameter
174 string inputDir = validParameter.validFile(parameters, "inputdir", false);
175 if (inputDir == "not found"){ inputDir = ""; }
178 it = parameters.find("shared");
179 //user has given a template file
180 if(it != parameters.end()){
181 path = m->hasPath(it->second);
182 //if the user has not given a path then, add inputdir. else leave path alone.
183 if (path == "") { parameters["shared"] = inputDir + it->second; }
186 it = parameters.find("contaxonomy");
187 //user has given a template file
188 if(it != parameters.end()){
189 path = m->hasPath(it->second);
190 //if the user has not given a path then, add inputdir. else leave path alone.
191 if (path == "") { parameters["contaxonomy"] = inputDir + it->second; }
196 sharedfile = validParameter.validFile(parameters, "shared", true);
197 if (sharedfile == "not open") { sharedfile = ""; abort = true; }
198 else if (sharedfile == "not found") {
199 //if there is a current shared file, use it
200 sharedfile = m->getSharedFile();
201 if (sharedfile != "") { m->mothurOut("Using " + sharedfile + " as input file for the shared parameter."); m->mothurOutEndLine(); }
202 else { m->mothurOut("You have no current sharedfile and the shared parameter is required."); m->mothurOutEndLine(); abort = true; }
203 }else { m->setSharedFile(sharedfile); }
206 //if the user changes the output directory command factory will send this info to us in the output parameter
207 outputDir = validParameter.validFile(parameters, "outputdir", false); if (outputDir == "not found"){ outputDir = m->hasPath(sharedfile); }
209 contaxonomyfile = validParameter.validFile(parameters, "contaxonomy", true);
210 if (contaxonomyfile == "not found") { contaxonomyfile = ""; }
211 else if (contaxonomyfile == "not open") { contaxonomyfile = ""; abort = true; }
214 //check for optional parameter and set defaults
215 // ...at some point should added some additional type checking...
216 label = validParameter.validFile(parameters, "label", false);
217 if (label == "not found") { label = ""; }
219 if(label != "all") { m->splitAtDash(label, labels); allLines = 0; }
220 else { allLines = 1; }
223 groups = validParameter.validFile(parameters, "groups", false);
224 if (groups == "not found") { groups = ""; }
226 m->splitAtDash(groups, Groups);
227 m->setGroups(Groups);
230 if ((contaxonomyfile != "") && (labels.size() > 1)) { m->mothurOut("[ERROR]: the contaxonomy parameter cannot be used with multiple labels."); m->mothurOutEndLine(); abort = true; }
232 format = validParameter.validFile(parameters, "matrixtype", false); if (format == "not found") { format = "sparse"; }
234 if ((format != "sparse") && (format != "dense")) {
235 m->mothurOut(format + " is not a valid option for the matrixtype parameter. Options are sparse and dense."); m->mothurOutEndLine(); abort = true;
240 catch(exception& e) {
241 m->errorOut(e, "MakeBiomCommand", "MakeBiomCommand");
245 //**********************************************************************************************************************
247 int MakeBiomCommand::execute(){
250 if (abort == true) { if (calledHelp) { return 0; } return 2; }
252 InputData input(sharedfile, "sharedfile");
253 vector<SharedRAbundVector*> lookup = input.getSharedRAbundVectors();
254 string lastLabel = lookup[0]->getLabel();
256 //if user did not specify a label, then use first one
257 if ((contaxonomyfile != "") && (labels.size() == 0)) {
259 labels.insert(lastLabel);
262 //if the users enters label "0.06" and there is no "0.06" in their file use the next lowest label.
263 set<string> processedLabels;
264 set<string> userLabels = labels;
266 //as long as you are not at the end of the file or done wih the lines you want
267 while((lookup[0] != NULL) && ((allLines == 1) || (userLabels.size() != 0))) {
269 if (m->control_pressed) { for (int i = 0; i < outputNames.size(); i++) { m->mothurRemove(outputNames[i]); } for (int i = 0; i < lookup.size(); i++) { delete lookup[i]; } return 0; }
271 if(allLines == 1 || labels.count(lookup[0]->getLabel()) == 1){
273 m->mothurOut(lookup[0]->getLabel()); m->mothurOutEndLine();
276 processedLabels.insert(lookup[0]->getLabel());
277 userLabels.erase(lookup[0]->getLabel());
280 if ((m->anyLabelsToProcess(lookup[0]->getLabel(), userLabels, "") == true) && (processedLabels.count(lastLabel) != 1)) {
281 string saveLabel = lookup[0]->getLabel();
283 for (int i = 0; i < lookup.size(); i++) { delete lookup[i]; }
284 lookup = input.getSharedRAbundVectors(lastLabel);
285 m->mothurOut(lookup[0]->getLabel()); m->mothurOutEndLine();
289 processedLabels.insert(lookup[0]->getLabel());
290 userLabels.erase(lookup[0]->getLabel());
292 //restore real lastlabel to save below
293 lookup[0]->setLabel(saveLabel);
296 lastLabel = lookup[0]->getLabel();
298 //prevent memory leak and get next set
299 for (int i = 0; i < lookup.size(); i++) { delete lookup[i]; lookup[i] = NULL; }
300 lookup = input.getSharedRAbundVectors();
303 if (m->control_pressed) { for (int i = 0; i < outputNames.size(); i++) { m->mothurRemove(outputNames[i]); } return 0; }
305 //output error messages about any remaining user labels
306 set<string>::iterator it;
307 bool needToRun = false;
308 for (it = userLabels.begin(); it != userLabels.end(); it++) {
309 m->mothurOut("Your file does not include the label " + *it);
310 if (processedLabels.count(lastLabel) != 1) {
311 m->mothurOut(". I will use " + lastLabel + "."); m->mothurOutEndLine();
314 m->mothurOut(". Please refer to " + lastLabel + "."); m->mothurOutEndLine();
318 //run last label if you need to
319 if (needToRun == true) {
320 for (int i = 0; i < lookup.size(); i++) { if (lookup[i] != NULL) { delete lookup[i]; } }
321 lookup = input.getSharedRAbundVectors(lastLabel);
323 m->mothurOut(lookup[0]->getLabel()); m->mothurOutEndLine();
326 for (int i = 0; i < lookup.size(); i++) { delete lookup[i]; }
329 if (m->control_pressed) { for (int i = 0; i < outputNames.size(); i++) { m->mothurRemove(outputNames[i]); } return 0; }
331 //set sabund file as new current sabundfile
333 itTypes = outputTypes.find("biom");
334 if (itTypes != outputTypes.end()) {
335 if ((itTypes->second).size() != 0) { current = (itTypes->second)[0]; m->setBiomFile(current); }
339 m->mothurOutEndLine();
340 m->mothurOut("Output File Names: "); m->mothurOutEndLine();
341 for (int i = 0; i < outputNames.size(); i++) { m->mothurOut(outputNames[i]); m->mothurOutEndLine(); }
342 m->mothurOutEndLine();
346 catch(exception& e) {
347 m->errorOut(e, "MakeBiomCommand", "execute");
351 //**********************************************************************************************************************
352 int MakeBiomCommand::getBiom(vector<SharedRAbundVector*>& lookup){
355 string outputFileName = outputDir + m->getRootName(m->getSimpleName(sharedfile)) + lookup[0]->getLabel() + ".biom";
357 m->openOutputFile(outputFileName, out);
358 outputNames.push_back(outputFileName); outputTypes["biom"].push_back(outputFileName);
360 string mothurString = "mothur" + toString(m->getVersion());
362 struct tm * timeinfo;
364 timeinfo = localtime ( &rawtime );
365 string dateString = asctime (timeinfo);
366 int pos = dateString.find('\n');
367 if (pos != string::npos) { dateString = dateString.substr(0, pos);}
371 out << "{\n" + spaces + "\"id\":\"" + sharedfile + "-" + lookup[0]->getLabel() + "\",\n" + spaces + "\"format\": \"Biological Observation Matrix 0.9.1\",\n" + spaces + "\"format_url\": \"http://biom-format.org\",\n";
372 out << spaces + "\"type\": \"OTU table\",\n" + spaces + "\"generated_by\": \"" << mothurString << "\",\n" + spaces + "\"date\": \"" << dateString << "\",\n";
374 vector<string> metadata = getMetaData(lookup);
376 if (m->control_pressed) { out.close(); return 0; }
380 {"id":"GG_OTU_1", "metadata":null},
381 {"id":"GG_OTU_2", "metadata":null},
382 {"id":"GG_OTU_3", "metadata":null},
383 {"id":"GG_OTU_4", "metadata":null},
384 {"id":"GG_OTU_5", "metadata":null}
386 out << spaces + "\"rows\":[\n";
387 string rowFront = spaces + spaces + "{\"id\":\"";
388 string rowBack = "\", \"metadata\":";
389 for (int i = 0; i < m->currentBinLabels.size()-1; i++) {
390 if (m->control_pressed) { out.close(); return 0; }
391 out << rowFront << m->currentBinLabels[i] << rowBack << metadata[i] << "},\n";
393 out << rowFront << m->currentBinLabels[(m->currentBinLabels.size()-1)] << rowBack << metadata[(m->currentBinLabels.size()-1)] << "}\n" + spaces + "],\n";
397 {"id":"Sample1", "metadata":null},
398 {"id":"Sample2", "metadata":null},
399 {"id":"Sample3", "metadata":null},
400 {"id":"Sample4", "metadata":null},
401 {"id":"Sample5", "metadata":null},
402 {"id":"Sample6", "metadata":null}
405 string colBack = "\", \"metadata\":null}";
406 out << spaces + "\"columns\":[\n";
407 for (int i = 0; i < lookup.size()-1; i++) {
408 if (m->control_pressed) { out.close(); return 0; }
409 out << rowFront << lookup[i]->getGroup() << colBack << ",\n";
411 out << rowFront << lookup[(lookup.size()-1)]->getGroup() << colBack << "\n" + spaces + "],\n";
413 out << spaces + "\"matrix_type\": \"" << format << "\",\n" + spaces + "\"matrix_element_type\": \"int\",\n";
414 out << spaces + "\"shape\": [" << m->currentBinLabels.size() << "," << lookup.size() << "],\n";
415 out << spaces + "\"data\": [";
417 vector<string> dataRows;
418 if (format == "sparse") {
436 for (int i = 0; i < lookup[0]->getNumBins(); i++) {
438 if (m->control_pressed) { out.close(); return 0; }
440 for (int j = 0; j < lookup.size(); j++) {
441 string binInfo = "[" + toString(i) + "," + toString(j) + "," + toString(lookup[j]->getAbundance(i)) + "]";
442 //only print non zero values
443 if (lookup[j]->getAbundance(i) != 0) { dataRows.push_back(binInfo); }
448 /* "matrix_type": "dense",
449 "matrix_element_type": "int",
451 "data": [[0,0,1,0,0,0],
457 for (int i = 0; i < lookup[0]->getNumBins(); i++) {
459 if (m->control_pressed) { out.close(); return 0; }
461 string binInfo = "[";
462 for (int j = 0; j < lookup.size()-1; j++) {
463 binInfo += toString(lookup[j]->getAbundance(i)) + ",";
465 binInfo += toString(lookup[lookup.size()-1]->getAbundance(i)) + "]";
466 dataRows.push_back(binInfo);
470 for (int i = 0; i < dataRows.size()-1; i++) {
471 out << dataRows[i] << ",\n" + spaces + spaces;
473 out << dataRows[dataRows.size()-1] << "]\n";
480 catch(exception& e) {
481 m->errorOut(e, "MakeBiomCommand", "getBiom");
485 //**********************************************************************************************************************
486 vector<string> MakeBiomCommand::getMetaData(vector<SharedRAbundVector*>& lookup){
488 vector<string> metadata;
490 if (contaxonomyfile == "") { for (int i = 0; i < lookup[0]->getNumBins(); i++) { metadata.push_back("null"); } }
493 //read constaxonomy file storing in a map, otulabel -> taxonomy
494 //constaxonomy file will most likely contain more labels than the shared file, because sharedfile could have been subsampled.
496 m->openInputFile(contaxonomyfile, in);
499 m->getline(in); m->gobble(in);
501 string otuLabel, tax;
503 vector<string> otuLabels;
507 if (m->control_pressed) { in.close(); return metadata; }
509 in >> otuLabel >> size >> tax; m->gobble(in);
511 otuLabels.push_back(otuLabel);
516 //should the labels be Otu001 or PhyloType001
517 string firstBin = m->currentBinLabels[0];
518 string binTag = "Otu";
519 if ((firstBin.find("Otu")) == string::npos) { binTag = "PhyloType"; }
521 //convert list file bin labels to shared file bin labels
524 map<string, string> labelTaxMap;
525 string snumBins = toString(otuLabels.size());
526 for (int i = 0; i < otuLabels.size(); i++) {
528 if (m->control_pressed) { return metadata; }
530 //if there is a bin label use it otherwise make one
531 string binLabel = binTag;
532 string sbinNumber = otuLabels[i];
533 if (sbinNumber.length() < snumBins.length()) {
534 int diff = snumBins.length() - sbinNumber.length();
535 for (int h = 0; h < diff; h++) { binLabel += "0"; }
537 binLabel += sbinNumber;
539 labelTaxMap[binLabel] = taxs[i];
543 //{"taxonomy":["k__Bacteria", "p__Proteobacteria", "c__Gammaproteobacteria", "o__Enterobacteriales", "f__Enterobacteriaceae", "g__Escherichia", "s__"]}
545 //traverse the binLabels forming the metadata strings and saving them
546 //make sure to sanity check
547 map<string, string>::iterator it;
548 for (int i = 0; i < m->currentBinLabels.size(); i++) {
550 if (m->control_pressed) { return metadata; }
552 it = labelTaxMap.find(m->currentBinLabels[i]);
554 if (it == labelTaxMap.end()) { m->mothurOut("[ERROR]: can't find taxonomy information for " + m->currentBinLabels[i] + ".\n"); m->control_pressed = true; }
556 vector<string> bootstrapValues;
557 string data = "{\"taxonomy\":[";
559 vector<string> scores;
560 vector<string> taxonomies = parseTax(it->second, scores);
562 for (int j = 0; j < taxonomies.size()-1; j ++) { data += "\"" + taxonomies[j] + "\", "; }
563 data += "\"" + taxonomies[taxonomies.size()-1] + "\"]";
565 //add bootstrap values if available
566 if (scores[0] != "null") {
567 data += ", \"bootstrap\":[";
569 for (int j = 0; j < scores.size()-1; j ++) { data += scores[j] + ", "; }
570 data += scores[scores.size()-1] + "]";
575 metadata.push_back(data);
583 catch(exception& e) {
584 m->errorOut(e, "MakeBiomCommand", "getMetadata");
589 /**************************************************************************************************/
590 //returns {Bacteria, Bacteroidetes, ..} and scores is filled with {100, 98, ...} or {null, null, null}
591 vector<string> MakeBiomCommand::parseTax(string tax, vector<string>& scores) {
597 while (tax.find_first_of(';') != -1) {
599 if (m->control_pressed) { return taxs; }
602 taxon = tax.substr(0,tax.find_first_of(';'));
604 int pos = taxon.find_last_of('(');
607 int pos2 = taxon.find_last_of(')');
609 string confidenceScore = taxon.substr(pos+1, (pos2-(pos+1)));
610 if (m->isNumeric1(confidenceScore)) {
611 taxon = taxon.substr(0, pos); //rip off confidence
612 scores.push_back(confidenceScore);
613 }else{ scores.push_back("null"); }
617 //strip "" if they are there
618 pos = taxon.find("\"");
619 if (pos != string::npos) {
621 for (int k = 0; k < taxon.length(); k++) {
622 if (taxon[k] != '\"') { newTax += taxon[k]; }
627 //look for bootstrap value
628 taxs.push_back(taxon);
629 tax = tax.substr(tax.find_first_of(';')+1, tax.length());
634 catch(exception& e) {
635 m->errorOut(e, "MakeBiomCommand", "parseTax");
640 //**********************************************************************************************************************