5 * Created by westcott on 10/27/10.
6 * Copyright 2010 Schloss Lab. All rights reserved.
10 #include "subsamplecommand.h"
11 #include "sharedutilities.h"
13 //**********************************************************************************************************************
14 vector<string> SubSampleCommand::getValidParameters(){
16 string Array[] = {"fasta", "group", "list","shared","rabund", "name","sabund","size","groups","label","outputdir","inputdir"};
17 vector<string> myArray (Array, Array+(sizeof(Array)/sizeof(string)));
21 m->errorOut(e, "SubSampleCommand", "getValidParameters");
25 //**********************************************************************************************************************
26 SubSampleCommand::SubSampleCommand(){
29 //initialize outputTypes
30 vector<string> tempOutNames;
31 outputTypes["shared"] = tempOutNames;
32 outputTypes["list"] = tempOutNames;
33 outputTypes["rabund"] = tempOutNames;
34 outputTypes["sabund"] = tempOutNames;
35 outputTypes["fasta"] = tempOutNames;
36 outputTypes["name"] = tempOutNames;
37 outputTypes["group"] = tempOutNames;
40 m->errorOut(e, "SubSampleCommand", "GetRelAbundCommand");
44 //**********************************************************************************************************************
45 vector<string> SubSampleCommand::getRequiredParameters(){
47 string Array[] = {"fasta","list","shared","rabund", "sabund","or"};
48 vector<string> myArray (Array, Array+(sizeof(Array)/sizeof(string)));
52 m->errorOut(e, "SubSampleCommand", "getRequiredParameters");
56 //**********************************************************************************************************************
57 vector<string> SubSampleCommand::getRequiredFiles(){
59 vector<string> myArray;
63 m->errorOut(e, "SubSampleCommand", "getRequiredFiles");
67 //**********************************************************************************************************************
68 SubSampleCommand::SubSampleCommand(string option) {
70 globaldata = GlobalData::getInstance();
75 //allow user to run help
76 if(option == "help") { help(); abort = true; }
79 //valid paramters for this command
80 string Array[] = {"fasta", "group", "list","shared","rabund", "sabund","name","size","groups","label","outputdir","inputdir"};
81 vector<string> myArray (Array, Array+(sizeof(Array)/sizeof(string)));
83 OptionParser parser(option);
84 map<string,string> parameters = parser.getParameters();
86 ValidParameters validParameter;
88 //check to make sure all parameters are valid for command
89 map<string,string>::iterator it;
90 for (it = parameters.begin(); it != parameters.end(); it++) {
91 if (validParameter.isValidParameter(it->first, myArray, it->second) != true) { abort = true; }
94 //initialize outputTypes
95 vector<string> tempOutNames;
96 outputTypes["shared"] = tempOutNames;
97 outputTypes["list"] = tempOutNames;
98 outputTypes["rabund"] = tempOutNames;
99 outputTypes["sabund"] = tempOutNames;
100 outputTypes["fasta"] = tempOutNames;
101 outputTypes["name"] = tempOutNames;
102 outputTypes["group"] = tempOutNames;
104 //if the user changes the output directory command factory will send this info to us in the output parameter
105 outputDir = validParameter.validFile(parameters, "outputdir", false); if (outputDir == "not found"){ outputDir = ""; }
107 //if the user changes the input directory command factory will send this info to us in the output parameter
108 string inputDir = validParameter.validFile(parameters, "inputdir", false);
109 if (inputDir == "not found"){ inputDir = ""; }
112 it = parameters.find("list");
113 //user has given a template file
114 if(it != parameters.end()){
115 path = m->hasPath(it->second);
116 //if the user has not given a path then, add inputdir. else leave path alone.
117 if (path == "") { parameters["list"] = inputDir + it->second; }
120 it = parameters.find("fasta");
121 //user has given a template file
122 if(it != parameters.end()){
123 path = m->hasPath(it->second);
124 //if the user has not given a path then, add inputdir. else leave path alone.
125 if (path == "") { parameters["fasta"] = inputDir + it->second; }
128 it = parameters.find("shared");
129 //user has given a template file
130 if(it != parameters.end()){
131 path = m->hasPath(it->second);
132 //if the user has not given a path then, add inputdir. else leave path alone.
133 if (path == "") { parameters["shared"] = inputDir + it->second; }
136 it = parameters.find("group");
137 //user has given a template file
138 if(it != parameters.end()){
139 path = m->hasPath(it->second);
140 //if the user has not given a path then, add inputdir. else leave path alone.
141 if (path == "") { parameters["group"] = inputDir + it->second; }
144 it = parameters.find("sabund");
145 //user has given a template file
146 if(it != parameters.end()){
147 path = m->hasPath(it->second);
148 //if the user has not given a path then, add inputdir. else leave path alone.
149 if (path == "") { parameters["sabund"] = inputDir + it->second; }
152 it = parameters.find("rabund");
153 //user has given a template file
154 if(it != parameters.end()){
155 path = m->hasPath(it->second);
156 //if the user has not given a path then, add inputdir. else leave path alone.
157 if (path == "") { parameters["rabund"] = inputDir + it->second; }
160 it = parameters.find("name");
161 //user has given a template file
162 if(it != parameters.end()){
163 path = m->hasPath(it->second);
164 //if the user has not given a path then, add inputdir. else leave path alone.
165 if (path == "") { parameters["name"] = inputDir + it->second; }
169 //check for required parameters
170 listfile = validParameter.validFile(parameters, "list", true);
171 if (listfile == "not open") { listfile = ""; abort = true; }
172 else if (listfile == "not found") { listfile = ""; }
174 sabundfile = validParameter.validFile(parameters, "sabund", true);
175 if (sabundfile == "not open") { sabundfile = ""; abort = true; }
176 else if (sabundfile == "not found") { sabundfile = ""; }
178 rabundfile = validParameter.validFile(parameters, "rabund", true);
179 if (rabundfile == "not open") { rabundfile = ""; abort = true; }
180 else if (rabundfile == "not found") { rabundfile = ""; }
182 fastafile = validParameter.validFile(parameters, "fasta", true);
183 if (fastafile == "not open") { fastafile = ""; abort = true; }
184 else if (fastafile == "not found") { fastafile = ""; }
186 sharedfile = validParameter.validFile(parameters, "shared", true);
187 if (sharedfile == "not open") { sharedfile = ""; abort = true; }
188 else if (sharedfile == "not found") { sharedfile = ""; }
190 namefile = validParameter.validFile(parameters, "name", true);
191 if (namefile == "not open") { namefile = ""; abort = true; }
192 else if (namefile == "not found") { namefile = ""; }
194 groupfile = validParameter.validFile(parameters, "group", true);
195 if (groupfile == "not open") { groupfile = ""; abort = true; }
196 else if (groupfile == "not found") { groupfile = ""; }
199 //check for optional parameter and set defaults
200 // ...at some point should added some additional type checking...
201 label = validParameter.validFile(parameters, "label", false);
202 if (label == "not found") { label = ""; }
204 if(label != "all") { m->splitAtDash(label, labels); allLines = 0; }
205 else { allLines = 1; }
208 groups = validParameter.validFile(parameters, "groups", false);
209 if (groups == "not found") { groups = ""; pickedGroups = false; }
212 m->splitAtDash(groups, Groups);
213 globaldata->Groups = Groups;
216 string temp = validParameter.validFile(parameters, "size", false); if (temp == "not found"){ temp = "0"; }
219 if ((namefile != "") && (fastafile == "")) { m->mothurOut("You may only use a namefile with a fastafile."); m->mothurOutEndLine(); abort = true; }
221 if ((fastafile == "") && (listfile == "") && (sabundfile == "") && (rabundfile == "") && (sharedfile == "")) {
222 m->mothurOut("You must provide a fasta, list, sabund, rabund or shared file as an input file."); m->mothurOutEndLine(); abort = true; }
224 if (pickedGroups && ((groupfile == "") && (sharedfile == ""))) {
225 m->mothurOut("You cannot pick groups without a valid group file or shared file."); m->mothurOutEndLine(); abort = true; }
227 if ((groupfile != "") && ((fastafile == "") && (listfile == ""))) {
228 m->mothurOut("Group file only valid with listfile or fastafile."); m->mothurOutEndLine(); abort = true; }
230 if ((groupfile != "") && ((fastafile != "") && (listfile != ""))) {
231 m->mothurOut("A new group file can only be made from the subsample of a listfile or fastafile, not both. Please correct."); m->mothurOutEndLine(); abort = true; }
236 catch(exception& e) {
237 m->errorOut(e, "SubSampleCommand", "SubSampleCommand");
242 //**********************************************************************************************************************
244 void SubSampleCommand::help(){
246 m->mothurOut("The sub.sample command is designed to be used as a way to normalize your data, or create a smaller set from your original set.\n");
247 m->mothurOut("The sub.sample command parameters are fasta, name, list, group, rabund, sabund, shared, groups, size and label. You must provide a fasta, list, sabund, rabund or shared file as an input file.\n");
248 m->mothurOut("The groups parameter allows you to specify which of the groups in your groupfile you would like included. The group names are separated by dashes.\n");
249 m->mothurOut("The label parameter allows you to select what distance levels you would like, and are also separated by dashes.\n");
250 m->mothurOut("The size parameter allows you indicate the size of your subsample.\n");
251 m->mothurOut("The size parameter is not set: with shared file size=number of seqs in smallest sample, with all other files, 10% of number of seqs.\n");
252 m->mothurOut("The sub.sample command should be in the following format: sub.sample(list=yourListFile, group=yourGroupFile, groups=yourGroups, label=yourLabels).\n");
253 m->mothurOut("Example sub.sample(list=abrecovery.fn.list, group=abrecovery.groups, groups=B-C, size=20).\n");
254 m->mothurOut("The default value for groups is all the groups in your groupfile, and all labels in your inputfile will be used.\n");
255 m->mothurOut("The sub.sample command outputs a .subsample file.\n");
256 m->mothurOut("Note: No spaces between parameter labels (i.e. groups), '=' and parameters (i.e.yourGroups).\n\n");
259 catch(exception& e) {
260 m->errorOut(e, "SubSampleCommand", "help");
265 //**********************************************************************************************************************
267 SubSampleCommand::~SubSampleCommand(){}
269 //**********************************************************************************************************************
271 int SubSampleCommand::execute(){
274 if (abort == true) { return 0; }
276 if (sharedfile != "") { getSubSampleShared(); }
277 if (m->control_pressed) { for (int i = 0; i < outputNames.size(); i++) { remove(outputNames[i].c_str()); return 0; } }
278 if (listfile != "") { getSubSampleList(); }
279 if (m->control_pressed) { for (int i = 0; i < outputNames.size(); i++) { remove(outputNames[i].c_str()); return 0; } }
280 //if (rabund != "") { getSubSampleRabund(); }
281 if (m->control_pressed) { for (int i = 0; i < outputNames.size(); i++) { remove(outputNames[i].c_str()); return 0; } }
282 //if (sabundfile != "") { getSubSampleSabund(); }
283 if (m->control_pressed) { for (int i = 0; i < outputNames.size(); i++) { remove(outputNames[i].c_str()); return 0; } }
284 //if (fastafile != "") { getSubSampleFasta(); }
285 if (m->control_pressed) { for (int i = 0; i < outputNames.size(); i++) { remove(outputNames[i].c_str()); return 0; } }
288 m->mothurOutEndLine();
289 m->mothurOut("Output File Names: "); m->mothurOutEndLine();
290 for (int i = 0; i < outputNames.size(); i++) { m->mothurOut(outputNames[i]); m->mothurOutEndLine(); }
291 m->mothurOutEndLine();
295 catch(exception& e) {
296 m->errorOut(e, "SubSampleCommand", "execute");
300 //**********************************************************************************************************************
301 int SubSampleCommand::getSubSampleShared() {
304 string thisOutputDir = outputDir;
305 if (outputDir == "") { thisOutputDir += m->hasPath(sharedfile); }
306 string outputFileName = thisOutputDir + m->getRootName(m->getSimpleName(sharedfile)) + "subsample" + m->getExtension(sharedfile);
309 m->openOutputFile(outputFileName, out);
310 outputTypes["shared"].push_back(outputFileName); outputNames.push_back(outputFileName);
312 InputData* input = new InputData(sharedfile, "sharedfile");
313 vector<SharedRAbundVector*> lookup = input->getSharedRAbundVectors();
314 string lastLabel = lookup[0]->getLabel();
316 //if the users enters label "0.06" and there is no "0.06" in their file use the next lowest label.
317 set<string> processedLabels;
318 set<string> userLabels = labels;
321 //as long as you are not at the end of the file or done wih the lines you want
322 while((lookup[0] != NULL) && ((allLines == 1) || (userLabels.size() != 0))) {
323 if (m->control_pressed) { delete input; out.close(); return 0; }
325 if(allLines == 1 || labels.count(lookup[0]->getLabel()) == 1){
327 m->mothurOut(lookup[0]->getLabel()); m->mothurOutEndLine();
329 processShared(lookup, out);
331 processedLabels.insert(lookup[0]->getLabel());
332 userLabels.erase(lookup[0]->getLabel());
335 if ((m->anyLabelsToProcess(lookup[0]->getLabel(), userLabels, "") == true) && (processedLabels.count(lastLabel) != 1)) {
336 string saveLabel = lookup[0]->getLabel();
338 for (int i = 0; i < lookup.size(); i++) { delete lookup[i]; }
340 lookup = input->getSharedRAbundVectors(lastLabel);
341 m->mothurOut(lookup[0]->getLabel()); m->mothurOutEndLine();
343 processShared(lookup, out);
345 processedLabels.insert(lookup[0]->getLabel());
346 userLabels.erase(lookup[0]->getLabel());
348 //restore real lastlabel to save below
349 lookup[0]->setLabel(saveLabel);
352 lastLabel = lookup[0]->getLabel();
353 //prevent memory leak
354 for (int i = 0; i < lookup.size(); i++) { delete lookup[i]; lookup[i] = NULL; }
356 //get next line to process
357 lookup = input->getSharedRAbundVectors();
361 if (m->control_pressed) { out.close(); return 0; }
363 //output error messages about any remaining user labels
364 set<string>::iterator it;
365 bool needToRun = false;
366 for (it = userLabels.begin(); it != userLabels.end(); it++) {
367 m->mothurOut("Your file does not include the label " + *it);
368 if (processedLabels.count(lastLabel) != 1) {
369 m->mothurOut(". I will use " + lastLabel + "."); m->mothurOutEndLine();
372 m->mothurOut(". Please refer to " + lastLabel + "."); m->mothurOutEndLine();
376 //run last label if you need to
377 if (needToRun == true) {
378 for (int i = 0; i < lookup.size(); i++) { if (lookup[i] != NULL) { delete lookup[i]; } }
379 lookup = input->getSharedRAbundVectors(lastLabel);
381 m->mothurOut(lookup[0]->getLabel()); m->mothurOutEndLine();
383 processShared(lookup, out);
385 for (int i = 0; i < lookup.size(); i++) { delete lookup[i]; }
394 catch(exception& e) {
395 m->errorOut(e, "SubSampleCommand", "getSubSampleShared");
399 //**********************************************************************************************************************
400 int SubSampleCommand::processShared(vector<SharedRAbundVector*>& thislookup, ofstream& out) {
403 //if (pickedGroups) { eliminateZeroOTUS(thislookup); }
405 if (size == 0) { //user has not set size, set size = smallest samples size
406 size = thislookup[0]->getNumSeqs();
407 for (int i = 1; i < thislookup.size(); i++) {
408 int thisSize = thislookup[i]->getNumSeqs();
410 if (thisSize < size) { size = thisSize; }
414 int numBins = thislookup[0]->getNumBins();
415 for (int i = 0; i < thislookup.size(); i++) {
416 int thisSize = thislookup[i]->getNumSeqs();
418 if (thisSize != size) {
420 string thisgroup = thislookup[i]->getGroup();
422 OrderVector* order = new OrderVector();
423 for(int p=0;p<numBins;p++){
424 for(int j=0;j<thislookup[i]->getAbundance(p);j++){
428 random_shuffle(order->begin(), order->end());
430 SharedRAbundVector* temp = new SharedRAbundVector(numBins);
431 temp->setLabel(thislookup[i]->getLabel());
432 temp->setGroup(thislookup[i]->getGroup());
434 delete thislookup[i];
435 thislookup[i] = temp;
438 for (int j = 0; j < size; j++) {
440 if (m->control_pressed) { delete order; return 0; }
442 //get random number to sample from order between 0 and thisSize-1.
443 int myrand = (int)((float)(rand()) / (RAND_MAX / (thisSize-1) + 1));
445 int bin = order->get(myrand);
447 int abund = thislookup[i]->getAbundance(bin);
448 thislookup[i]->set(bin, (abund+1), thisgroup);
454 //subsampling may have created some otus with no sequences in them
455 eliminateZeroOTUS(thislookup);
457 if (m->control_pressed) { return 0; }
459 for (int i = 0; i < thislookup.size(); i++) {
460 out << thislookup[i]->getLabel() << '\t' << thislookup[i]->getGroup() << '\t';
461 thislookup[i]->print(out);
467 catch(exception& e) {
468 m->errorOut(e, "SubSampleCommand", "processShared");
472 //**********************************************************************************************************************
473 int SubSampleCommand::getSubSampleList() {
476 string thisOutputDir = outputDir;
477 if (outputDir == "") { thisOutputDir += m->hasPath(listfile); }
478 string outputFileName = thisOutputDir + m->getRootName(m->getSimpleName(listfile)) + "subsample" + m->getExtension(listfile);
481 m->openOutputFile(outputFileName, out);
482 outputTypes["list"].push_back(outputFileName); outputNames.push_back(outputFileName);
485 //if you have a groupfile you want to read a shared list
486 if (groupfile != "") {
488 GroupMap* groupMap = new GroupMap(groupfile);
491 //takes care of user setting groupNames that are invalid or setting groups=all
492 SharedUtil* util = new SharedUtil();
493 util->setGroups(Groups, groupMap->namesOfGroups);
497 string groupOutputDir = outputDir;
498 if (outputDir == "") { groupOutputDir += m->hasPath(groupfile); }
499 string groupOutputFileName = groupOutputDir + m->getRootName(m->getSimpleName(groupfile)) + "subsample" + m->getExtension(groupfile);
502 m->openOutputFile(groupOutputFileName, outGroup);
503 outputTypes["group"].push_back(groupOutputFileName); outputNames.push_back(groupOutputFileName);
505 globaldata->setGroupFile(groupfile); //shared list needs this
507 input = new InputData(listfile, "list");
508 ListVector* list = input->getListVector();
509 string lastLabel = list->getLabel();
511 //if the users enters label "0.06" and there is no "0.06" in their file use the next lowest label.
512 set<string> processedLabels;
513 set<string> userLabels = labels;
516 if (list->getNumSeqs() != groupMap->getNumSeqs()) {
517 m->mothurOut("[ERROR]: your list file contains " + toString(list->getNumSeqs()) + " sequences, and your groupfile contains " + toString(groupMap->getNumSeqs()) + ", please correct.");
518 m->mothurOutEndLine();
527 //as long as you are not at the end of the file or done wih the lines you want
528 while((list != NULL) && ((allLines == 1) || (userLabels.size() != 0))) {
530 if (m->control_pressed) { delete list; delete input; delete groupMap; out.close(); outGroup.close(); return 0; }
532 if(allLines == 1 || labels.count(list->getLabel()) == 1){
534 m->mothurOut(list->getLabel()); m->mothurOutEndLine();
536 processListGroup(list, groupMap, out, outGroup);
538 processedLabels.insert(list->getLabel());
539 userLabels.erase(list->getLabel());
542 if ((m->anyLabelsToProcess(list->getLabel(), userLabels, "") == true) && (processedLabels.count(lastLabel) != 1)) {
543 string saveLabel = list->getLabel();
547 list = input->getListVector(lastLabel);
548 m->mothurOut(list->getLabel()); m->mothurOutEndLine();
550 processListGroup(list, groupMap, out, outGroup);
552 processedLabels.insert(list->getLabel());
553 userLabels.erase(list->getLabel());
555 //restore real lastlabel to save below
556 list->setLabel(saveLabel);
559 lastLabel = list->getLabel();
561 delete list; list = NULL;
563 //get next line to process
564 list = input->getListVector();
568 if (m->control_pressed) { if (list != NULL) { delete list; } delete input; delete groupMap; out.close(); outGroup.close(); return 0; }
570 //output error messages about any remaining user labels
571 set<string>::iterator it;
572 bool needToRun = false;
573 for (it = userLabels.begin(); it != userLabels.end(); it++) {
574 m->mothurOut("Your file does not include the label " + *it);
575 if (processedLabels.count(lastLabel) != 1) {
576 m->mothurOut(". I will use " + lastLabel + "."); m->mothurOutEndLine();
579 m->mothurOut(". Please refer to " + lastLabel + "."); m->mothurOutEndLine();
583 //run last label if you need to
584 if (needToRun == true) {
585 if (list != NULL) { delete list; }
587 list = input->getListVector(lastLabel);
589 m->mothurOut(list->getLabel()); m->mothurOutEndLine();
591 processListGroup(list, groupMap, out, outGroup);
593 delete list; list = NULL;
596 out.close(); outGroup.close();
597 if (list != NULL) { delete list; }
610 catch(exception& e) {
611 m->errorOut(e, "SubSampleCommand", "getSubSampleList");
615 //**********************************************************************************************************************
616 int SubSampleCommand::processListGroup(ListVector*& list, GroupMap*& groupMap, ofstream& out, ofstream& outGroup) {
619 if (size == 0) { //user has not set size, set size = 10% samples size
620 size = int (list->getNumSeqs() * 0.10);
623 int numBins = list->getNumBins();
624 int thisSize = list->getNumSeqs();
626 if (size > thisSize) { m->mothurOut("Your list file only contains " + toString(thisSize) + " sequences. Setting size to " + toString(thisSize) + "."); m->mothurOutEndLine();
630 vector<nameToBin> seqs;
631 for (int i = 0; i < numBins; i++) {
632 string names = list->get(i);
635 string individual = "";
636 int length = names.length();
637 for(int j=0;j<length;j++){
639 nameToBin temp(individual, i);
640 seqs.push_back(temp);
644 individual += names[j];
647 nameToBin temp(individual, i);
648 seqs.push_back(temp);
652 ListVector* temp = new ListVector(numBins);
653 temp->setLabel(list->getLabel());
658 set<int> alreadySelected; //dont want repeat sequence names added
659 alreadySelected.insert(-1);
660 for (int j = 0; j < size; j++) {
662 if (m->control_pressed) { return 0; }
664 //get random sequence to add, making sure we have not already added it
666 while (alreadySelected.count(myrand) != 0) {
667 myrand = (int)((float)(rand()) / (RAND_MAX / (thisSize-1) + 1));
669 alreadySelected.insert(myrand);
672 string oldNames = temp->get(seqs[myrand].bin);
673 if (oldNames == "") { oldNames += seqs[myrand].name; }
674 else { oldNames += "," + seqs[myrand].name; }
676 temp->set(seqs[myrand].bin, oldNames);
679 string group = groupMap->getGroup(seqs[myrand].name);
680 if (group == "not found") { m->mothurOut("[ERROR]: " + seqs[myrand].name + " is not in your groupfile. please correct."); m->mothurOutEndLine(); group = "NOTFOUND"; }
682 outGroup << seqs[myrand].name << '\t' << group << endl;
685 if (m->control_pressed) { return 0; }
692 catch(exception& e) {
693 m->errorOut(e, "SubSampleCommand", "processListGroup");
697 //**********************************************************************************************************************
698 int SubSampleCommand::eliminateZeroOTUS(vector<SharedRAbundVector*>& thislookup) {
701 vector<SharedRAbundVector*> newLookup;
702 for (int i = 0; i < thislookup.size(); i++) {
703 SharedRAbundVector* temp = new SharedRAbundVector();
704 temp->setLabel(thislookup[i]->getLabel());
705 temp->setGroup(thislookup[i]->getGroup());
706 newLookup.push_back(temp);
710 for (int i = 0; i < thislookup[0]->getNumBins(); i++) {
711 if (m->control_pressed) { for (int j = 0; j < newLookup.size(); j++) { delete newLookup[j]; } return 0; }
713 //look at each sharedRabund and make sure they are not all zero
715 for (int j = 0; j < thislookup.size(); j++) {
716 if (thislookup[j]->getAbundance(i) != 0) { allZero = false; break; }
719 //if they are not all zero add this bin
721 for (int j = 0; j < thislookup.size(); j++) {
722 newLookup[j]->push_back(thislookup[j]->getAbundance(i), thislookup[j]->getGroup());
727 for (int j = 0; j < thislookup.size(); j++) { delete thislookup[j]; }
730 thislookup = newLookup;
735 catch(exception& e) {
736 m->errorOut(e, "SubSampleCommand", "eliminateZeroOTUS");
741 //**********************************************************************************************************************