From: westcott Date: Mon, 24 Jan 2011 14:16:21 +0000 (+0000) Subject: remove.rare command X-Git-Url: https://git.donarmstrong.com/?p=mothur.git;a=commitdiff_plain;h=99254b7a399bb635a4acfe8d9c8a586601db5a11 remove.rare command --- diff --git a/Mothur.xcodeproj/project.pbxproj b/Mothur.xcodeproj/project.pbxproj index ce1354b..ade49c9 100644 --- a/Mothur.xcodeproj/project.pbxproj +++ b/Mothur.xcodeproj/project.pbxproj @@ -1588,7 +1588,7 @@ attributes = { ORGANIZATIONNAME = "Schloss Lab"; }; - buildConfigurationList = 1DEB928908733DD80010E9CD /* Build configuration list for PBXProject "Mothur" */; + buildConfigurationList = 1DEB928908733DD80010E9CD /* Build configuration list for PBXProject "mothur" */; compatibilityVersion = "Xcode 3.1"; developmentRegion = English; hasScannedForEncodings = 1; @@ -1934,6 +1934,7 @@ "VERSION=\"\\\"1.15.0\\\"\"", "RELEASE_DATE=\"\\\"12/10/2010\\\"\"", ); + GCC_WARN_ABOUT_MISSING_NEWLINE = YES; GCC_WARN_ABOUT_RETURN_TYPE = YES; GCC_WARN_UNUSED_VARIABLE = YES; INSTALL_PATH = ""; @@ -1998,7 +1999,7 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; - 1DEB928908733DD80010E9CD /* Build configuration list for PBXProject "Mothur" */ = { + 1DEB928908733DD80010E9CD /* Build configuration list for PBXProject "mothur" */ = { isa = XCConfigurationList; buildConfigurations = ( 1DEB928A08733DD80010E9CD /* Debug */, diff --git a/matrixoutputcommand.cpp b/matrixoutputcommand.cpp index 2ea9377..02b30e3 100644 --- a/matrixoutputcommand.cpp +++ b/matrixoutputcommand.cpp @@ -8,6 +8,10 @@ */ #include "matrixoutputcommand.h" +#include "sharedsobscollectsummary.h" +#include "sharedchao1.h" +#include "sharedace.h" +#include "sharednseqs.h" #include "sharedjabund.h" #include "sharedsorabund.h" #include "sharedjclass.h" @@ -16,9 +20,35 @@ #include "sharedsorest.h" #include "sharedthetayc.h" #include "sharedthetan.h" +#include "sharedkstest.h" +#include "whittaker.h" +#include "sharedochiai.h" +#include "sharedanderbergs.h" +#include "sharedkulczynski.h" +#include "sharedkulczynskicody.h" +#include "sharedlennon.h" #include "sharedmorisitahorn.h" #include "sharedbraycurtis.h" - +#include "sharedjackknife.h" +#include "whittaker.h" +#include "odum.h" +#include "canberra.h" +#include "structeuclidean.h" +#include "structchord.h" +#include "hellinger.h" +#include "manhattan.h" +#include "structpearson.h" +#include "soergel.h" +#include "spearman.h" +#include "structkulczynski.h" +#include "structchi2.h" +#include "speciesprofile.h" +#include "hamming.h" +#include "gower.h" +#include "memchi2.h" +#include "memchord.h" +#include "memeuclidean.h" +#include "mempearson.h" //********************************************************************************************************************** vector MatrixOutputCommand::getValidParameters(){ try { @@ -151,7 +181,13 @@ MatrixOutputCommand::MatrixOutputCommand(string option) { int i; for (i=0; iisValidCalculator("matrix", Estimators[i]) == true) { - if (Estimators[i] == "jabund") { + if (Estimators[i] == "sharedsobs") { + matrixCalculators.push_back(new SharedSobsCS()); + }else if (Estimators[i] == "sharedchao") { + matrixCalculators.push_back(new SharedChao1()); + }else if (Estimators[i] == "sharedace") { + matrixCalculators.push_back(new SharedAce()); + }else if (Estimators[i] == "jabund") { matrixCalculators.push_back(new JAbund()); }else if (Estimators[i] == "sorabund") { matrixCalculators.push_back(new SorAbund()); @@ -167,10 +203,62 @@ MatrixOutputCommand::MatrixOutputCommand(string option) { matrixCalculators.push_back(new ThetaYC()); }else if (Estimators[i] == "thetan") { matrixCalculators.push_back(new ThetaN()); + }else if (Estimators[i] == "kstest") { + matrixCalculators.push_back(new KSTest()); + }else if (Estimators[i] == "sharednseqs") { + matrixCalculators.push_back(new SharedNSeqs()); + }else if (Estimators[i] == "ochiai") { + matrixCalculators.push_back(new Ochiai()); + }else if (Estimators[i] == "anderberg") { + matrixCalculators.push_back(new Anderberg()); + }else if (Estimators[i] == "kulczynski") { + matrixCalculators.push_back(new Kulczynski()); + }else if (Estimators[i] == "kulczynskicody") { + matrixCalculators.push_back(new KulczynskiCody()); + }else if (Estimators[i] == "lennon") { + matrixCalculators.push_back(new Lennon()); }else if (Estimators[i] == "morisitahorn") { matrixCalculators.push_back(new MorHorn()); }else if (Estimators[i] == "braycurtis") { matrixCalculators.push_back(new BrayCurtis()); + }else if (Estimators[i] == "whittaker") { + matrixCalculators.push_back(new Whittaker()); + }else if (Estimators[i] == "odum") { + matrixCalculators.push_back(new Odum()); + }else if (Estimators[i] == "canberra") { + matrixCalculators.push_back(new Canberra()); + }else if (Estimators[i] == "structeuclidean") { + matrixCalculators.push_back(new StructEuclidean()); + }else if (Estimators[i] == "structchord") { + matrixCalculators.push_back(new StructChord()); + }else if (Estimators[i] == "hellinger") { + matrixCalculators.push_back(new Hellinger()); + }else if (Estimators[i] == "manhattan") { + matrixCalculators.push_back(new Manhattan()); + }else if (Estimators[i] == "structpearson") { + matrixCalculators.push_back(new StructPearson()); + }else if (Estimators[i] == "soergel") { + matrixCalculators.push_back(new Soergel()); + }else if (Estimators[i] == "spearman") { + matrixCalculators.push_back(new Spearman()); + }else if (Estimators[i] == "structkulczynski") { + matrixCalculators.push_back(new StructKulczynski()); + }else if (Estimators[i] == "speciesprofile") { + matrixCalculators.push_back(new SpeciesProfile()); + }else if (Estimators[i] == "hamming") { + matrixCalculators.push_back(new Hamming()); + }else if (Estimators[i] == "structchi2") { + matrixCalculators.push_back(new StructChi2()); + }else if (Estimators[i] == "gower") { + matrixCalculators.push_back(new Gower()); + }else if (Estimators[i] == "memchi2") { + matrixCalculators.push_back(new MemChi2()); + }else if (Estimators[i] == "memchord") { + matrixCalculators.push_back(new MemChord()); + }else if (Estimators[i] == "memeuclidean") { + matrixCalculators.push_back(new MemEuclidean()); + }else if (Estimators[i] == "mempearson") { + matrixCalculators.push_back(new MemPearson()); } } } @@ -390,6 +478,14 @@ int MatrixOutputCommand::process(vector thisLookup){ //add new pair of sharedrabunds subset.push_back(thisLookup[k]); subset.push_back(thisLookup[l]); + //if this calc needs all groups to calculate the pair load all groups + if (matrixCalculators[i]->getNeedsAll()) { + //load subset with rest of lookup for those calcs that need everyone to calc for a pair + for (int w = 0; w < thisLookup.size(); w++) { + if ((w != k) && (w != l)) { subset.push_back(thisLookup[w]); } + } + } + data = matrixCalculators[i]->getValues(subset); //saves the calculator outputs //save values in similarity matrix simMatrix[k][l] = 1.0 - data[0]; //convert similiarity to distance diff --git a/removerarecommand.cpp b/removerarecommand.cpp index 4ba493f..0ba90f3 100644 --- a/removerarecommand.cpp +++ b/removerarecommand.cpp @@ -16,7 +16,7 @@ //********************************************************************************************************************** vector RemoveRareCommand::getValidParameters(){ try { - string Array[] = {"rabund","sabund", "group", "list", "shared", "nseqs","groups","label","outputdir","inputdir"}; + string Array[] = {"rabund","sabund", "group", "list", "shared","bygroup","nseqs","groups","label","outputdir","inputdir"}; vector myArray (Array, Array+(sizeof(Array)/sizeof(string))); return myArray; } @@ -68,6 +68,7 @@ vector RemoveRareCommand::getRequiredFiles(){ //********************************************************************************************************************** RemoveRareCommand::RemoveRareCommand(string option) { try { + globaldata = GlobalData::getInstance(); abort = false; allLines = 1; @@ -76,7 +77,7 @@ RemoveRareCommand::RemoveRareCommand(string option) { else { //valid paramters for this command - string Array[] = {"rabund","sabund", "group", "list", "shared", "nseqs","groups","label","outputdir","inputdir"}; + string Array[] = {"rabund","sabund", "group", "list", "shared", "bygroup", "nseqs","groups","label","outputdir","inputdir"}; vector myArray (Array, Array+(sizeof(Array)/sizeof(string))); OptionParser parser(option); @@ -162,11 +163,11 @@ RemoveRareCommand::RemoveRareCommand(string option) { else if (rabundfile == "not found") { rabundfile = ""; } groupfile = validParameter.validFile(parameters, "group", true); - if (groupfile == "not open") { abort = true; } + if (groupfile == "not open") { groupfile = ""; abort = true; } else if (groupfile == "not found") { groupfile = ""; } sharedfile = validParameter.validFile(parameters, "shared", true); - if (sharedfile == "not open") { abort = true; } + if (sharedfile == "not open") { sharedfile = ""; abort = true; } else if (sharedfile == "not found") { sharedfile = ""; } if ((sabundfile == "") && (rabundfile == "") && (sharedfile == "") && (listfile == "")) { m->mothurOut("You must provide at least one of the following: rabund, sabund, shared or list."); m->mothurOutEndLine(); abort = true; } @@ -185,6 +186,13 @@ RemoveRareCommand::RemoveRareCommand(string option) { string temp = validParameter.validFile(parameters, "nseqs", false); if (temp == "not found") { m->mothurOut("nseqs is a required parameter."); m->mothurOutEndLine(); abort = true; } else { convert(temp, nseqs); } + + temp = validParameter.validFile(parameters, "bygroup", false); if (temp == "not found") { temp = "f"; } + byGroup = m->isTrue(temp); + + if (byGroup && (sharedfile == "")) { m->mothurOut("The byGroup parameter is only valid with a shared file."); m->mothurOutEndLine(); } + + if ((groupfile != "") && (listfile == "")) { m->mothurOut("A groupfile is only valid with a list file."); m->mothurOutEndLine(); groupfile = ""; } } } @@ -197,13 +205,16 @@ RemoveRareCommand::RemoveRareCommand(string option) { void RemoveRareCommand::help(){ try { - //m->mothurOut("The remove.seqs command reads an .accnos file and at least one of the following file types: fasta, name, group, list, taxonomy, quality or alignreport file.\n"); - //m->mothurOut("It outputs a file containing the sequences NOT in the .accnos file.\n"); - //m->mothurOut("The remove.seqs command parameters are accnos, fasta, name, group, list, taxonomy, qfile, alignreport and dups. You must provide accnos and at least one of the file parameters.\n"); - //m->mothurOut("The dups parameter allows you to remove the entire line from a name file if you remove any name from the line. default=true. \n"); - //m->mothurOut("The remove.seqs command should be in the following format: remove.seqs(accnos=yourAccnos, fasta=yourFasta).\n"); - //m->mothurOut("Example remove.seqs(accnos=amazon.accnos, fasta=amazon.fasta).\n"); - //m->mothurOut("Note: No spaces between parameter labels (i.e. fasta), '=' and parameters (i.e.yourFasta).\n\n"); + m->mothurOut("The remove.rare command parameters are list, rabund, sabund, shared, group, label, groups, bygroup and nseqs.\n"); + m->mothurOut("The remove.rare command reads one of the following file types: list, rabund, sabund or shared file. It outputs a new file after removing the rare otus.\n"); + m->mothurOut("The groups parameter allows you to specify which of the groups you would like analyzed. Default=all. You may separate group names with dashes.\n"); + m->mothurOut("The label parameter is used to analyze specific labels in your input. default=all. You may separate label names with dashes.\n"); + m->mothurOut("The bygroup parameter is only valid with the shared file. default=f, meaning remove any OTU that has nseqs or fewer sequences across all groups.\n"); + m->mothurOut("bygroups=T means remove any OTU that has nseqs or fewer sequences in each group (if groupA has 1 sequence and group B has 100 sequences in OTUZ and nseqs=1, then set the groupA count for OTUZ to 0 and keep groupB's count at 100.) \n"); + m->mothurOut("The nseqs parameter allows you to set the cutoff for an otu to be deemed rare. It is required.\n"); + m->mothurOut("The remove.rare command should be in the following format: remove.rare(shared=yourSharedFile, nseqs=yourRareCutoff).\n"); + m->mothurOut("Example remove.rare(shared=amazon.fn.shared, nseqs=2).\n"); + m->mothurOut("Note: No spaces between parameter labels (i.e. shared), '=' and parameters (i.e.yourSharedFile).\n\n"); } catch(exception& e) { m->errorOut(e, "RemoveRareCommand", "help"); @@ -224,7 +235,7 @@ int RemoveRareCommand::execute(){ if (sabundfile != "") { processSabund(); } if (rabundfile != "") { processRabund(); } if (listfile != "") { processList(); } - //if (sharedfile != "") { processShared(); } + if (sharedfile != "") { processShared(); } if (m->control_pressed) { for (int i = 0; i < outputNames.size(); i++) { remove(outputNames[i].c_str()); } return 0; } @@ -564,12 +575,167 @@ int RemoveRareCommand::processRabund(){ return 0; } + catch(exception& e) { + m->errorOut(e, "RemoveRareCommand", "processRabund"); + exit(1); + } +} +//********************************************************************************************************************** +int RemoveRareCommand::processShared(){ + try { + globaldata->Groups = Groups; + + string thisOutputDir = outputDir; + if (outputDir == "") { thisOutputDir += m->hasPath(sharedfile); } + string outputFileName = thisOutputDir + m->getRootName(m->getSimpleName(sharedfile)) + "pick" + m->getExtension(sharedfile); + outputTypes["shared"].push_back(outputFileName); outputNames.push_back(outputFileName); + + ofstream out; + m->openOutputFile(outputFileName, out); + + //if the users enters label "0.06" and there is no "0.06" in their file use the next lowest label. + InputData input(sharedfile, "sharedfile"); + vector lookup = input.getSharedRAbundVectors(); + string lastLabel = lookup[0]->getLabel(); + set processedLabels; + set userLabels = labels; + + while((lookup[0] != NULL) && ((allLines == 1) || (userLabels.size() != 0))) { + + if (m->control_pressed) { for (int i = 0; i < lookup.size(); i++) { delete lookup[i]; } out.close(); return 0; } + + if(allLines == 1 || labels.count(lookup[0]->getLabel()) == 1){ + + m->mothurOut(lookup[0]->getLabel()); m->mothurOutEndLine(); + processedLabels.insert(lookup[0]->getLabel()); + userLabels.erase(lookup[0]->getLabel()); + + processLookup(lookup, out); + } + + if ((m->anyLabelsToProcess(lookup[0]->getLabel(), userLabels, "") == true) && (processedLabels.count(lastLabel) != 1)) { + string saveLabel = lookup[0]->getLabel(); + + for (int i = 0; i < lookup.size(); i++) { delete lookup[i]; } + lookup = input.getSharedRAbundVectors(lastLabel); + + m->mothurOut(lookup[0]->getLabel()); m->mothurOutEndLine(); + processedLabels.insert(lookup[0]->getLabel()); + userLabels.erase(lookup[0]->getLabel()); + + processLookup(lookup, out); + + //restore real lastlabel to save below + lookup[0]->setLabel(saveLabel); + } + + lastLabel = lookup[0]->getLabel(); + + for (int i = 0; i < lookup.size(); i++) { delete lookup[i]; } + lookup = input.getSharedRAbundVectors(); + } + + if (m->control_pressed) { out.close(); return 0; } + + //output error messages about any remaining user labels + set::iterator it; + bool needToRun = false; + for (it = userLabels.begin(); it != userLabels.end(); it++) { + m->mothurOut("Your file does not include the label " + *it); + if (processedLabels.count(lastLabel) != 1) { + m->mothurOut(". I will use " + lastLabel + "."); m->mothurOutEndLine(); + needToRun = true; + }else { + m->mothurOut(". Please refer to " + lastLabel + "."); m->mothurOutEndLine(); + } + } + + //run last label if you need to + if (needToRun == true) { + for (int i = 0; i < lookup.size(); i++) { if (lookup[i] != NULL) { delete lookup[i]; } } + lookup = input.getSharedRAbundVectors(lastLabel); + + m->mothurOut(lookup[0]->getLabel()); m->mothurOutEndLine(); + + processLookup(lookup, out); + + for (int i = 0; i < lookup.size(); i++) { delete lookup[i]; } + } + + return 0; + } catch(exception& e) { m->errorOut(e, "RemoveRareCommand", "processSabund"); exit(1); } } //********************************************************************************************************************** +int RemoveRareCommand::processLookup(vector& lookup, ofstream& out){ + try { + + vector newRabunds; newRabunds.resize(lookup.size()); + for (int i = 0; i < lookup.size(); i++) { + newRabunds[i].setGroup(lookup[i]->getGroup()); + newRabunds[i].setLabel(lookup[i]->getLabel()); + } + + if (byGroup) { + + //for each otu + for (int i = 0; i < lookup[0]->getNumBins(); i++) { + bool allZero = true; + + if (m->control_pressed) { return 0; } + + //for each group + for (int j = 0; j < lookup.size(); j++) { + + //are you rare? + if (lookup[j]->getAbundance(i) > nseqs) { + newRabunds[j].push_back(lookup[j]->getAbundance(i), newRabunds[j].getGroup()); + allZero = false; + }else { + newRabunds[j].push_back(0, newRabunds[j].getGroup()); + } + } + + //eliminates zero otus + if (allZero) { for (int j = 0; j < newRabunds.size(); j++) { newRabunds[j].pop_back(); } } + } + }else { + //for each otu + for (int i = 0; i < lookup[0]->getNumBins(); i++) { + + if (m->control_pressed) { return 0; } + + int totalAbund = 0; + //get total otu abundance + for (int j = 0; j < lookup.size(); j++) { + newRabunds[j].push_back(lookup[j]->getAbundance(i), newRabunds[j].getGroup()); + totalAbund += lookup[j]->getAbundance(i); + } + + //eliminates otus below rare cutoff + if (totalAbund <= nseqs) { for (int j = 0; j < newRabunds.size(); j++) { newRabunds[j].pop_back(); } } + } + } + + //do we have any otus above the rare cutoff + if (newRabunds[0].getNumBins() != 0) { + for (int j = 0; j < newRabunds.size(); j++) { + out << newRabunds[j].getLabel() << '\t' << newRabunds[j].getGroup() << '\t'; + newRabunds[j].print(out); + } + } + + return 0; + } + catch(exception& e) { + m->errorOut(e, "RemoveRareCommand", "processLookup"); + exit(1); + } +} +//********************************************************************************************************************** diff --git a/removerarecommand.h b/removerarecommand.h index 7db54b4..388ec2d 100644 --- a/removerarecommand.h +++ b/removerarecommand.h @@ -13,6 +13,7 @@ #include "command.hpp" #include "listvector.hpp" +#include "globaldata.hpp" class RemoveRareCommand : public Command { @@ -29,6 +30,7 @@ public: void help(); private: + GlobalData* globaldata; string sabundfile, rabundfile, sharedfile, groupfile, listfile, outputDir, groups, label; int nseqs, allLines; bool abort, byGroup; @@ -40,6 +42,7 @@ private: int processRabund(); int processList(); int processShared(); + int processLookup(vector&, ofstream&); }; diff --git a/shhhercommand.h b/shhhercommand.h index b423b4d..650bf8c 100644 --- a/shhhercommand.h +++ b/shhhercommand.h @@ -110,4 +110,5 @@ private: }; -#endif \ No newline at end of file +#endif + diff --git a/soergel.h b/soergel.h index 7156ee7..c26473a 100644 --- a/soergel.h +++ b/soergel.h @@ -27,4 +27,7 @@ private: /***********************************************************************/ -#endif \ No newline at end of file +#endif + + + diff --git a/spearman.h b/spearman.h index 084514e..f49d86f 100644 --- a/spearman.h +++ b/spearman.h @@ -28,4 +28,9 @@ private: /***********************************************************************/ -#endif \ No newline at end of file +#endif + + + + + diff --git a/structchi2.h b/structchi2.h index aa82e75..614db07 100644 --- a/structchi2.h +++ b/structchi2.h @@ -27,4 +27,8 @@ private: /***********************************************************************/ -#endif \ No newline at end of file +#endif + + + + diff --git a/treegroupscommand.cpp b/treegroupscommand.cpp index 6e32a37..adf140a 100644 --- a/treegroupscommand.cpp +++ b/treegroupscommand.cpp @@ -747,6 +747,14 @@ int TreeGroupCommand::process(vector thisLookup) { //add new pair of sharedrabunds subset.push_back(thisLookup[k]); subset.push_back(thisLookup[l]); + //if this calc needs all groups to calculate the pair load all groups + if (treeCalculators[i]->getNeedsAll()) { + //load subset with rest of lookup for those calcs that need everyone to calc for a pair + for (int w = 0; w < thisLookup.size(); w++) { + if ((w != k) && (w != l)) { subset.push_back(thisLookup[w]); } + } + } + data = treeCalculators[i]->getValues(subset); //saves the calculator outputs //cout << thisLookup[k]->getGroup() << '\t' << thisLookup[l]->getGroup() << '\t' << (1.0 - data[0]) << endl; if (m->control_pressed) { return 1; } diff --git a/validcalculator.cpp b/validcalculator.cpp index 6583bbe..f8c7cc3 100644 --- a/validcalculator.cpp +++ b/validcalculator.cpp @@ -516,16 +516,46 @@ void ValidCalculators::initialHeat() { /********************************************************************/ void ValidCalculators::initialMatrix() { try { - matrix["jabund"] = "jabund"; + matrix["sharedsobs"] = "sharedsobs"; + matrix["sharedchao"] = "sharedchao"; + matrix["sharedace"] = "sharedace"; + matrix["jabund"] = "jabund"; matrix["sorabund"] = "sorabund"; - matrix["jclass"] = "jclass"; + matrix["jclass"] = "jclass"; matrix["sorclass"] = "sorclass"; matrix["jest"] = "jest"; - matrix["sorest"] = "sorest"; + matrix["sorest"] = "sorest"; matrix["thetayc"] = "thetayc"; - matrix["thetan"] = "thetan"; + matrix["thetan"] = "thetan"; + matrix["kstest"] = "kstest"; + matrix["whittaker"] = "whittaker"; + matrix["sharednseqs"] = "sharednseqs"; + matrix["ochiai"] = "ochiai"; + matrix["anderberg"] = "anderberg"; + matrix["kulczynski"] = "kulczynski"; + matrix["kulczynskicody"] = "kulczynskicody"; + matrix["lennon"] = "lennon"; matrix["morisitahorn"] = "morisitahorn"; - matrix["braycurtis"] = "braycurtis"; + matrix["braycurtis"] = "braycurtis"; + matrix["odum"] = "odum"; + matrix["canberra"] = "canberra"; + matrix["structeuclidean"] = "structeuclidean"; + matrix["structchord"] = "structchord"; + matrix["hellinger"] = "hellinger"; + matrix["manhattan"] = "manhattan"; + matrix["structpearson"] = "structpearson"; + matrix["structkulczynski"] = "structkulczynski"; + matrix["structchi2"] = "structchi2"; + matrix["soergel"] = "soergel"; + matrix["spearman"] = "spearman"; + matrix["speciesprofile"] = "speciesprofile"; + matrix["hamming"] = "hamming"; + matrix["gower"] = "gower"; + matrix["memchi2"] = "memchi2"; + matrix["memchord"] = "memchord"; + matrix["memeuclidean"] = "memeuclidean"; + matrix["mempearson"] = "mempearson"; + } catch(exception& e) { m->errorOut(e, "ValidCalculator", "initialMatrix");