X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=metastatscommand.cpp;h=9dff3543f49efa25037ce7fb3dd9eeeebbe8c787;hb=70491a12902e89b85cfa6b44a7b7fbe066ee2ac1;hp=1aaa41e91a3d15eb9d07bb73723ae7514c9b1216;hpb=fc1ed1ae1b022719176910ab71993bd6535810ad;p=mothur.git diff --git a/metastatscommand.cpp b/metastatscommand.cpp index 1aaa41e..9dff354 100644 --- a/metastatscommand.cpp +++ b/metastatscommand.cpp @@ -8,23 +8,22 @@ */ #include "metastatscommand.h" -#include "metastats.h" #include "sharedutilities.h" -#include "mothurmetastats.h" + //********************************************************************************************************************** vector MetaStatsCommand::setParameters(){ try { - CommandParameter pshared("shared", "InputTypes", "", "", "none", "none", "none",false,true); parameters.push_back(pshared); - CommandParameter pdesign("design", "InputTypes", "", "", "none", "none", "none",false,true); parameters.push_back(pdesign); - CommandParameter pprocessors("processors", "Number", "", "1", "", "", "",false,false); parameters.push_back(pprocessors); - CommandParameter piters("iters", "Number", "", "1000", "", "", "",false,false); parameters.push_back(piters); - CommandParameter pthreshold("threshold", "Number", "", "0.05", "", "", "",false,false); parameters.push_back(pthreshold); - CommandParameter plabel("label", "String", "", "", "", "", "",false,false); parameters.push_back(plabel); - CommandParameter pgroups("groups", "String", "", "", "", "", "",false,false); parameters.push_back(pgroups); - CommandParameter psets("sets", "String", "", "", "", "", "",false,false); parameters.push_back(psets); - CommandParameter pinputdir("inputdir", "String", "", "", "", "", "",false,false); parameters.push_back(pinputdir); - CommandParameter poutputdir("outputdir", "String", "", "", "", "", "",false,false); parameters.push_back(poutputdir); + CommandParameter pshared("shared", "InputTypes", "", "", "none", "none", "none","metastats",false,true,true); parameters.push_back(pshared); + CommandParameter pdesign("design", "InputTypes", "", "", "none", "none", "none","",false,true,true); parameters.push_back(pdesign); + CommandParameter pprocessors("processors", "Number", "", "1", "", "", "","",false,false,true); parameters.push_back(pprocessors); + CommandParameter piters("iters", "Number", "", "1000", "", "", "","",false,false); parameters.push_back(piters); + CommandParameter pthreshold("threshold", "Number", "", "0.05", "", "", "","",false,false); parameters.push_back(pthreshold); + CommandParameter plabel("label", "String", "", "", "", "", "","",false,false); parameters.push_back(plabel); + CommandParameter pgroups("groups", "String", "", "", "", "", "","",false,false); parameters.push_back(pgroups); + CommandParameter psets("sets", "String", "", "", "", "", "","",false,false); parameters.push_back(psets); + CommandParameter pinputdir("inputdir", "String", "", "", "", "", "","",false,false); parameters.push_back(pinputdir); + CommandParameter poutputdir("outputdir", "String", "", "", "", "", "","",false,false); parameters.push_back(poutputdir); vector myArray; for (int i = 0; i < parameters.size(); i++) { myArray.push_back(parameters[i].name); } @@ -62,6 +61,21 @@ string MetaStatsCommand::getHelpString(){ } } //********************************************************************************************************************** +string MetaStatsCommand::getOutputPattern(string type) { + try { + string pattern = ""; + + if (type == "metastats") { pattern = "[filename],[distance],[group],metastats"; } + else { m->mothurOut("[ERROR]: No definition for type " + type + " output pattern.\n"); m->control_pressed = true; } + + return pattern; + } + catch(exception& e) { + m->errorOut(e, "MetaStatsCommand", "getOutputPattern"); + exit(1); + } +} +//********************************************************************************************************************** MetaStatsCommand::MetaStatsCommand(){ try { abort = true; calledHelp = true; @@ -167,7 +181,7 @@ MetaStatsCommand::MetaStatsCommand(string option) { else { pickedGroups = true; m->splitAtDash(groups, Groups); - m->Groups = Groups; + m->setGroups(Groups); } sets = validParameter.validFile(parameters, "sets", false); @@ -178,14 +192,14 @@ MetaStatsCommand::MetaStatsCommand(string option) { string temp = validParameter.validFile(parameters, "iters", false); if (temp == "not found") { temp = "1000"; } - convert(temp, iters); + m->mothurConvert(temp, iters); temp = validParameter.validFile(parameters, "threshold", false); if (temp == "not found") { temp = "0.05"; } - convert(temp, threshold); + m->mothurConvert(temp, threshold); temp = validParameter.validFile(parameters, "processors", false); if (temp == "not found"){ temp = m->getProcessors(); } m->setProcessors(temp); - convert(temp, processors); + m->mothurConvert(temp, processors); } } @@ -215,8 +229,9 @@ int MetaStatsCommand::execute(){ //setup the pairwise comparions of sets for metastats //calculate number of comparisons i.e. with groups A,B,C = AB, AC, BC = 3; //make sure sets are all in designMap - SharedUtil* util = new SharedUtil(); - util->setGroups(Sets, designMap->namesOfGroups); + SharedUtil* util = new SharedUtil(); + vector dGroups = designMap->getNamesOfGroups(); + util->setGroups(Sets, dGroups); delete util; int numGroups = Sets.size(); @@ -231,26 +246,24 @@ int MetaStatsCommand::execute(){ //only 1 combo if (numGroups == 2) { processors = 1; } else if (numGroups < 2) { m->mothurOut("Not enough sets, I need at least 2 valid sets. Unable to complete command."); m->mothurOutEndLine(); m->control_pressed = true; } - - #if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) - if(processors != 1){ - int numPairs = namesOfGroupCombos.size(); - int numPairsPerProcessor = numPairs / processors; + + if(processors != 1){ + int numPairs = namesOfGroupCombos.size(); + int numPairsPerProcessor = numPairs / processors; - for (int i = 0; i < processors; i++) { - int startPos = i * numPairsPerProcessor; - if(i == processors - 1){ - numPairsPerProcessor = numPairs - i * numPairsPerProcessor; - } - lines.push_back(linePair(startPos, numPairsPerProcessor)); - } - } - #endif + for (int i = 0; i < processors; i++) { + int startPos = i * numPairsPerProcessor; + if(i == processors - 1){ + numPairsPerProcessor = numPairs - i * numPairsPerProcessor; + } + lines.push_back(linePair(startPos, numPairsPerProcessor)); + } + } //as long as you are not at the end of the file or done wih the lines you want while((lookup[0] != NULL) && ((allLines == 1) || (userLabels.size() != 0))) { - if (m->control_pressed) { outputTypes.clear(); for (int i = 0; i < lookup.size(); i++) { delete lookup[i]; } m->Groups.clear(); delete input; delete designMap; for (int i = 0; i < outputNames.size(); i++) { m->mothurRemove(outputNames[i]); } return 0; } + if (m->control_pressed) { outputTypes.clear(); for (int i = 0; i < lookup.size(); i++) { delete lookup[i]; } m->clearGroups(); delete input; delete designMap; for (int i = 0; i < outputNames.size(); i++) { m->mothurRemove(outputNames[i]); } return 0; } if(allLines == 1 || labels.count(lookup[0]->getLabel()) == 1){ @@ -281,13 +294,13 @@ int MetaStatsCommand::execute(){ //prevent memory leak for (int i = 0; i < lookup.size(); i++) { delete lookup[i]; lookup[i] = NULL; } - if (m->control_pressed) { outputTypes.clear(); m->Groups.clear(); delete input; delete designMap; for (int i = 0; i < outputNames.size(); i++) { m->mothurRemove(outputNames[i]); } return 0; } + if (m->control_pressed) { outputTypes.clear(); m->clearGroups(); delete input; delete designMap; for (int i = 0; i < outputNames.size(); i++) { m->mothurRemove(outputNames[i]); } return 0; } //get next line to process lookup = input->getSharedRAbundVectors(); } - if (m->control_pressed) { outputTypes.clear(); m->Groups.clear(); delete input; delete designMap; for (int i = 0; i < outputNames.size(); i++) { m->mothurRemove(outputNames[i]); } return 0; } + if (m->control_pressed) { outputTypes.clear(); m->clearGroups(); delete input; delete designMap; for (int i = 0; i < outputNames.size(); i++) { m->mothurRemove(outputNames[i]); } return 0; } //output error messages about any remaining user labels set::iterator it; @@ -315,7 +328,7 @@ int MetaStatsCommand::execute(){ } //reset groups parameter - m->Groups.clear(); + m->clearGroups(); delete input; delete designMap; @@ -338,13 +351,13 @@ int MetaStatsCommand::execute(){ int MetaStatsCommand::process(vector& thisLookUp){ try { - #if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) + if(processors == 1){ driver(0, namesOfGroupCombos.size(), thisLookUp); }else{ int process = 1; vector processIDS; - + #if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix) //loop through and create all the processes you want while (process != processors) { int pid = fork(); @@ -370,11 +383,70 @@ int MetaStatsCommand::process(vector& thisLookUp){ int temp = processIDS[i]; wait(&temp); } - } - #else - driver(0, namesOfGroupCombos.size(), thisLookUp); - #endif + #else + + ////////////////////////////////////////////////////////////////////////////////////////////////////// + //Windows version shared memory, so be careful when passing variables through the summarySharedData struct. + //Above fork() will clone, so memory is separate, but that's not the case with windows, + //Taking advantage of shared memory to pass results vectors. + ////////////////////////////////////////////////////////////////////////////////////////////////////// + + vector pDataArray; + DWORD dwThreadIdArray[processors-1]; + HANDLE hThreadArray[processors-1]; + + //Create processor worker threads. + for( int i=1; i newLookup; + vector designMapGroups; + for (int k = 0; k < thisLookUp.size(); k++) { + SharedRAbundVector* temp = new SharedRAbundVector(); + temp->setLabel(thisLookUp[k]->getLabel()); + temp->setGroup(thisLookUp[k]->getGroup()); + newLookup.push_back(temp); + designMapGroups.push_back(designMap->getGroup(thisLookUp[k]->getGroup())); + } + + //for each bin + for (int k = 0; k < thisLookUp[0]->getNumBins(); k++) { + if (m->control_pressed) { for (int j = 0; j < newLookup.size(); j++) { delete newLookup[j]; } return 0; } + for (int j = 0; j < thisLookUp.size(); j++) { newLookup[j]->push_back(thisLookUp[j]->getAbundance(k), thisLookUp[j]->getGroup()); } + } + + // Allocate memory for thread data. + metastatsData* tempSum = new metastatsData(sharedfile, outputDir, m, lines[i].start, lines[i].num, namesOfGroupCombos, newLookup, designMapGroups, iters, threshold); + pDataArray.push_back(tempSum); + processIDS.push_back(i); + + hThreadArray[i-1] = CreateThread(NULL, 0, MyMetastatsThreadFunction, pDataArray[i-1], 0, &dwThreadIdArray[i-1]); + } + + //do my part + driver(lines[0].start, lines[0].num, thisLookUp); + + //Wait until all threads have terminated. + WaitForMultipleObjects(processors-1, hThreadArray, TRUE, INFINITE); + + //Close all thread handles and free memory allocations. + for(int i=0; i < pDataArray.size(); i++){ + if (pDataArray[i]->count != (pDataArray[i]->num)) { + m->mothurOut("[ERROR]: process " + toString(i) + " only processed " + toString(pDataArray[i]->count) + " of " + toString(pDataArray[i]->num) + " groups assigned to it, quitting. \n"); m->control_pressed = true; + } + for (int j = 0; j < pDataArray[i]->thisLookUp.size(); j++) { delete pDataArray[i]->thisLookUp[j]; } + for (int j = 0; j < pDataArray[i]->outputNames.size(); j++) { + outputNames.push_back(pDataArray[i]->outputNames[j]); + outputTypes["metastats"].push_back(pDataArray[i]->outputNames[j]); + } + + CloseHandle(hThreadArray[i]); + delete pDataArray[i]; + } + #endif + } + return 0; } @@ -395,7 +467,11 @@ int MetaStatsCommand::driver(int start, int num, vector& th string setB = namesOfGroupCombos[c][1]; //get filename - string outputFileName = outputDir + m->getRootName(m->getSimpleName(sharedfile)) + thisLookUp[0]->getLabel() + "." + setA + "-" + setB + ".metastats"; + map variables; + variables["[filename]"] = outputDir + m->getRootName(m->getSimpleName(sharedfile)); + variables["[distance]"] = thisLookUp[0]->getLabel(); + variables["[group]"] = setA + "-" + setB; + string outputFileName = getOutputFileName("metastats",variables); outputNames.push_back(outputFileName); outputTypes["metastats"].push_back(outputFileName); //int nameLength = outputFileName.length(); //char * output = new char[nameLength]; @@ -423,19 +499,18 @@ int MetaStatsCommand::driver(int start, int num, vector& th setACount++; } } - - //for (int i = 0; i < subset.size(); i++) { cout << subset[i]->getGroup() << endl; } - + if ((setACount == 0) || (setBCount == 0)) { m->mothurOut("Missing shared info for " + setA + " or " + setB + ". Skipping comparison."); m->mothurOutEndLine(); outputNames.pop_back(); }else { + //fill data for (int j = 0; j < thisLookUp[0]->getNumBins(); j++) { //data[j] = new double[subset.size()]; data2[j].resize(subset.size(), 0.0); + for (int i = 0; i < subset.size(); i++) { - //data[j][i] = (subset[i]->getAbundance(j)); data2[j][i] = (subset[i]->getAbundance(j)); } } @@ -447,7 +522,6 @@ int MetaStatsCommand::driver(int start, int num, vector& th MothurMetastats mothurMeta(threshold, iters); mothurMeta.runMetastats(outputFileName , data2, setACount); m->mothurOutEndLine(); - m->mothurOutEndLine(); }