X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=mothurout.cpp;h=9f678fef98505cae0de916b1db66914d5f41074f;hb=4e3080b15a3d0c02f7ef2fd779ead433ffdece31;hp=1b72137a66d6cd8038446dbc823033bb3293869d;hpb=14cddf859d97118481161bf3e85a647976020758;p=mothur.git diff --git a/mothurout.cpp b/mothurout.cpp index 1b72137..9f678fe 100644 --- a/mothurout.cpp +++ b/mothurout.cpp @@ -453,7 +453,7 @@ void MothurOut::errorOut(exception& e, string object, string function) { if (object == "cluster"){ mothurOut(" has occurred in the " + object + " class function " + function + ". This error indicates your computer is running out of memory. There are two common causes for this, file size and format.\n\nFile Size:\nThe cluster command loads your distance matrix into RAM, and your distance file is most likely too large to fit in RAM. There are two options to help with this. The first is to use a cutoff. By using a cutoff mothur will only load distances that are below the cutoff. If that is still not enough, there is a command called cluster.split, http://www.mothur.org/wiki/cluster.split which divides the distance matrix, and clusters the smaller pieces separately. You may also be able to reduce the size of the original distance matrix by using the commands outlined in the Schloss SOP, http://www.mothur.org/wiki/Schloss_SOP. \n\nWrong Format:\nThis error can be caused by trying to read a column formatted distance matrix using the phylip parameter. By default, the dist.seqs command generates a column formatted distance matrix. To make a phylip formatted matrix set the dist.seqs command parameter output to lt. \n\nIf you are uable to resolve the issue, please contact Pat Schloss at mothur.bugs@gmail.com, and be sure to include the mothur.logFile with your inquiry."); }else { - mothurOut(" has occurred in the " + object + " class function " + function + ". This error indicates your computer is running out of memory. This is most commonly caused by trying to process a dataset too large, or a file format issue. If you are running our 32bit version, your memory usage is limited to 4G. If you have more than 4G of RAM and are running a 64bit OS, using our 64bit version may resolve your issue. Also, you may be able to reduce the size of your dataset by using the commands outlined in the Schloss SOP, http://www.mothur.org/wiki/Schloss_SOP. If you are uable to resolve the issue, please contact Pat Schloss at mothur.bugs@gmail.com, and be sure to include the mothur.logFile with your inquiry."); + mothurOut(" has occurred in the " + object + " class function " + function + ". This error indicates your computer is running out of memory. This is most commonly caused by trying to process a dataset too large, using multiple processors, or a file format issue. If you are running our 32bit version, your memory usage is limited to 4G. If you have more than 4G of RAM and are running a 64bit OS, using our 64bit version may resolve your issue. If you are using multiple processors, try running the command with processors=1, the more processors you use the more memory is required. Also, you may be able to reduce the size of your dataset by using the commands outlined in the Schloss SOP, http://www.mothur.org/wiki/Schloss_SOP. If you are uable to resolve the issue, please contact Pat Schloss at mothur.bugs@gmail.com, and be sure to include the mothur.logFile with your inquiry."); } } } @@ -951,7 +951,7 @@ string MothurOut::getFullPathName(string fileName){ } for (int i = index; i >= 0; i--) { - newFileName = dirs[i] + "\\\\" + newFileName; + newFileName = dirs[i] + "\\" + newFileName; } return newFileName; @@ -1430,6 +1430,83 @@ vector MothurOut::divideFile(string filename, int& proc) { } } /**************************************************************************************************/ + +vector MothurOut::divideFilePerLine(string filename, int& proc) { + try{ + vector filePos; + filePos.push_back(0); + + FILE * pFile; + unsigned long long size; + + filename = getFullPathName(filename); + + //get num bytes in file + pFile = fopen (filename.c_str(),"rb"); + if (pFile==NULL) perror ("Error opening file"); + else{ + fseek (pFile, 0, SEEK_END); + size=ftell (pFile); + fclose (pFile); + } + +#if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix) + + //estimate file breaks + unsigned long long chunkSize = 0; + chunkSize = size / proc; + + //file to small to divide by processors + if (chunkSize == 0) { proc = 1; filePos.push_back(size); return filePos; } + + //for each process seekg to closest file break and search for next '>' char. make that the filebreak + for (int i = 0; i < proc; i++) { + unsigned long long spot = (i+1) * chunkSize; + + ifstream in; + openInputFile(filename, in); + in.seekg(spot); + + //look for next line break + unsigned long long newSpot = spot; + while (!in.eof()) { + char c = in.get(); + + if ((c == '\n') || (c == '\r') || (c == '\f')) { gobble(in); newSpot = in.tellg(); break; } + else if (int(c) == -1) { break; } + } + + //there was not another line before the end of the file + unsigned long long sanityPos = in.tellg(); + + if (sanityPos == -1) { break; } + else { filePos.push_back(newSpot); } + + in.close(); + } + + //save end pos + filePos.push_back(size); + + //sanity check filePos + for (int i = 0; i < (filePos.size()-1); i++) { + if (filePos[(i+1)] <= filePos[i]) { filePos.erase(filePos.begin()+(i+1)); i--; } + } + + proc = (filePos.size() - 1); +#else + mothurOut("[ERROR]: Windows version should not be calling the divideFile function."); mothurOutEndLine(); + proc=1; + filePos.push_back(size); +#endif + return filePos; + } + catch(exception& e) { + errorOut(e, "MothurOut", "divideFile"); + exit(1); + } +} +/**************************************************************************************************/ int MothurOut::divideFile(string filename, int& proc, vector& files) { try{ @@ -2643,7 +2720,7 @@ void MothurOut::splitAtDash(string& estim, vector& container) { string individual = ""; int estimLength = estim.size(); bool prevEscape = false; - for(int i=0;i& container) { prevEscape = false; } } - } + }*/ + + + for(int i=0;i& container) { string individual = ""; int estimLength = estim.size(); bool prevEscape = false; + /* for(int i=0;i& container) { } } } - container.insert(individual); + */ + + for(int i=0;i& container) { int lineNum; int estimLength = estim.size(); bool prevEscape = false; + /* for(int i=0;i& container) { prevEscape = false; } } - } + }*/ + + for(int i=0;i& container) { exit(1); } } + /***********************************************************************/ string MothurOut::makeList(vector& names) { try { @@ -2810,11 +2948,11 @@ void MothurOut::splitAtChar(string& prefix, string& suffix, char c){ string space = " "; while(suffix.at(0) == ' ') suffix = suffix.substr(1, suffix.length()); - } + }else { suffix = ""; } - } + } catch(exception& e) { - errorOut(e, "MothurOut", "splitAtComma"); + errorOut(e, "MothurOut", "splitAtChar"); exit(1); } } @@ -2830,7 +2968,7 @@ void MothurOut::splitAtComma(string& prefix, string& suffix){ string space = " "; while(suffix.at(0) == ' ') suffix = suffix.substr(1, suffix.length()); - } + }else { suffix = ""; } } catch(exception& e) {