pre.cluster unable to spawn necessary processes adjustment.

[mothur.git] / preclustercommand.cpp

diff --git a/preclustercommand.cpp b/preclustercommand.cpp
index bbf8df27ac5ee691f091f150638b781f85486a11..10ed82571150e9d61e26f62b12508a9e5f967701 100644 (file)
--- a/preclustercommand.cpp
+++ b/preclustercommand.cpp
@@ -192,7 +192,7 @@ PreClusterCommand::PreClusterCommand(string option) {
                        else if (countfile == "not open") { abort = true; countfile =  ""; }    
                        else {   
                 m->setCountTableFile(countfile); 
-                ct.readTable(countfile);
+                ct.readTable(countfile, true, false);
                 if (ct.hasGroupInfo()) { bygroup = true; }
                 else { bygroup = false;  }
             }
@@ -300,6 +300,7 @@ int PreClusterCommand::execute(){
                        m->mothurOut("It took " + toString(time(NULL) - start) + " secs to run pre.cluster."); m->mothurOutEndLine(); 
                                
                }else {
+            if (processors != 1) { m->mothurOut("When using running without group information mothur can only use 1 processor, continuing."); m->mothurOutEndLine(); processors = 1; }
                        if (namefile != "") { readNameFile(); }
                
                        //reads fasta file and return number of seqs
@@ -362,34 +363,37 @@ int PreClusterCommand::createProcessesGroups(string newFName, string newNName, s
                vector<int> processIDS;
                int process = 1;
                int num = 0;
-               
+               bool recalc = false;
+        
                //sanity check
                if (groups.size() < processors) { processors = groups.size(); }
                
                //divide the groups between the processors
                vector<linePair> lines;
-               int numGroupsPerProcessor = groups.size() / processors;
-               for (int i = 0; i < processors; i++) {
-                       int startIndex =  i * numGroupsPerProcessor;
-                       int endIndex = (i+1) * numGroupsPerProcessor;
-                       if(i == (processors - 1)){      endIndex = groups.size();       }
-                       lines.push_back(linePair(startIndex, endIndex));
-               }
+               int remainingPairs = groups.size();
+        int startIndex = 0;
+        for (int remainingProcessors = processors; remainingProcessors > 0; remainingProcessors--) {
+            int numPairs = remainingPairs; //case for last processor
+            if (remainingProcessors != 1) { numPairs = ceil(remainingPairs / remainingProcessors); }
+            lines.push_back(linePair(startIndex, (startIndex+numPairs))); //startIndex, endIndex
+            startIndex = startIndex + numPairs;
+            remainingPairs = remainingPairs - numPairs;
+        }
                
 #if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix)         
                
                //loop through and create all the processes you want
                while (process != processors) {
-                       int pid = fork();
+                       pid_t pid = fork();
                        
                        if (pid > 0) {
                                processIDS.push_back(pid);  //create map from line number to pid so you can append files in correct order later
                                process++;
                        }else if (pid == 0){
                 outputNames.clear();
-                               num = driverGroups(newFName + toString(getpid()) + ".temp", newNName + toString(getpid()) + ".temp", newMFile, lines[process].start, lines[process].end, groups);
+                               num = driverGroups(newFName + m->mothurGetpid(process) + ".temp", newNName + m->mothurGetpid(process) + ".temp", newMFile, lines[process].start, lines[process].end, groups);
                 
-                string tempFile = toString(getpid()) + ".outputNames.temp";
+                string tempFile = m->mothurGetpid(process) + ".outputNames.temp";
                 ofstream outTemp;
                 m->openOutputFile(tempFile, outTemp);
                 
@@ -398,12 +402,57 @@ int PreClusterCommand::createProcessesGroups(string newFName, string newNName, s
                 outTemp.close();
                 
                                exit(0);
-                       }else { 
-                               m->mothurOut("[ERROR]: unable to spawn the necessary processes."); m->mothurOutEndLine(); 
-                               for (int i = 0; i < processIDS.size(); i++) { kill (processIDS[i], SIGINT); }
-                               exit(0);
+                       }else {
+                m->mothurOut("[ERROR]: unable to spawn the number of processes you requested, reducing number to " + toString(process) + "\n"); processors = process;
+                for (int i = 0; i < processIDS.size(); i++) { kill (processIDS[i], SIGINT); }
+                recalc = true;
+                               break;
                        }
                }
+        
+        if (recalc) {
+            lines.clear();
+            num = 0;
+            processIDS.resize(0);
+            process = 1;
+            
+            int remainingPairs = groups.size();
+            int startIndex = 0;
+            for (int remainingProcessors = processors; remainingProcessors > 0; remainingProcessors--) {
+                int numPairs = remainingPairs; //case for last processor
+                if (remainingProcessors != 1) { numPairs = ceil(remainingPairs / remainingProcessors); }
+                lines.push_back(linePair(startIndex, (startIndex+numPairs))); //startIndex, endIndex
+                startIndex = startIndex + numPairs;
+                remainingPairs = remainingPairs - numPairs;
+            }
+            
+            while (process != processors) {
+                pid_t pid = fork();
+                
+                if (pid > 0) {
+                    processIDS.push_back(pid);  //create map from line number to pid so you can append files in correct order later
+                    process++;
+                }else if (pid == 0){
+                    outputNames.clear();
+                    num = driverGroups(newFName + m->mothurGetpid(process) + ".temp", newNName + m->mothurGetpid(process) + ".temp", newMFile, lines[process].start, lines[process].end, groups);
+                    
+                    string tempFile = m->mothurGetpid(process) + ".outputNames.temp";
+                    ofstream outTemp;
+                    m->openOutputFile(tempFile, outTemp);
+                    
+                    outTemp << outputNames.size();
+                    for (int i = 0; i < outputNames.size(); i++) { outTemp << outputNames[i] << endl; }
+                    outTemp.close();
+                    
+                    exit(0);
+                }else {
+                    m->mothurOut("[ERROR]: unable to spawn the necessary processes."); m->mothurOutEndLine();
+                    for (int i = 0; i < processIDS.size(); i++) { kill (processIDS[i], SIGINT); }
+                    exit(0);
+                }
+            }
+        }
+
                
                //do my part
                num = driverGroups(newFName, newNName, newMFile, lines[0].start, lines[0].end, groups);
@@ -903,15 +952,9 @@ void PreClusterCommand::readNameFile(){
                while (!in.eof()) {
                        in >> firstCol >> secondCol; m->gobble(in);
             
-            for (int i = 0; i < firstCol.length(); i++) {
-                if (firstCol[i] == ':') { firstCol[i] = '_'; m->changedSeqNames = true; }
-            }
-            
-            int size = 1;
-            for (int i = 0; i < secondCol.length(); i++) {
-                if (secondCol[i] == ':') { secondCol[i] = '_'; m->changedSeqNames = true; }
-                else if(secondCol[i] == ','){  size++; }
-            }
+            m->checkName(firstCol);
+            m->checkName(secondCol);
+            int size = m->getNumNames(secondCol);
             
                        names[firstCol] = secondCol;
             sizes[firstCol] = size;