]> git.donarmstrong.com Git - mothur.git/blob - makecontigscommand.cpp
Merge remote-tracking branch 'mothur/master'
[mothur.git] / makecontigscommand.cpp
1 //
2 //  makecontigscommand.cpp
3 //  Mothur
4 //
5 //  Created by Sarah Westcott on 5/15/12.
6 //  Copyright (c) 2012 Schloss Lab. All rights reserved.
7 //
8
9 #include "makecontigscommand.h"
10
11 //**********************************************************************************************************************
12 vector<string> MakeContigsCommand::setParameters(){     
13         try {
14                 CommandParameter pfastq("ffastq", "InputTypes", "", "", "FastaFastqFile", "FastaFastqFile", "fastqGroup","fasta-qfile",false,false,true); parameters.push_back(pfastq);
15         CommandParameter prfastq("rfastq", "InputTypes", "", "", "none", "none", "fastqGroup","fasta-qfile",false,false,true); parameters.push_back(prfastq);
16         CommandParameter pfasta("ffasta", "InputTypes", "", "", "FastaFastqFile", "FastaFastqFile", "fastaGroup","fasta",false,false,true); parameters.push_back(pfasta);
17         CommandParameter prfasta("rfasta", "InputTypes", "", "", "none", "none", "none","fastaGroup",false,false,true); parameters.push_back(prfasta);
18         CommandParameter pfqual("fqfile", "InputTypes", "", "", "none", "none", "qfileGroup","qfile",false,false,true); parameters.push_back(pfqual);
19         CommandParameter prqual("rqfile", "InputTypes", "", "", "none", "none", "qfileGroup","qfile",false,false,true); parameters.push_back(prqual);
20         CommandParameter pfile("file", "InputTypes", "", "", "FastaFastqFile", "FastaFastqFile", "none","fasta-qfile",false,false,true); parameters.push_back(pfile);
21         CommandParameter poligos("oligos", "InputTypes", "", "", "none", "none", "none","group",false,false,true); parameters.push_back(poligos);
22                 CommandParameter ppdiffs("pdiffs", "Number", "", "0", "", "", "","",false,false,true); parameters.push_back(ppdiffs);
23                 CommandParameter pbdiffs("bdiffs", "Number", "", "0", "", "", "","",false,false,true); parameters.push_back(pbdiffs);
24 //        CommandParameter pldiffs("ldiffs", "Number", "", "0", "", "", "",false,false); parameters.push_back(pldiffs);
25 //              CommandParameter psdiffs("sdiffs", "Number", "", "0", "", "", "",false,false); parameters.push_back(psdiffs);
26         CommandParameter ptdiffs("tdiffs", "Number", "", "0", "", "", "","",false,false); parameters.push_back(ptdiffs);
27
28         CommandParameter palign("align", "Multiple", "needleman-gotoh", "needleman", "", "", "","",false,false); parameters.push_back(palign);
29         CommandParameter pallfiles("allfiles", "Boolean", "", "F", "", "", "","",false,false); parameters.push_back(pallfiles);
30                 CommandParameter pmatch("match", "Number", "", "1.0", "", "", "","",false,false); parameters.push_back(pmatch);
31                 CommandParameter pmismatch("mismatch", "Number", "", "-1.0", "", "", "","",false,false); parameters.push_back(pmismatch);
32                 CommandParameter pgapopen("gapopen", "Number", "", "-2.0", "", "", "","",false,false); parameters.push_back(pgapopen);
33                 CommandParameter pgapextend("gapextend", "Number", "", "-1.0", "", "", "","",false,false); parameters.push_back(pgapextend);
34         CommandParameter pthreshold("threshold", "Number", "", "40", "", "", "","",false,false); parameters.push_back(pthreshold);
35                 CommandParameter pprocessors("processors", "Number", "", "1", "", "", "","",false,false,true); parameters.push_back(pprocessors);
36                 CommandParameter pinputdir("inputdir", "String", "", "", "", "", "","",false,false); parameters.push_back(pinputdir);
37                 CommandParameter poutputdir("outputdir", "String", "", "", "", "", "","",false,false); parameters.push_back(poutputdir);
38                 
39                 vector<string> myArray;
40                 for (int i = 0; i < parameters.size(); i++) {   myArray.push_back(parameters[i].name);          }
41                 return myArray;
42         }
43         catch(exception& e) {
44                 m->errorOut(e, "MakeContigsCommand", "setParameters");
45                 exit(1);
46         }
47 }
48 //**********************************************************************************************************************
49 string MakeContigsCommand::getHelpString(){     
50         try {
51                 string helpString = "";
52                 helpString += "The make.contigs command reads a file, forward fastq file and a reverse fastq file or forward fasta and reverse fasta files and outputs new fasta.  It will also provide new quality files if the fastq or file parameter is used.\n";
53         helpString += "If an oligos file is provided barcodes and primers will be trimmed, and a group file will be created.\n";
54                 helpString += "The make.contigs command parameters are ffastq, rfastq, oligos, tdiffs, bdiffs, ldiffs, sdiffs, pdiffs, align, match, mismatch, gapopen, gapextend, allfiles and processors.\n";
55                 helpString += "The ffastq and rfastq, file, or ffasta and rfasta parameters are required.\n";
56         helpString += "The file parameter is 2 column file containing the forward fastq files in the first column and their matching reverse fastq files in the second column.  Mothur will process each pair and create a combined fasta and qual file with all the sequences.\n";
57         helpString += "The ffastq and rfastq parameters are used to provide a forward fastq and reverse fastq file to process.  If you provide one, you must provide the other.\n";
58         helpString += "The ffasta and rfasta parameters are used to provide a forward fasta and reverse fasta file to process.  If you provide one, you must provide the other.\n";
59         helpString += "The fqfile and rqfile parameters are used to provide a forward quality and reverse quality files to process with the ffasta and rfasta parameters.  If you provide one, you must provide the other.\n";
60                 helpString += "The align parameter allows you to specify the alignment method to use.  Your options are: gotoh and needleman. The default is needleman.\n";
61         helpString += "The tdiffs parameter is used to specify the total number of differences allowed in the sequence. The default is pdiffs + bdiffs + sdiffs + ldiffs.\n";
62                 helpString += "The bdiffs parameter is used to specify the number of differences allowed in the barcode. The default is 0.\n";
63                 helpString += "The pdiffs parameter is used to specify the number of differences allowed in the primer. The default is 0.\n";
64         helpString += "The ldiffs parameter is used to specify the number of differences allowed in the linker. The default is 0.\n";
65                 helpString += "The sdiffs parameter is used to specify the number of differences allowed in the spacer. The default is 0.\n";
66                 helpString += "The match parameter allows you to specify the bonus for having the same base. The default is 1.0.\n";
67                 helpString += "The mistmatch parameter allows you to specify the penalty for having different bases.  The default is -1.0.\n";
68                 helpString += "The gapopen parameter allows you to specify the penalty for opening a gap in an alignment. The default is -2.0.\n";
69                 helpString += "The gapextend parameter allows you to specify the penalty for extending a gap in an alignment.  The default is -1.0.\n";
70         helpString += "The threshold parameter allows you to set a quality scores threshold. In the case where we are trying to decide whether to keep a base or remove it because the base is compared to a gap in the other fragment, if the base has a quality score below the threshold we eliminate it. Default=40.\n";
71         helpString += "The processors parameter allows you to specify how many processors you would like to use.  The default is 1. \n";
72         helpString += "The allfiles parameter will create separate group and fasta file for each grouping. The default is F.\n";
73         helpString += "The make.contigs command should be in the following format: \n";
74                 helpString += "make.contigs(ffastq=yourForwardFastqFile, rfastq=yourReverseFastqFile, align=yourAlignmentMethod) \n";
75                 helpString += "Note: No spaces between parameter labels (i.e. ffastq), '=' and parameters (i.e.yourForwardFastqFile).\n";
76                 return helpString;
77         }
78         catch(exception& e) {
79                 m->errorOut(e, "MakeContigsCommand", "getHelpString");
80                 exit(1);
81         }
82 }
83 //**********************************************************************************************************************
84 string MakeContigsCommand::getOutputPattern(string type) {
85     try {
86         string pattern = "";
87         
88         if (type == "fasta") {  pattern = "[filename],[tag],contigs.fasta"; } 
89         else if (type == "qfile") {  pattern = "[filename],[tag],contigs.qual"; } 
90         else if (type == "group") {  pattern = "[filename],[tag],contigs.groups"; }
91         else if (type == "mismatch") {  pattern = "[filename],[tag],contigs.mismatch"; }
92         else { m->mothurOut("[ERROR]: No definition for type " + type + " output pattern.\n"); m->control_pressed = true;  }
93         
94         return pattern;
95     }
96     catch(exception& e) {
97         m->errorOut(e, "MakeContigsCommand", "getOutputPattern");
98         exit(1);
99     }
100 }
101 //**********************************************************************************************************************
102 MakeContigsCommand::MakeContigsCommand(){       
103         try {
104                 abort = true; calledHelp = true; 
105                 setParameters();
106                 vector<string> tempOutNames;
107                 outputTypes["fasta"] = tempOutNames;
108                 outputTypes["qfile"] = tempOutNames;
109         outputTypes["group"] = tempOutNames;
110         outputTypes["mismatch"] = tempOutNames;
111         }
112         catch(exception& e) {
113                 m->errorOut(e, "MakeContigsCommand", "MakeContigsCommand");
114                 exit(1);
115         }
116 }
117 //**********************************************************************************************************************
118 MakeContigsCommand::MakeContigsCommand(string option)  {
119         try {
120                 abort = false; calledHelp = false;   
121         
122                 //allow user to run help
123                 if(option == "help") { help(); abort = true; calledHelp = true; }
124                 else if(option == "citation") { citation(); abort = true; calledHelp = true;}
125                 
126                 else {
127                         vector<string> myArray = setParameters();
128                         
129                         OptionParser parser(option);
130                         map<string, string> parameters = parser.getParameters(); 
131                         
132                         ValidParameters validParameter("pairwise.seqs");
133                         map<string, string>::iterator it;
134                         
135                         //check to make sure all parameters are valid for command
136                         for (it = parameters.begin(); it != parameters.end(); it++) { 
137                                 if (validParameter.isValidParameter(it->first, myArray, it->second) != true) {  abort = true;  }
138                         }
139                         
140                         //initialize outputTypes
141                         vector<string> tempOutNames;
142                         outputTypes["fasta"] = tempOutNames;
143                         outputTypes["qfile"] = tempOutNames;
144             outputTypes["mismatch"] = tempOutNames;
145             outputTypes["group"] = tempOutNames;
146                         
147             
148                         //if the user changes the input directory command factory will send this info to us in the output parameter 
149                         string inputDir = validParameter.validFile(parameters, "inputdir", false);              
150                         if (inputDir == "not found"){   inputDir = "";          }
151                         else { 
152                                 string path;
153                 it = parameters.find("ffastq");
154                                 //user has given a template file
155                                 if(it != parameters.end()){ 
156                                         path = m->hasPath(it->second);
157                                         //if the user has not given a path then, add inputdir. else leave path alone.
158                                         if (path == "") {       parameters["ffastq"] = inputDir + it->second;           }
159                                 }
160                 
161                 it = parameters.find("rfastq");
162                                 //user has given a template file
163                                 if(it != parameters.end()){ 
164                                         path = m->hasPath(it->second);
165                                         //if the user has not given a path then, add inputdir. else leave path alone.
166                                         if (path == "") {       parameters["rfastq"] = inputDir + it->second;           }
167                                 }
168                 
169                 it = parameters.find("ffasta");
170                                 //user has given a template file
171                                 if(it != parameters.end()){ 
172                                         path = m->hasPath(it->second);
173                                         //if the user has not given a path then, add inputdir. else leave path alone.
174                                         if (path == "") {       parameters["ffasta"] = inputDir + it->second;           }
175                                 }
176                 
177                 it = parameters.find("rfasta");
178                                 //user has given a template file
179                                 if(it != parameters.end()){ 
180                                         path = m->hasPath(it->second);
181                                         //if the user has not given a path then, add inputdir. else leave path alone.
182                                         if (path == "") {       parameters["rfasta"] = inputDir + it->second;           }
183                                 }
184                 
185                 it = parameters.find("fqfile");
186                                 //user has given a template file
187                                 if(it != parameters.end()){ 
188                                         path = m->hasPath(it->second);
189                                         //if the user has not given a path then, add inputdir. else leave path alone.
190                                         if (path == "") {       parameters["fqfile"] = inputDir + it->second;           }
191                                 }
192                 
193                 it = parameters.find("rqfile");
194                                 //user has given a template file
195                                 if(it != parameters.end()){ 
196                                         path = m->hasPath(it->second);
197                                         //if the user has not given a path then, add inputdir. else leave path alone.
198                                         if (path == "") {       parameters["rqfile"] = inputDir + it->second;           }
199                                 }
200                 
201                 it = parameters.find("file");
202                                 //user has given a template file
203                                 if(it != parameters.end()){ 
204                                         path = m->hasPath(it->second);
205                                         //if the user has not given a path then, add inputdir. else leave path alone.
206                                         if (path == "") {       parameters["file"] = inputDir + it->second;             }
207                                 }
208                 
209                 it = parameters.find("oligos");
210                                 //user has given a template file
211                                 if(it != parameters.end()){ 
212                                         path = m->hasPath(it->second);
213                                         //if the user has not given a path then, add inputdir. else leave path alone.
214                                         if (path == "") {       parameters["oligos"] = inputDir + it->second;           }
215                                 }
216             }
217             
218             ffastqfile = validParameter.validFile(parameters, "ffastq", true);
219                         if (ffastqfile == "not open") {  abort = true; }        
220                         else if (ffastqfile == "not found") { ffastqfile = ""; }
221                         
222                         rfastqfile = validParameter.validFile(parameters, "rfastq", true);
223                         if (rfastqfile == "not open") {  abort = true; }        
224                         else if (rfastqfile == "not found") { rfastqfile = "";  }
225             
226             ffastafile = validParameter.validFile(parameters, "ffasta", true);
227                         if (ffastafile == "not open") {  abort = true; }        
228                         else if (ffastafile == "not found") { ffastafile = ""; }
229                         
230                         rfastafile = validParameter.validFile(parameters, "rfasta", true);
231                         if (rfastafile == "not open") {  abort = true; }        
232                         else if (rfastafile == "not found") { rfastafile = "";  }
233             
234             fqualfile = validParameter.validFile(parameters, "fqfile", true);
235                         if (fqualfile == "not open") {  abort = true; } 
236                         else if (fqualfile == "not found") { fqualfile = ""; }
237                         
238                         rqualfile = validParameter.validFile(parameters, "rqfile", true);
239                         if (rqualfile == "not open") {  abort = true; } 
240                         else if (rqualfile == "not found") { rqualfile = "";  }
241             
242             file = validParameter.validFile(parameters, "file", true);
243                         if (file == "not open") {  abort = true; }      
244                         else if (file == "not found") { file = "";  }
245             
246             //provide at least
247             if ((file == "") && (ffastafile == "") && (ffastqfile == "")) { abort = true; m->mothurOut("[ERROR]: The file, ffastq and rfastq or ffasta and rfasta parameters are required.\n"); }
248             if ((file != "") && ((ffastafile != "") || (ffastqfile != ""))) { abort = true; m->mothurOut("[ERROR]: The file, ffastq and rfastq or ffasta and rfasta parameters are required.\n"); }
249             if ((ffastqfile != "") && (rfastqfile == "")) {  abort = true; m->mothurOut("[ERROR]: If you provide use the ffastq, you must provide a rfastq file.\n"); }
250             if ((ffastqfile == "") && (rfastqfile != "")) {  abort = true; m->mothurOut("[ERROR]: If you provide use the rfastq, you must provide a ffastq file.\n"); }
251             if ((ffastafile != "") && (rfastafile == "")) {  abort = true; m->mothurOut("[ERROR]: If you provide use the ffasta, you must provide a rfasta file.\n"); }
252             if ((ffastafile == "") && (rfastafile != "")) {  abort = true; m->mothurOut("[ERROR]: If you provide use the rfasta, you must provide a ffasta file.\n"); }
253             if ((fqualfile != "") && (rqualfile == "")) {  abort = true; m->mothurOut("[ERROR]: If you provide use the fqfile, you must provide a rqfile file.\n"); }
254             if ((fqualfile == "") && (rqualfile != "")) {  abort = true; m->mothurOut("[ERROR]: If you provide use the rqfile, you must provide a fqfile file.\n"); }
255             if (((fqualfile != "") || (rqualfile != "")) && ((ffastafile == "") || (rfastafile == ""))) {
256                 abort = true; m->mothurOut("[ERROR]: If you provide use the rqfile or fqfile file, you must provide the ffasta and rfasta parameters.\n");
257             }
258             
259             oligosfile = validParameter.validFile(parameters, "oligos", true);
260                         if (oligosfile == "not found")      {   oligosfile = "";        }
261                         else if(oligosfile == "not open")   {   abort = true;       } 
262                         else {   m->setOligosFile(oligosfile);          }
263             
264             //if the user changes the output directory command factory will send this info to us in the output parameter 
265                         outputDir = validParameter.validFile(parameters, "outputdir", false);           if (outputDir == "not found"){  
266                  outputDir = ""; 
267             }
268                         
269
270                         //check for optional parameter and set defaults
271                         // ...at some point should added some additional type checking...
272                         string temp;
273                         temp = validParameter.validFile(parameters, "match", false);            if (temp == "not found"){       temp = "1.0";                   }
274                         m->mothurConvert(temp, match);  
275                         
276                         temp = validParameter.validFile(parameters, "mismatch", false);         if (temp == "not found"){       temp = "-1.0";                  }
277                         m->mothurConvert(temp, misMatch);  
278             if (misMatch > 0) { m->mothurOut("[ERROR]: mismatch must be negative.\n"); abort=true; }
279                         
280                         temp = validParameter.validFile(parameters, "gapopen", false);          if (temp == "not found"){       temp = "-2.0";                  }
281                         m->mothurConvert(temp, gapOpen);  
282             if (gapOpen > 0) { m->mothurOut("[ERROR]: gapopen must be negative.\n"); abort=true; }
283                         
284                         temp = validParameter.validFile(parameters, "gapextend", false);        if (temp == "not found"){       temp = "-1.0";                  }
285                         m->mothurConvert(temp, gapExtend); 
286             if (gapExtend > 0) { m->mothurOut("[ERROR]: gapextend must be negative.\n"); abort=true; }
287                         
288             temp = validParameter.validFile(parameters, "threshold", false);    if (temp == "not found"){       temp = "40";                    }
289                         m->mothurConvert(temp, threshold); 
290             if ((threshold < 0) || (threshold > 40)) { m->mothurOut("[ERROR]: threshold must be between 0 and 40.\n"); abort=true; }
291
292                         temp = validParameter.validFile(parameters, "processors", false);       if (temp == "not found"){       temp = m->getProcessors();      }
293                         m->setProcessors(temp);
294                         m->mothurConvert(temp, processors);
295             
296             temp = validParameter.validFile(parameters, "bdiffs", false);               if (temp == "not found") { temp = "0"; }
297                         m->mothurConvert(temp, bdiffs);
298                         
299                         temp = validParameter.validFile(parameters, "pdiffs", false);           if (temp == "not found") { temp = "0"; }
300                         m->mothurConvert(temp, pdiffs);
301             
302   //          temp = validParameter.validFile(parameters, "ldiffs", false);             if (temp == "not found") { temp = "0"; }
303 //                      m->mothurConvert(temp, ldiffs);
304             ldiffs = 0;
305             
306  //           temp = validParameter.validFile(parameters, "sdiffs", false);             if (temp == "not found") { temp = "0"; }
307  //           m->mothurConvert(temp, sdiffs);
308             sdiffs = 0;
309                         
310                         temp = validParameter.validFile(parameters, "tdiffs", false);           if (temp == "not found") { int tempTotal = pdiffs + bdiffs;  temp = toString(tempTotal); }
311                         m->mothurConvert(temp, tdiffs);
312                         
313                         if(tdiffs == 0){        tdiffs = bdiffs + pdiffs;       }  //+ ldiffs + sdiffs;
314
315             temp = validParameter.validFile(parameters, "allfiles", false);             if (temp == "not found") { temp = "F"; }
316                         allFiles = m->isTrue(temp);
317                         
318                         align = validParameter.validFile(parameters, "align", false);           if (align == "not found"){      align = "needleman";    }
319                         if ((align != "needleman") && (align != "gotoh")) { m->mothurOut(align + " is not a valid alignment method. Options are needleman or gotoh. I will use needleman."); m->mothurOutEndLine(); align = "needleman"; }
320         }
321                 
322         }
323         catch(exception& e) {
324                 m->errorOut(e, "MakeContigsCommand", "MakeContigsCommand");
325                 exit(1);
326         }
327 }
328 //**********************************************************************************************************************
329 int MakeContigsCommand::execute(){
330         try {
331                 if (abort == true) { if (calledHelp) { return 0; }  return 2;   }
332         
333         //read ffastq and rfastq files creating fasta and qual files.
334         //this function will create a forward and reverse, fasta and qual files for each processor.
335         //files has an entry for each processor. files[i][0] = forwardFasta, files[i][1] = forwardQual, files[i][2] = reverseFasta, files[i][3] = reverseQual.  filesToProcess is for each filepair in the file parameter file.  for ffastq and rfastq this will be size 1.
336         unsigned long int numReads = 0;
337         int start = time(NULL);
338         longestBase = 1000;
339         m->mothurOut("Reading fastq data...\n"); 
340         vector < vector< vector<string> > > filesToProcess = preProcessData(numReads);
341         m->mothurOut("Done.\n");
342        
343         if (m->control_pressed) { return 0; }
344         
345         map<string, string> cvars;
346         string compOutputDir = outputDir;
347         if (outputDir == "") { compOutputDir = m->hasPath(file); }
348         cvars["[filename]"] = compOutputDir + m->getRootName(m->getSimpleName(file));
349         cvars["[tag]"] = "";
350         string compositeGroupFile = getOutputFileName("group",cvars);
351         cvars["[tag]"] = "trim";
352         string compositeFastaFile = getOutputFileName("fasta",cvars);
353         string compositeQualFile = getOutputFileName("qfile",cvars);
354         cvars["[tag]"] = "scrap";
355         string compositeScrapFastaFile = getOutputFileName("fasta",cvars);
356         string compositeScrapQualFile = getOutputFileName("qfile",cvars);
357         cvars["[tag]"] = "";
358         string compositeMisMatchFile = getOutputFileName("mismatch",cvars);
359         
360         if (filesToProcess.size() > 1) { //clear files for append below
361             ofstream outCTFasta, outCTQual, outCSFasta, outCSQual, outCMisMatch;
362             m->openOutputFile(compositeFastaFile, outCTFasta); outCTFasta.close();
363             m->openOutputFile(compositeScrapFastaFile, outCSFasta); outCSFasta.close();
364             m->openOutputFile(compositeMisMatchFile, outCMisMatch); outCMisMatch.close();
365             m->openOutputFile(compositeQualFile, outCTQual); outCTQual.close();
366             m->openOutputFile(compositeScrapQualFile, outCSQual); outCSQual.close();
367             outputNames.push_back(compositeFastaFile); outputTypes["fasta"].push_back(compositeFastaFile);
368             outputNames.push_back(compositeQualFile); outputTypes["qfile"].push_back(compositeQualFile);
369             outputNames.push_back(compositeMisMatchFile); outputTypes["mismatch"].push_back(compositeMisMatchFile);
370             outputNames.push_back(compositeScrapFastaFile); outputTypes["fasta"].push_back(compositeScrapFastaFile);
371             outputNames.push_back(compositeScrapQualFile); outputTypes["qfile"].push_back(compositeScrapQualFile);
372         }
373         
374         for (int l = 0; l < filesToProcess.size(); l++) {
375             
376             m->mothurOut("\n>>>>>\tProcessing " + filesToProcess[l][0][0] + " (file " + toString(l+1) + " of " + toString(filesToProcess.size()) + ")\t<<<<<\n");
377             
378             vector<vector<string> > fastaFileNames;
379             vector<vector<string> > qualFileNames;
380             createGroup = false;
381             string outputGroupFileName;
382             map<string, string> variables; 
383             string thisOutputDir = outputDir;
384             if (outputDir == "") {  thisOutputDir = m->hasPath(filesToProcess[l][0][0]); }
385             variables["[filename]"] = thisOutputDir + m->getRootName(m->getSimpleName(filesToProcess[l][0][0]));
386             variables["[tag]"] = "";
387             if(oligosfile != ""){
388                 createGroup = getOligos(fastaFileNames, qualFileNames, variables["[filename]"]);
389                 if (createGroup) { 
390                     outputGroupFileName = getOutputFileName("group",variables);
391                     outputNames.push_back(outputGroupFileName); outputTypes["group"].push_back(outputGroupFileName);
392                 }
393             }
394             
395             variables["[tag]"] = "trim";
396             string outFastaFile = getOutputFileName("fasta",variables);
397             string outQualFile = getOutputFileName("qfile",variables);
398             variables["[tag]"] = "scrap";
399             string outScrapFastaFile = getOutputFileName("fasta",variables);
400             string outScrapQualFile = getOutputFileName("qfile",variables);
401             variables["[tag]"] = "";
402             string outMisMatchFile = getOutputFileName("mismatch",variables);
403             outputNames.push_back(outFastaFile); outputTypes["fasta"].push_back(outFastaFile);
404             outputNames.push_back(outScrapFastaFile); outputTypes["fasta"].push_back(outScrapFastaFile);
405             if (filesToProcess[l][0][1] != "") {
406                 outputNames.push_back(outQualFile); outputTypes["qfile"].push_back(outQualFile);
407                 outputNames.push_back(outScrapQualFile); outputTypes["qfile"].push_back(outScrapQualFile);
408             }
409             outputNames.push_back(outMisMatchFile); outputTypes["mismatch"].push_back(outMisMatchFile);
410             
411             m->mothurOut("Making contigs...\n"); 
412             createProcesses(filesToProcess[l], outFastaFile, outQualFile, outScrapFastaFile, outScrapQualFile, outMisMatchFile, fastaFileNames, qualFileNames);
413             m->mothurOut("Done.\n");
414             
415             //remove temp fasta and qual files
416             for (int i = 0; i < processors; i++) { for(int j = 0; j < filesToProcess[l][i].size(); j++) { m->mothurRemove(filesToProcess[l][i][j]); }  }
417             
418             if (m->control_pressed) { for (int i = 0; i < outputNames.size(); i++) {    m->mothurRemove(outputNames[i]); }  return 0; }
419             
420             if(allFiles){
421                 map<string, string> uniqueFastaNames;// so we don't add the same groupfile multiple times
422                 map<string, string>::iterator it;
423                 set<string> namesToRemove;
424                 for(int i=0;i<fastaFileNames.size();i++){
425                     for(int j=0;j<fastaFileNames[0].size();j++){
426                         if (fastaFileNames[i][j] != "") {
427                             if (namesToRemove.count(fastaFileNames[i][j]) == 0) {
428                                 if(m->isBlank(fastaFileNames[i][j])){
429                                     m->mothurRemove(fastaFileNames[i][j]);
430                                     namesToRemove.insert(fastaFileNames[i][j]);
431                                     
432                                     if (filesToProcess[l][0][1] != "") {
433                                         m->mothurRemove(qualFileNames[i][j]);
434                                         namesToRemove.insert(qualFileNames[i][j]);
435                                     }
436                                 }else{  
437                                     it = uniqueFastaNames.find(fastaFileNames[i][j]);
438                                     if (it == uniqueFastaNames.end()) { 
439                                         uniqueFastaNames[fastaFileNames[i][j]] = barcodeNameVector[i];  
440                                     }   
441                                 }
442                             }
443                         }
444                     }
445                 }
446                 
447                 //remove names for outputFileNames, just cleans up the output
448                 vector<string> outputNames2;
449                 for(int i = 0; i < outputNames.size(); i++) { if (namesToRemove.count(outputNames[i]) == 0) { outputNames2.push_back(outputNames[i]); } }
450                 outputNames = outputNames2;
451                 
452                 for (it = uniqueFastaNames.begin(); it != uniqueFastaNames.end(); it++) {
453                     ifstream in;
454                     m->openInputFile(it->first, in);
455                     
456                     ofstream out;
457                     string thisGroupName = thisOutputDir + m->getRootName(m->getSimpleName(it->first));
458                     thisGroupName += getOutputFileName("group",variables); outputNames.push_back(thisGroupName); outputTypes["group"].push_back(thisGroupName); 
459                     m->openOutputFile(thisGroupName, out);
460                     
461                     while (!in.eof()){
462                         if (m->control_pressed) { break; }
463                         
464                         Sequence currSeq(in); m->gobble(in);
465                         out << currSeq.getName() << '\t' << it->second << endl;  
466                     }
467                     in.close();
468                     out.close();
469                 }
470             }
471             
472             if (createGroup) {
473                 ofstream outGroup;
474                 m->openOutputFile(outputGroupFileName, outGroup);
475                 for (map<string, string>::iterator itGroup = groupMap.begin(); itGroup != groupMap.end(); itGroup++) {
476                     outGroup << itGroup->first << '\t' << itGroup->second << endl;
477                 }
478                 outGroup.close();
479             }
480             
481             if (filesToProcess.size() > 1) { //merge into large combo files
482                 if (createGroup) {  
483                     if (l == 0) { 
484                         ofstream outCGroup;
485                         m->openOutputFile(compositeGroupFile, outCGroup); outCGroup.close();
486                         outputNames.push_back(compositeGroupFile); outputTypes["group"].push_back(compositeGroupFile);
487                     }
488                     m->appendFiles(outputGroupFileName, compositeGroupFile);  
489                 }
490                 m->appendFiles(outMisMatchFile, compositeMisMatchFile);
491                 m->appendFiles(outFastaFile, compositeFastaFile);
492                 m->appendFiles(outQualFile, compositeQualFile);
493                 m->appendFiles(outScrapFastaFile, compositeScrapFastaFile);
494                 m->appendFiles(outScrapQualFile, compositeScrapQualFile);
495             }
496         }
497         m->mothurOut("It took " + toString(time(NULL) - start) + " secs to process " + toString(numReads) + " sequences.\n");
498         
499         if (m->control_pressed) {       for (int i = 0; i < outputNames.size(); i++) {  m->mothurRemove(outputNames[i]); } return 0;    }
500         
501                 //output group counts
502                 m->mothurOutEndLine();
503                 int total = 0;
504                 if (groupCounts.size() != 0) {  m->mothurOut("Group count: \n");  }
505                 for (map<string, int>::iterator it = groupCounts.begin(); it != groupCounts.end(); it++) {
506             total += it->second; m->mothurOut(it->first + "\t" + toString(it->second)); m->mothurOutEndLine(); 
507                 }
508                 if (total != 0) { m->mothurOut("Total of all groups is " + toString(total)); m->mothurOutEndLine(); }
509                 
510                 if (m->control_pressed) {       for (int i = 0; i < outputNames.size(); i++) {  m->mothurRemove(outputNames[i]); } return 0;    }
511         
512         string currentFasta = "";
513                 itTypes = outputTypes.find("fasta");
514                 if (itTypes != outputTypes.end()) {
515                         if ((itTypes->second).size() != 0) { currentFasta = (itTypes->second)[0]; m->setFastaFile(currentFasta); }
516                 }
517         
518         string currentQual = "";
519                 itTypes = outputTypes.find("qfile");
520                 if (itTypes != outputTypes.end()) {
521                         if ((itTypes->second).size() != 0) { currentQual = (itTypes->second)[0]; m->setQualFile(currentQual); }
522                 }
523         
524         string currentGroup = "";
525                 itTypes = outputTypes.find("group");
526                 if (itTypes != outputTypes.end()) {
527                         if ((itTypes->second).size() != 0) { currentGroup = (itTypes->second)[0]; m->setGroupFile(currentGroup); }
528                 }
529                 
530         //output files created by command
531                 m->mothurOutEndLine();
532                 m->mothurOut("Output File Names: "); m->mothurOutEndLine();
533                 for (int i = 0; i < outputNames.size(); i++) {  m->mothurOut(outputNames[i]); m->mothurOutEndLine();    }
534                 m->mothurOutEndLine();
535
536         return 0;
537     }
538         catch(exception& e) {
539                 m->errorOut(e, "MakeContigsCommand", "execute");
540                 exit(1);
541         }
542 }
543 //**********************************************************************************************************************
544 vector< vector< vector<string> > > MakeContigsCommand::preProcessData(unsigned long int& numReads) {
545         try {
546         vector< vector< vector<string> > > filesToProcess;
547         
548         if (ffastqfile != "") { //reading one file
549             vector< vector<string> > files = readFastqFiles(numReads, ffastqfile, rfastqfile); 
550             //adjust for really large processors or really small files
551             if (numReads == 0) {  m->mothurOut("[ERROR]: no good reads.\n"); m->control_pressed = true; }
552             if (numReads < processors) { 
553                 for (int i = numReads; i < processors; i++) { for(int j = 0; j < files[i].size(); j++) { m->mothurRemove(files[i][j]); } files[i].clear(); }
554                 files.resize(numReads);
555                 processors = numReads; 
556             }
557             filesToProcess.push_back(files);
558         }else if (file != "") { //reading multiple files
559             //return only valid pairs
560             vector< vector<string> > filePairsToProcess = readFileNames(file);
561             
562             if (m->control_pressed) { return filesToProcess; }
563             
564             if (filePairsToProcess.size() != 0) {
565                 for (int i = 0; i < filePairsToProcess.size(); i++) {
566                     
567                     if (m->control_pressed) { for (int l = 0; l < filesToProcess.size(); l++) { for (int k = 0; k < filesToProcess[l].size(); k++) { for(int j = 0; j < filesToProcess[l][k].size(); j++) { m->mothurRemove(filesToProcess[l][k][j]); } filesToProcess[l][k].clear(); } return filesToProcess; } }
568                     
569                     unsigned long int thisFilesReads;
570                     vector< vector<string> > files = readFastqFiles(thisFilesReads, filePairsToProcess[i][0], filePairsToProcess[i][1]); 
571                     
572                     //adjust for really large processors or really small files
573                     if (thisFilesReads < processors) { 
574                         m->mothurOut("[ERROR]: " + filePairsToProcess[i][0] + " has less than " + toString(processors) + " good reads, skipping\n"); 
575                         for (int k = 0; k < files.size(); k++) { for(int j = 0; j < files[k].size(); j++) { m->mothurRemove(files[k][j]); } files[k].clear(); }
576                     }else {
577                         filesToProcess.push_back(files);
578                         numReads += thisFilesReads;
579                     }
580                 }
581                 //all files are bad
582                 if (numReads == 0) {  m->control_pressed = true; }
583             }
584         }else if (ffastafile != "") {
585             vector< vector<string> > files = readFastaFiles(numReads, ffastafile, rfastafile);
586             //adjust for really large processors or really small files
587             if (numReads == 0) {  m->mothurOut("[ERROR]: no good reads.\n"); m->control_pressed = true; }
588             if (numReads < processors) { 
589                 for (int i = numReads; i < processors; i++) { for(int j = 0; j < files[i].size(); j++) { m->mothurRemove(files[i][j]); } files[i].clear(); }
590                 files.resize(numReads);
591                 processors = numReads; 
592             }
593             filesToProcess.push_back(files);
594         }else { m->control_pressed = true; } //should not get here
595         
596         return filesToProcess;
597     }
598         catch(exception& e) {
599                 m->errorOut(e, "MakeContigsCommand", "preProcessData");
600                 exit(1);
601         }
602 }
603 //**********************************************************************************************************************
604 int MakeContigsCommand::createProcesses(vector< vector<string> > files, string outputFasta, string outputQual, string outputScrapFasta, string outputScrapQual, string outputMisMatches, vector<vector<string> > fastaFileNames, vector<vector<string> > qualFileNames) {
605         try {
606                 int num = 0;
607                 vector<int> processIDS;
608 #if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix)
609                 int process = 0;
610                 
611                 //loop through and create all the processes you want
612                 while (process != processors-1) {
613                         int pid = fork();
614                         
615                         if (pid > 0) {
616                                 processIDS.push_back(pid);  //create map from line number to pid so you can append files in correct order later
617                                 process++;
618                         }else if (pid == 0){
619                 vector<vector<string> > tempFASTAFileNames = fastaFileNames;
620                                 vector<vector<string> > tempPrimerQualFileNames = qualFileNames;
621                 
622                                 if(allFiles){
623                                         ofstream temp;
624                     
625                                         for(int i=0;i<tempFASTAFileNames.size();i++){
626                                                 for(int j=0;j<tempFASTAFileNames[i].size();j++){
627                                                         if (tempFASTAFileNames[i][j] != "") {
628                                                                 tempFASTAFileNames[i][j] += toString(getpid()) + ".temp";
629                                                                 m->openOutputFile(tempFASTAFileNames[i][j], temp);                      temp.close();
630                                 
631                                 tempPrimerQualFileNames[i][j] += toString(getpid()) + ".temp";
632                                 m->openOutputFile(tempPrimerQualFileNames[i][j], temp);         temp.close();
633                                                         }
634                                                 }
635                                         }
636                                 }
637
638                                 num = driver(files[process], 
639                              outputFasta + toString(getpid()) + ".temp", 
640                              outputQual + toString(getpid()) + ".temp", 
641                              outputScrapFasta + toString(getpid()) + ".temp", 
642                              outputScrapQual + toString(getpid()) + ".temp",
643                              outputMisMatches + toString(getpid()) + ".temp",
644                              tempFASTAFileNames,
645                              tempPrimerQualFileNames);
646                                 
647                                 //pass groupCounts to parent
648                 ofstream out;
649                 string tempFile = toString(getpid()) + ".num.temp";
650                 m->openOutputFile(tempFile, out);
651                 out << num << endl;
652                                 if(createGroup){
653                                         out << groupCounts.size() << endl;
654                                         
655                                         for (map<string, int>::iterator it = groupCounts.begin(); it != groupCounts.end(); it++) {
656                                                 out << it->first << '\t' << it->second << endl;
657                                         }
658                     
659                     out << groupMap.size() << endl;
660                     for (map<string, string>::iterator it = groupMap.begin(); it != groupMap.end(); it++) {
661                                                 out << it->first << '\t' << it->second << endl;
662                                         }
663                                 }
664                 out.close();
665                                 
666                                 exit(0);
667                         }else { 
668                                 m->mothurOut("[ERROR]: unable to spawn the necessary processes."); m->mothurOutEndLine(); 
669                                 for (int i = 0; i < processIDS.size(); i++) { kill (processIDS[i], SIGINT); }
670                                 exit(0);
671                         }
672                 }
673                 
674         ofstream temp;
675                 m->openOutputFile(outputFasta, temp);           temp.close();
676         m->openOutputFile(outputScrapFasta, temp);              temp.close();
677         if (files[processors-1][1] != "") {
678             m->openOutputFile(outputScrapQual, temp);           temp.close();
679             m->openOutputFile(outputQual, temp);        temp.close();
680         }
681         
682                 //do my part
683                 num = driver(files[processors-1], outputFasta, outputQual, outputScrapFasta, outputScrapQual, outputMisMatches, fastaFileNames, qualFileNames);
684                 
685                 //force parent to wait until all the processes are done
686                 for (int i=0;i<processIDS.size();i++) { 
687                         int temp = processIDS[i];
688                         wait(&temp);
689                 }
690         
691                 for (int i = 0; i < processIDS.size(); i++) {
692             ifstream in;
693             string tempFile = toString(processIDS[i]) + ".num.temp";
694             m->openInputFile(tempFile, in);
695             int tempNum;
696             in >> tempNum; num += tempNum; m->gobble(in);
697             
698                         if(createGroup){
699                                 string group;
700                                 in >> tempNum; m->gobble(in);
701                                 
702                                 if (tempNum != 0) {
703                                         for (int j = 0; j < tempNum; j++) { 
704                         int groupNum;
705                                                 in >> group >> groupNum; m->gobble(in);
706                         
707                                                 map<string, int>::iterator it = groupCounts.find(group);
708                                                 if (it == groupCounts.end()) {  groupCounts[group] = groupNum; }
709                                                 else { groupCounts[it->first] += groupNum; }
710                                         }
711                                 }
712                 in >> tempNum; m->gobble(in);
713                 if (tempNum != 0) {
714                                         for (int j = 0; j < tempNum; j++) { 
715                         string group, seqName;
716                                                 in >> seqName >> group; m->gobble(in);
717                         
718                                                 map<string, string>::iterator it = groupMap.find(seqName);
719                                                 if (it == groupMap.end()) {     groupMap[seqName] = group; }
720                                                 else { m->mothurOut("[ERROR]: " + seqName + " is in your fasta file more than once. Sequence names must be unique. please correct.\n");  }
721                                         }
722                                 }
723                         }
724             in.close(); m->mothurRemove(tempFile);
725         }
726     #else
727         
728         //////////////////////////////////////////////////////////////////////////////////////////////////////
729                 //Windows version shared memory, so be careful when passing variables through the contigsData struct. 
730                 //Above fork() will clone, so memory is separate, but that's not the case with windows, 
731                 //////////////////////////////////////////////////////////////////////////////////////////////////////
732                 
733                 vector<contigsData*> pDataArray; 
734                 DWORD   dwThreadIdArray[processors-1];
735                 HANDLE  hThreadArray[processors-1]; 
736                 
737                 //Create processor worker threads.
738                 for( int h=0; h<processors-1; h++ ){
739                         string extension = "";
740                         if (h != 0) { extension = toString(h) + ".temp"; processIDS.push_back(h); }
741             vector<vector<string> > tempFASTAFileNames = fastaFileNames;
742             vector<vector<string> > tempPrimerQualFileNames = qualFileNames;
743             
744             if(allFiles){
745                 ofstream temp;
746                 
747                 for(int i=0;i<tempFASTAFileNames.size();i++){
748                     for(int j=0;j<tempFASTAFileNames[i].size();j++){
749                         if (tempFASTAFileNames[i][j] != "") {
750                             tempFASTAFileNames[i][j] += extension;
751                             m->openOutputFile(tempFASTAFileNames[i][j], temp);                  temp.close();
752                             
753                             
754                             tempPrimerQualFileNames[i][j] += extension;
755                             m->openOutputFile(tempPrimerQualFileNames[i][j], temp);             temp.close();
756                         }
757                     }
758                 }
759             }
760
761                                   
762                         contigsData* tempcontig = new contigsData(files[h], (outputFasta + extension), (outputQual + extension), (outputScrapFasta + extension), (outputScrapQual + extension),(outputMisMatches + extension), align, m, match, misMatch, gapOpen, gapExtend, threshold, barcodes, primers, tempFASTAFileNames, tempPrimerQualFileNames, barcodeNameVector, primerNameVector, pdiffs, bdiffs, tdiffs, createGroup, allFiles, h);
763                         pDataArray.push_back(tempcontig);
764             
765                         hThreadArray[h] = CreateThread(NULL, 0, MyContigsThreadFunction, pDataArray[h], 0, &dwThreadIdArray[h]);   
766                 }
767         
768         vector<vector<string> > tempFASTAFileNames = fastaFileNames;
769         vector<vector<string> > tempPrimerQualFileNames = qualFileNames;
770
771         if(allFiles){
772             ofstream temp;
773             string extension = toString(processors-1) + ".temp";
774             
775             for(int i=0;i<tempFASTAFileNames.size();i++){
776                 for(int j=0;j<tempFASTAFileNames[i].size();j++){
777                     if (tempFASTAFileNames[i][j] != "") {
778                         tempFASTAFileNames[i][j] += extension;
779                         m->openOutputFile(tempFASTAFileNames[i][j], temp);                      temp.close();
780                         
781                         
782                         tempPrimerQualFileNames[i][j] += extension;
783                         m->openOutputFile(tempPrimerQualFileNames[i][j], temp);         temp.close();
784                     }
785                 }
786             }
787         }
788
789                 //parent do my part
790                 ofstream temp;
791                 m->openOutputFile(outputFasta, temp);           temp.close();
792         m->openOutputFile(outputScrapFasta, temp);              temp.close();
793         if (files[processors-1][1] != "") {
794             m->openOutputFile(outputScrapQual, temp);           temp.close();
795             m->openOutputFile(outputQual, temp);        temp.close();
796                 }
797         
798         //do my part
799                 num = driver(files[processors-1], (outputFasta+ toString(processors-1) + ".temp"), (outputQual+ toString(processors-1) + ".temp"), (outputScrapFasta+ toString(processors-1) + ".temp"), (outputScrapQual+ toString(processors-1) + ".temp"), (outputMisMatches+ toString(processors-1) + ".temp"), tempFASTAFileNames, tempPrimerQualFileNames);       
800         
801                 //Wait until all threads have terminated.
802                 WaitForMultipleObjects(processors-1, hThreadArray, TRUE, INFINITE);
803                 
804                 //Close all thread handles and free memory allocations.
805                 for(int i=0; i < pDataArray.size(); i++){
806                         num += pDataArray[i]->count;
807             for (map<string, int>::iterator it = pDataArray[i]->groupCounts.begin(); it != pDataArray[i]->groupCounts.end(); it++) {
808                 map<string, int>::iterator it2 = groupCounts.find(it->first);
809                 if (it2 == groupCounts.end()) { groupCounts[it->first] = it->second; }
810                 else { groupCounts[it->first] += it->second; }
811             }
812             for (map<string, string>::iterator it = pDataArray[i]->groupMap.begin(); it != pDataArray[i]->groupMap.end(); it++) {
813                 map<string, string>::iterator it2 = groupMap.find(it->first);
814                 if (it2 == groupMap.end()) {    groupMap[it->first] = it->second; }
815                 else { m->mothurOut("[ERROR]: " + it->first + " is in your fasta file more than once. Sequence names must be unique. please correct.\n");  }
816             }
817             CloseHandle(hThreadArray[i]);
818                         delete pDataArray[i];
819         }
820                                 
821     #endif      
822         
823         for (int i = 0; i < processIDS.size(); i++) {
824                         m->appendFiles((outputFasta + toString(processIDS[i]) + ".temp"), outputFasta);
825                         m->mothurRemove((outputFasta + toString(processIDS[i]) + ".temp"));
826                         
827                         m->appendFiles((outputScrapFasta + toString(processIDS[i]) + ".temp"), outputScrapFasta);
828                         m->mothurRemove((outputScrapFasta + toString(processIDS[i]) + ".temp"));
829                         
830             if (files[processors-1][1] != "") {
831                 m->appendFiles((outputScrapQual + toString(processIDS[i]) + ".temp"), outputScrapQual);
832                 m->mothurRemove((outputScrapQual + toString(processIDS[i]) + ".temp"));
833                 
834                 m->appendFiles((outputQual + toString(processIDS[i]) + ".temp"), outputQual);
835                 m->mothurRemove((outputQual + toString(processIDS[i]) + ".temp"));
836             }
837             
838             m->appendFiles((outputMisMatches + toString(processIDS[i]) + ".temp"), outputMisMatches);
839                         m->mothurRemove((outputMisMatches + toString(processIDS[i]) + ".temp"));
840             
841             if(allFiles){
842                                 for(int j=0;j<fastaFileNames.size();j++){
843                                         for(int k=0;k<fastaFileNames[j].size();k++){
844                                                 if (fastaFileNames[j][k] != "") {
845                                                         m->appendFiles((fastaFileNames[j][k] + toString(processIDS[i]) + ".temp"), fastaFileNames[j][k]);
846                                                         m->mothurRemove((fastaFileNames[j][k] + toString(processIDS[i]) + ".temp"));
847                                                         
848                             if (files[processors-1][1] != "") {
849                                 m->appendFiles((qualFileNames[j][k] + toString(processIDS[i]) + ".temp"), qualFileNames[j][k]);
850                                 m->mothurRemove((qualFileNames[j][k] + toString(processIDS[i]) + ".temp"));
851                             }
852                                                 }
853                                         }
854                                 }
855                         }
856                 }
857                 
858                 return num;
859         }
860         catch(exception& e) {
861                 m->errorOut(e, "MakeContigsCommand", "createProcesses");
862                 exit(1);
863         }
864 }
865 //**********************************************************************************************************************
866 int MakeContigsCommand::driver(vector<string> files, string outputFasta, string outputQual, string outputScrapFasta, string outputScrapQual, string outputMisMatches, vector<vector<string> > fastaFileNames, vector<vector<string> > qualFileNames){
867     try {
868         
869         Alignment* alignment;
870         if(align == "gotoh")                    {       alignment = new GotohOverlap(gapOpen, gapExtend, match, misMatch, longestBase);                 }
871                 else if(align == "needleman")   {       alignment = new NeedlemanOverlap(gapOpen, match, misMatch, longestBase);                                }
872         
873         int num = 0;
874         string thisffastafile = files[0];
875         string thisfqualfile = files[1];
876         string thisrfastafile = files[2];
877         string thisrqualfile = files[3];
878         
879         if (m->debug) {  m->mothurOut("[DEBUG]: ffasta = " + thisffastafile + ".\n[DEBUG]: fqual = " + thisfqualfile + ".\n[DEBUG]: rfasta = " + thisrfastafile + ".\n[DEBUG]: rqual = " + thisrqualfile + ".\n"); }
880         
881         ifstream inFFasta, inRFasta, inFQual, inRQual;
882         ofstream outFasta, outQual, outMisMatch, outScrapFasta, outScrapQual;
883         m->openInputFile(thisffastafile, inFFasta);
884         m->openInputFile(thisrfastafile, inRFasta);
885         if (thisfqualfile != "") {
886             m->openInputFile(thisfqualfile, inFQual);
887             m->openInputFile(thisrqualfile, inRQual);
888             m->openOutputFile(outputScrapQual, outScrapQual);
889             m->openOutputFile(outputQual, outQual);
890         }
891         m->openOutputFile(outputFasta, outFasta);
892         m->openOutputFile(outputScrapFasta, outScrapFasta);
893         m->openOutputFile(outputMisMatches, outMisMatch);
894         outMisMatch << "Name\tLength\tMisMatches\n";
895         
896         TrimOligos trimOligos(pdiffs, bdiffs, 0, 0, primers, barcodes);
897         
898         while ((!inFFasta.eof()) && (!inRFasta.eof())) {
899             
900             if (m->control_pressed) { break; }
901             
902             int success = 1;
903             string trashCode = "";
904             int currentSeqsDiffs = 0;
905
906             //read seqs and quality info
907             Sequence fSeq(inFFasta); m->gobble(inFFasta);
908             Sequence rSeq(inRFasta); m->gobble(inRFasta);
909             QualityScores* fQual = NULL; QualityScores* rQual = NULL;
910             if (thisfqualfile != "") {
911                 fQual = new QualityScores(inFQual); m->gobble(inFQual);
912                 rQual = new QualityScores(inRQual); m->gobble(inRQual);
913             }
914             
915             int barcodeIndex = 0;
916             int primerIndex = 0;
917             
918             if(barcodes.size() != 0){
919                 if (thisfqualfile != "") {
920                     success = trimOligos.stripBarcode(fSeq, rSeq, *fQual, *rQual, barcodeIndex);
921                 }else {
922                     success = trimOligos.stripBarcode(fSeq, rSeq, barcodeIndex);
923                 }
924                 if(success > bdiffs)            {       trashCode += 'b';       }
925                 else{ currentSeqsDiffs += success;  }
926             }
927             
928             if(primers.size() != 0){
929                 if (thisfqualfile != "") {
930                     success = trimOligos.stripForward(fSeq, rSeq, *fQual, *rQual, primerIndex);
931                 }else {
932                     success = trimOligos.stripForward(fSeq, rSeq, primerIndex);
933                 }
934                 if(success > pdiffs)            {       trashCode += 'f';       }
935                 else{ currentSeqsDiffs += success;  }
936             }
937             
938             if (currentSeqsDiffs > tdiffs)      {       trashCode += 't';   }
939             
940             //flip the reverse reads
941             rSeq.reverseComplement();
942             if (thisfqualfile != "") { rQual->flipQScores(); }
943
944             //pairwise align
945             alignment->align(fSeq.getUnaligned(), rSeq.getUnaligned());
946             map<int, int> ABaseMap = alignment->getSeqAAlnBaseMap();
947             map<int, int> BBaseMap = alignment->getSeqBAlnBaseMap();
948             fSeq.setAligned(alignment->getSeqAAln());
949             rSeq.setAligned(alignment->getSeqBAln());
950             int length = fSeq.getAligned().length();
951             
952             //traverse alignments merging into one contiguous seq
953             string contig = "";
954             vector<int> contigScores; 
955             int numMismatches = 0;
956             string seq1 = fSeq.getAligned();
957             string seq2 = rSeq.getAligned();
958             vector<int> scores1, scores2; 
959             if (thisfqualfile != "") {
960                 scores1 = fQual->getQualityScores();
961                 scores2 = rQual->getQualityScores();
962                 delete fQual; delete rQual;
963             }
964             
965             // if (num < 5) {  cout << fSeq.getStartPos() << '\t' << fSeq.getEndPos() << '\t' << rSeq.getStartPos() << '\t' << rSeq.getEndPos() << endl; }
966             int overlapStart = fSeq.getStartPos();
967             int seq2Start = rSeq.getStartPos();
968             //bigger of the 2 starting positions is the location of the overlapping start
969             if (overlapStart < seq2Start) { //seq2 starts later so take from 0 to seq2Start from seq1
970                 overlapStart = seq2Start; 
971                 for (int i = 0; i < overlapStart; i++) {
972                     contig += seq1[i];
973                     if (thisfqualfile != "") { contigScores.push_back(scores1[ABaseMap[i]]); }
974                 }
975             }else { //seq1 starts later so take from 0 to overlapStart from seq2
976                 for (int i = 0; i < overlapStart; i++) {
977                     contig += seq2[i];
978                     if (thisfqualfile != "") { contigScores.push_back(scores2[BBaseMap[i]]); }
979                 }
980             }
981             
982             int seq1End = fSeq.getEndPos();
983             int seq2End = rSeq.getEndPos();
984             int overlapEnd = seq1End;
985             if (seq2End < overlapEnd) { overlapEnd = seq2End; }  //smallest end position is where overlapping ends
986             
987             for (int i = overlapStart; i < overlapEnd; i++) {
988                 if (seq1[i] == seq2[i]) { //match, add base and choose highest score
989                     contig += seq1[i];
990                     if (thisfqualfile != "") { 
991                         contigScores.push_back(scores1[ABaseMap[i]]); 
992                         if (scores1[ABaseMap[i]] < scores2[BBaseMap[i]]) { contigScores[contigScores.size()-1] = scores2[BBaseMap[i]]; }
993                     }
994                 }else if (((seq1[i] == '.') || (seq1[i] == '-')) && ((seq2[i] != '-') && (seq2[i] != '.'))) { //seq1 is a gap and seq2 is a base, choose seq2, unless quality score for base is below threshold. In that case eliminate base
995                     if (thisfqualfile != "") {
996                         if (scores2[BBaseMap[i]] < threshold) { } //
997                         else {
998                             contig += seq2[i];
999                             contigScores.push_back(scores2[BBaseMap[i]]);
1000                         }
1001                     }else { contig += seq2[i]; } //with no quality info, then we keep it?
1002                 }else if (((seq2[i] == '.') || (seq2[i] == '-')) && ((seq1[i] != '-') && (seq1[i] != '.'))) { //seq2 is a gap and seq1 is a base, choose seq1, unless quality score for base is below threshold. In that case eliminate base
1003                     if (thisfqualfile != "") {
1004                         if (scores1[ABaseMap[i]] < threshold) { } //
1005                         else {
1006                             contig += seq1[i];
1007                             contigScores.push_back(scores1[ABaseMap[i]]);
1008                         }
1009                     }else { contig += seq1[i]; } //with no quality info, then we keep it?
1010                 }else if (((seq1[i] != '-') && (seq1[i] != '.')) && ((seq2[i] != '-') && (seq2[i] != '.'))) { //both bases choose one with better quality
1011                     if (thisfqualfile != "") {
1012                         char c = seq1[i];
1013                         contigScores.push_back(scores1[ABaseMap[i]]);
1014                         if (scores1[ABaseMap[i]] < scores2[BBaseMap[i]]) { contigScores[contigScores.size()-1] = scores2[BBaseMap[i]]; c = seq2[i]; }
1015                         contig += c;
1016                         numMismatches++;
1017                     }else { numMismatches++; } //cant decide, so eliminate and mark as mismatch
1018                 }else { //should never get here
1019                     m->mothurOut("[ERROR]: case I didn't think of seq1 = " + toString(seq1[i]) + " and seq2 = " + toString(seq2[i]) + "\n");
1020                 }
1021             }
1022             
1023             if (seq1End < seq2End) { //seq1 ends before seq2 so take from overlap to length from seq2
1024                 for (int i = overlapEnd; i < length; i++) {
1025                     contig += seq2[i];
1026                     if (thisfqualfile != "") { contigScores.push_back(scores2[BBaseMap[i]]); }
1027                 }
1028             }else { //seq2 ends before seq1 so take from overlap to length from seq1
1029                 for (int i = overlapEnd; i < length; i++) {
1030                     contig += seq1[i];
1031                     if (thisfqualfile != "") { contigScores.push_back(scores1[ABaseMap[i]]); }
1032                 }
1033                 
1034             }
1035
1036             if(trashCode.length() == 0){
1037                 if (createGroup) {
1038                     if(barcodes.size() != 0){
1039                         string thisGroup = barcodeNameVector[barcodeIndex];
1040                         if (primers.size() != 0) { 
1041                             if (primerNameVector[primerIndex] != "") { 
1042                                 if(thisGroup != "") {
1043                                     thisGroup += "." + primerNameVector[primerIndex]; 
1044                                 }else {
1045                                     thisGroup = primerNameVector[primerIndex]; 
1046                                 }
1047                             } 
1048                         }
1049                         
1050                         if (m->debug) { m->mothurOut(", group= " + thisGroup + "\n"); }
1051                         
1052                         groupMap[fSeq.getName()] = thisGroup; 
1053                         
1054                         map<string, int>::iterator it = groupCounts.find(thisGroup);
1055                         if (it == groupCounts.end()) {  groupCounts[thisGroup] = 1; }
1056                         else { groupCounts[it->first] ++; }
1057                         
1058                     }
1059                 }
1060                 
1061                 if(allFiles){
1062                     ofstream output;
1063                     m->openOutputFileAppend(fastaFileNames[barcodeIndex][primerIndex], output);
1064                     output << ">" << fSeq.getName() << endl << contig << endl;
1065                     output.close();
1066                     
1067                     if (thisfqualfile != "") {
1068                         m->openOutputFileAppend(qualFileNames[barcodeIndex][primerIndex], output);
1069                         output << ">" << fSeq.getName() << endl;
1070                         for (int i = 0; i < contigScores.size(); i++) { output << contigScores[i] << ' '; }
1071                         output << endl;
1072                         output.close(); 
1073                     }
1074                 }
1075                 
1076                 //output
1077                 outFasta << ">" << fSeq.getName() << endl << contig << endl;
1078                 if (thisfqualfile != "") {
1079                     outQual << ">" << fSeq.getName() << endl;
1080                     for (int i = 0; i < contigScores.size(); i++) { outQual << contigScores[i] << ' '; }
1081                     outQual << endl;
1082                 }
1083                 outMisMatch << fSeq.getName() << '\t' << contig.length() << '\t' << numMismatches << endl;
1084             }else {
1085                 //output
1086                 outScrapFasta << ">" << fSeq.getName() << " | " << trashCode << endl << contig << endl;
1087                 if (thisfqualfile != "") {
1088                     outScrapQual << ">" << fSeq.getName() << " | " << trashCode << endl;
1089                     for (int i = 0; i < contigScores.size(); i++) { outScrapQual << contigScores[i] << ' '; }
1090                     outScrapQual << endl;
1091                 }
1092             }
1093             num++;
1094             
1095                         //report progress
1096                         if((num) % 1000 == 0){  m->mothurOut(toString(num)); m->mothurOutEndLine();             }
1097                 }
1098         
1099                 //report progress
1100                 if((num) % 1000 != 0){  m->mothurOut(toString(num)); m->mothurOutEndLine();             }
1101         
1102         inFFasta.close();
1103         inRFasta.close();
1104         outFasta.close();
1105         outScrapFasta.close();
1106         outMisMatch.close();
1107         if (thisfqualfile != "") {
1108             inFQual.close();
1109             inRQual.close();
1110             outQual.close();
1111             outScrapQual.close();
1112         }
1113         delete alignment;
1114         
1115         if (m->control_pressed) {  m->mothurRemove(outputFasta); m->mothurRemove(outputScrapFasta);m->mothurRemove(outputMisMatches); if (thisfqualfile != "") { m->mothurRemove(outputQual); m->mothurRemove(outputScrapQual); } }
1116     
1117         return num;
1118     }
1119         catch(exception& e) {
1120                 m->errorOut(e, "MakeContigsCommand", "driver");
1121                 exit(1);
1122         }
1123 }
1124 //**********************************************************************************************************************
1125 vector< vector<string> > MakeContigsCommand::readFastqFiles(unsigned long int& count, string ffastq, string rfastq){
1126     try {
1127         vector< vector<string> > files;
1128         //maps processors number to file pointer
1129         map<int, vector<ofstream*> > tempfiles;  //tempfiles[0] = forwardFasta, [1] = forwardQual, [2] = reverseFasta, [3] = reverseQual
1130         map<int, vector<ofstream*> >::iterator it;
1131         
1132         //create files to write to
1133         for (int i = 0; i < processors; i++) {
1134             vector<ofstream*> temp;
1135             ofstream* outFF = new ofstream;     temp.push_back(outFF);
1136             ofstream* outFQ = new ofstream;     temp.push_back(outFQ);
1137             ofstream* outRF = new ofstream;     temp.push_back(outRF);
1138             ofstream* outRQ = new ofstream;     temp.push_back(outRQ);
1139             tempfiles[i] = temp;
1140             
1141             vector<string> names;
1142             string thisOutputDir = outputDir;
1143             if (outputDir == "") { thisOutputDir = m->hasPath(ffastq); }
1144             string ffastafilename = thisOutputDir + m->getRootName(m->getSimpleName(ffastq)) + toString(i) + "ffastatemp";
1145             string rfastafilename = thisOutputDir + m->getRootName(m->getSimpleName(rfastq)) + toString(i) + "rfastatemp";
1146             string fqualfilename = thisOutputDir + m->getRootName(m->getSimpleName(ffastq)) + toString(i) + "fqualtemp";
1147             string rqualfilename = thisOutputDir + m->getRootName(m->getSimpleName(rfastq)) + toString(i) + "rqualtemp";
1148             names.push_back(ffastafilename); names.push_back(fqualfilename);
1149             names.push_back(rfastafilename); names.push_back(rqualfilename);
1150             files.push_back(names);
1151             
1152             m->openOutputFile(ffastafilename, *outFF);
1153             m->openOutputFile(rfastafilename, *outRF);
1154             m->openOutputFile(fqualfilename, *outFQ);
1155             m->openOutputFile(rqualfilename, *outRQ);
1156         }
1157         
1158         if (m->control_pressed) {
1159             //close files, delete ofstreams
1160             for (it = tempfiles.begin(); it!=tempfiles.end(); it++) { for (int i = 0; i < (it->second).size(); i++) { (*(it->second)[i]).close();  delete (it->second)[i]; } }
1161             //remove files
1162             for (int i = 0; i < files.size(); i++) {  
1163                 for(int j = 0; j < files[i].size(); j++) { m->mothurRemove(files[i][j]); }
1164             }
1165         }
1166         
1167         ifstream inForward;
1168         m->openInputFile(ffastq, inForward);
1169         
1170         ifstream inReverse;
1171         m->openInputFile(rfastq, inReverse);
1172         
1173         count = 0;
1174         map<string, fastqRead> uniques;
1175         map<string, fastqRead>::iterator itUniques;
1176         while ((!inForward.eof()) || (!inReverse.eof())) {
1177             
1178             if (m->control_pressed) { for (it = tempfiles.begin(); it!=tempfiles.end(); it++) { for (int i = 0; i < (it->second).size(); i++) { (*(it->second)[i]).close();  delete (it->second)[i]; } } for (int i = 0; i < files.size(); i++) {  for(int j = 0; j < files[i].size(); j++) { m->mothurRemove(files[i][j]); } } inForward.close(); inReverse.close(); return files; }
1179             
1180             //get a read from forward and reverse fastq files
1181             bool ignoref, ignorer;
1182             fastqRead thisFread, thisRread;
1183             if (!inForward.eof()) {  thisFread = readFastq(inForward, ignoref); }
1184             else { ignoref = true; }
1185             if (!inReverse.eof()) { thisRread = readFastq(inReverse, ignorer);  }
1186             else { ignorer = true; }
1187             
1188             vector<pairFastqRead> reads = getReads(ignoref, ignorer, thisFread, thisRread, uniques);
1189            
1190             for (int i = 0; i < reads.size(); i++) {
1191                 fastqRead fread = reads[i].forward;
1192                 fastqRead rread = reads[i].reverse;
1193                 
1194                 if (m->debug) { m->mothurOut(toString(count) + '\t' + fread.name + '\t' + rread.name + '\n'); }
1195                
1196                 if (checkReads(fread, rread, ffastq, rfastq)) {
1197                     if (m->control_pressed) { for (it = tempfiles.begin(); it!=tempfiles.end(); it++) { for (int i = 0; i < (it->second).size(); i++) { (*(it->second)[i]).close();  delete (it->second)[i]; } } for (int i = 0; i < files.size(); i++) {  for(int j = 0; j < files[i].size(); j++) { m->mothurRemove(files[i][j]); } } inForward.close(); inReverse.close(); return files; }
1198                     
1199                     //if the reads are okay write to output files
1200                     int process = count % processors;
1201                     
1202                     *(tempfiles[process][0]) << ">" << fread.name << endl << fread.sequence << endl;
1203                     *(tempfiles[process][1]) << ">" << fread.name << endl;
1204                     for (int i = 0; i < fread.scores.size(); i++) { *(tempfiles[process][1]) << fread.scores[i] << " "; }
1205                     *(tempfiles[process][1]) << endl;
1206                     *(tempfiles[process][2]) << ">" << rread.name << endl << rread.sequence << endl;
1207                     *(tempfiles[process][3]) << ">" << rread.name << endl;
1208                     for (int i = 0; i < rread.scores.size(); i++) { *(tempfiles[process][3]) << rread.scores[i] << " "; }
1209                     *(tempfiles[process][3]) << endl;
1210                     
1211                     count++;
1212                     
1213                     //report progress
1214                     if((count) % 10000 == 0){   m->mothurOut(toString(count)); m->mothurOutEndLine();           }
1215                 }
1216             }
1217                 }
1218                 //report progress
1219                 if((count) % 10000 != 0){       m->mothurOut(toString(count)); m->mothurOutEndLine();           }
1220         
1221         if (uniques.size() != 0) {
1222             for (itUniques = uniques.begin(); itUniques != uniques.end(); itUniques++) {
1223                 m->mothurOut("[WARNING]: did not find paired read for " + itUniques->first + ", ignoring.\n");
1224             }
1225             m->mothurOutEndLine();
1226         }
1227         
1228         //close files, delete ofstreams
1229         for (it = tempfiles.begin(); it!=tempfiles.end(); it++) { for (int i = 0; i < (it->second).size(); i++) { (*(it->second)[i]).close();  delete (it->second)[i]; } }
1230         inForward.close();
1231         inReverse.close();
1232         
1233         return files;
1234     }
1235     catch(exception& e) {
1236         m->errorOut(e, "MakeContigsCommand", "readFastqFiles");
1237         exit(1);
1238     }
1239 }
1240 //**********************************************************************************************************************
1241 vector< vector<string> > MakeContigsCommand::readFastaFiles(unsigned long int& count, string ffasta, string rfasta){
1242     try {
1243         vector< vector<string> > files;
1244         //maps processors number to file pointer
1245         map<int, vector<ofstream*> > tempfiles;  //tempfiles[0] = forwardFasta, [1] = forwardQual, [2] = reverseFasta, [3] = reverseQual
1246         map<int, vector<ofstream*> >::iterator it;
1247         
1248         //create files to write to
1249         for (int i = 0; i < processors; i++) {
1250             vector<ofstream*> temp;
1251             ofstream* outFF = new ofstream;     temp.push_back(outFF);
1252             ofstream* outFQ = new ofstream;     temp.push_back(outFQ);
1253             ofstream* outRF = new ofstream;     temp.push_back(outRF);
1254             ofstream* outRQ = new ofstream;     temp.push_back(outRQ);
1255             tempfiles[i] = temp;
1256             
1257             vector<string> names;
1258             string thisOutputDir = outputDir;
1259             if (outputDir == "") { thisOutputDir = m->hasPath(ffasta); }
1260             string ffastafilename = thisOutputDir + m->getRootName(m->getSimpleName(ffasta)) + toString(i) + "ffastatemp";
1261             string rfastafilename = thisOutputDir + m->getRootName(m->getSimpleName(rfasta)) + toString(i) + "rfastatemp";
1262             string fqualfilename = "";
1263             if (fqualfile != "") { fqualfilename = thisOutputDir + m->getRootName(m->getSimpleName(fqualfile)) + toString(i) + "fqual.temp";  m->openOutputFile(fqualfilename, *outFQ); }
1264             string rqualfilename = "";
1265             if (rqualfile != "") { rqualfilename = thisOutputDir + m->getRootName(m->getSimpleName(rqualfile)) + toString(i) + "rqual.temp"; m->openOutputFile(rqualfilename, *outRQ); }
1266             names.push_back(ffastafilename); names.push_back(fqualfilename);
1267             names.push_back(rfastafilename); names.push_back(rqualfilename);
1268             files.push_back(names);
1269             
1270             m->openOutputFile(ffastafilename, *outFF);
1271             m->openOutputFile(rfastafilename, *outRF);
1272         }
1273         
1274         if (m->control_pressed) {
1275             //close files, delete ofstreams
1276             for (it = tempfiles.begin(); it!=tempfiles.end(); it++) { for (int i = 0; i < (it->second).size(); i++) { (*(it->second)[i]).close();  delete (it->second)[i]; } }
1277             //remove files
1278             for (int i = 0; i < files.size(); i++) {  
1279                 for(int j = 0; j < files[i].size(); j++) { m->mothurRemove(files[i][j]); }
1280             }
1281         }
1282         
1283         ifstream inForwardFasta;
1284         m->openInputFile(ffasta, inForwardFasta);
1285         
1286         ifstream inReverseFasta;
1287         m->openInputFile(rfasta, inReverseFasta);
1288         
1289         ifstream inForwardQual, inReverseQual;
1290         if (fqualfile != "") { m->openInputFile(fqualfile, inForwardQual); m->openInputFile(rqualfile, inReverseQual); }
1291         
1292         count = 0;
1293         map<string, fastqRead> uniques;
1294         map<string, fastqRead>::iterator itUniques;
1295         while ((!inForwardFasta.eof()) || (!inReverseFasta.eof())) {
1296             
1297             if (m->control_pressed) { for (it = tempfiles.begin(); it!=tempfiles.end(); it++) { for (int i = 0; i < (it->second).size(); i++) { (*(it->second)[i]).close();  delete (it->second)[i]; } } for (int i = 0; i < files.size(); i++) {  for(int j = 0; j < files[i].size(); j++) { m->mothurRemove(files[i][j]); } } inReverseFasta.close(); inForwardFasta.close(); if (fqualfile != "") { inReverseQual.close(); inReverseQual.close(); } return files; }
1298             
1299             //get a reads from forward and reverse fasta files
1300             bool ignoref, ignorer;
1301             fastqRead thisFread, thisRread;
1302             if (!inForwardFasta.eof()) {  
1303                 ignoref = false; 
1304                 Sequence temp(inForwardFasta); m->gobble(inForwardFasta);
1305                 thisFread.name = temp.getName();
1306                 thisFread.sequence = temp.getUnaligned();
1307             }else { ignoref = true; }
1308             if (!inReverseFasta.eof()) {  
1309                 ignorer = false; 
1310                 Sequence temp(inReverseFasta); m->gobble(inReverseFasta);
1311                 thisRread.name = temp.getName();
1312                 thisRread.sequence = temp.getUnaligned();  
1313             }else { ignorer = true; }
1314             
1315             //get qual reads if given
1316             if (fqualfile != "") {
1317                 if (!inForwardQual.eof() && !ignoref) {  
1318                     QualityScores temp(inForwardQual); m->gobble(inForwardQual);
1319                     //if forward files dont match ignore read
1320                     if (thisFread.name != temp.getName()) { ignoref = true; } 
1321                     else { thisFread.scores = temp.getQualityScores(); }
1322                 }else { ignoref = true; }
1323                 if (!inReverseQual.eof() && !ignorer) {  
1324                     QualityScores temp(inReverseQual); m->gobble(inReverseQual);
1325                     //if reverse files dont match ignore read
1326                     if (thisRread.name != temp.getName()) { ignorer = true; } 
1327                     else { thisRread.scores = temp.getQualityScores(); }
1328                 }else { ignorer = true; }
1329             }
1330             
1331             vector<pairFastqRead> reads = getReads(ignoref, ignorer, thisFread, thisRread, uniques);
1332             
1333             for (int i = 0; i < reads.size(); i++) {
1334                 fastqRead fread = reads[i].forward;
1335                 fastqRead rread = reads[i].reverse;
1336                 
1337                 if (m->debug) { m->mothurOut(toString(count) + '\t' + fread.name + '\t' + rread.name + '\n'); }
1338                 
1339                 if (checkReads(fread, rread, ffasta, rfasta)) {
1340                     if (m->control_pressed) { for (it = tempfiles.begin(); it!=tempfiles.end(); it++) { for (int i = 0; i < (it->second).size(); i++) { (*(it->second)[i]).close();  delete (it->second)[i]; } } for (int i = 0; i < files.size(); i++) {  for(int j = 0; j < files[i].size(); j++) { m->mothurRemove(files[i][j]); } } inReverseFasta.close(); inForwardFasta.close(); if (fqualfile != "") { inReverseQual.close(); inReverseQual.close(); } return files; }
1341                     
1342                     //if the reads are okay write to output files
1343                     int process = count % processors;
1344                     
1345                     *(tempfiles[process][0]) << ">" << fread.name << endl << fread.sequence << endl;
1346                     *(tempfiles[process][2]) << ">" << rread.name << endl << rread.sequence << endl;
1347                     if (fqualfile != "") { //if you have quality info, print it
1348                         *(tempfiles[process][1]) << ">" << fread.name << endl;
1349                         for (int i = 0; i < fread.scores.size(); i++) { *(tempfiles[process][1]) << fread.scores[i] << " "; }
1350                         *(tempfiles[process][1]) << endl;
1351                         *(tempfiles[process][3]) << ">" << rread.name << endl;
1352                         for (int i = 0; i < rread.scores.size(); i++) { *(tempfiles[process][3]) << rread.scores[i] << " "; }
1353                         *(tempfiles[process][3]) << endl;
1354                     }
1355                     count++;
1356                     
1357                     //report progress
1358                     if((count) % 10000 == 0){   m->mothurOut(toString(count)); m->mothurOutEndLine();           }
1359                 }
1360             }
1361                 }
1362                 //report progress
1363                 if((count) % 10000 != 0){       m->mothurOut(toString(count)); m->mothurOutEndLine();           }
1364         
1365         if (uniques.size() != 0) {
1366             for (itUniques = uniques.begin(); itUniques != uniques.end(); itUniques++) {
1367                 m->mothurOut("[WARNING]: did not find paired read for " + itUniques->first + ", ignoring.\n");
1368             }
1369             m->mothurOutEndLine();
1370         }
1371         
1372         //close files, delete ofstreams
1373         for (it = tempfiles.begin(); it!=tempfiles.end(); it++) { for (int i = 0; i < (it->second).size(); i++) { (*(it->second)[i]).close();  delete (it->second)[i]; } }
1374         inReverseFasta.close(); 
1375         inForwardFasta.close(); 
1376         if (fqualfile != "") { inReverseQual.close(); inReverseQual.close(); }
1377         
1378         return files;
1379     }
1380     catch(exception& e) {
1381         m->errorOut(e, "MakeContigsCommand", "readFastaFiles");
1382         exit(1);
1383     }
1384 }
1385 //**********************************************************************************************************************
1386 vector<pairFastqRead> MakeContigsCommand::getReads(bool ignoref, bool ignorer, fastqRead forward, fastqRead reverse, map<string, fastqRead>& uniques){
1387     try {
1388         vector<pairFastqRead> reads;
1389         map<string, fastqRead>::iterator itUniques;
1390             
1391         if (!ignoref && !ignorer) {
1392             if (forward.name == reverse.name) { 
1393                 pairFastqRead temp(forward, reverse);
1394                 reads.push_back(temp);
1395             }else {
1396                 bool match = false;
1397                 //if no match are the names only different by 1 and 2?
1398                 string tempFRead = forward.name.substr(0, forward.name.length()-1);
1399                 string tempRRead = reverse.name.substr(0, reverse.name.length()-1);
1400                 if (tempFRead == tempRRead) {
1401                     if ((forward.name[forward.name.length()-1] == '1') && (reverse.name[reverse.name.length()-1] == '2')) {
1402                         forward.name = tempFRead;
1403                         reverse.name = tempRRead;
1404                         pairFastqRead temp(forward, reverse);
1405                         reads.push_back(temp);
1406                         match = true;
1407                     }
1408                 }
1409                 
1410                 if (!match) {
1411                     //look for forward pair
1412                     itUniques = uniques.find(forward.name);
1413                     if (itUniques != uniques.end()) {  //we have the pair for this read
1414                         pairFastqRead temp(forward, itUniques->second);
1415                         reads.push_back(temp);
1416                         uniques.erase(itUniques);
1417                     }else { //save this read for later
1418                         uniques[forward.name] = forward;
1419                     }
1420                     
1421                     //look for reverse pair
1422                     itUniques = uniques.find(reverse.name);
1423                     if (itUniques != uniques.end()) {  //we have the pair for this read
1424                         pairFastqRead temp(itUniques->second, reverse);
1425                         reads.push_back(temp);
1426                         uniques.erase(itUniques);
1427                     }else { //save this read for later
1428                         uniques[reverse.name] = reverse;
1429                     }
1430                 }
1431                                 
1432             }
1433         }else if (!ignoref && ignorer) { //ignore reverse keep forward
1434             //look for forward pair
1435             itUniques = uniques.find(forward.name);
1436             if (itUniques != uniques.end()) {  //we have the pair for this read
1437                 pairFastqRead temp(forward, itUniques->second);
1438                 reads.push_back(temp);
1439                 uniques.erase(itUniques);
1440             }else { //save this read for later
1441                 uniques[forward.name] = forward;
1442             }
1443
1444         }else if (ignoref && !ignorer) { //ignore forward keep reverse
1445             //look for reverse pair
1446             itUniques = uniques.find(reverse.name);
1447             if (itUniques != uniques.end()) {  //we have the pair for this read
1448                 pairFastqRead temp(itUniques->second, reverse);
1449                 reads.push_back(temp);
1450                 uniques.erase(itUniques);
1451             }else { //save this read for later
1452                 uniques[reverse.name] = reverse;
1453             }
1454         }//else ignore both and do nothing
1455         
1456         return reads;
1457     }
1458     catch(exception& e) {
1459         m->errorOut(e, "MakeContigsCommand", "readFastqFiles");
1460         exit(1);
1461     }
1462 }
1463 //**********************************************************************************************************************
1464 fastqRead MakeContigsCommand::readFastq(ifstream& in, bool& ignore){
1465     try {
1466         fastqRead read;
1467         
1468         ignore = false;
1469         
1470         //read sequence name
1471         string line = m->getline(in); m->gobble(in);
1472         vector<string> pieces = m->splitWhiteSpace(line);
1473         string name = "";  if (pieces.size() != 0) { name = pieces[0]; }
1474         if (name == "") {  m->mothurOut("[WARNING]: Blank fasta name, ignoring read."); m->mothurOutEndLine(); ignore=true;  }
1475         else if (name[0] != '@') { m->mothurOut("[WARNING]: reading " + name + " expected a name with @ as a leading character, ignoring read."); m->mothurOutEndLine(); ignore=true; }
1476         else { name = name.substr(1); }
1477         
1478         //read sequence
1479         string sequence = m->getline(in); m->gobble(in);
1480         if (sequence == "") {  m->mothurOut("[WARNING]: missing sequence for " + name + ", ignoring."); ignore=true; }
1481         
1482         //read sequence name
1483         line = m->getline(in); m->gobble(in);
1484         pieces = m->splitWhiteSpace(line);
1485         string name2 = "";  if (pieces.size() != 0) { name2 = pieces[0]; }
1486         if (name2 == "") {  m->mothurOut("[WARNING]: expected a name with + as a leading character, ignoring."); ignore=true; }
1487         else if (name2[0] != '+') { m->mothurOut("[WARNING]: reading " + name2 + " expected a name with + as a leading character, ignoring."); ignore=true; }
1488         else { name2 = name2.substr(1); if (name2 == "") { name2 = name; } }
1489         
1490         //read quality scores
1491         string quality = m->getline(in); m->gobble(in);
1492         if (quality == "") {  m->mothurOut("[WARNING]: missing quality for " + name2 + ", ignoring."); ignore=true; }
1493          
1494         //sanity check sequence length and number of quality scores match
1495         if (name2 != "") { if (name != name2) { m->mothurOut("[WARNING]: names do not match. read " + name + " for fasta and " + name2 + " for quality, ignoring."); ignore=true; } }
1496         if (quality.length() != sequence.length()) { m->mothurOut("[WARNING]: Lengths do not match for sequence " + name + ". Read " + toString(sequence.length()) + " characters for fasta and " + toString(quality.length()) + " characters for quality scores, ignoring read."); ignore=true; }
1497         
1498         vector<int> qualScores;
1499                 int controlChar = int('!');
1500                 for (int i = 0; i < quality.length(); i++) { 
1501                         int temp = int(quality[i]);
1502                         temp -= controlChar;
1503                         
1504                         qualScores.push_back(temp);
1505                 }
1506     
1507         read.name = name;
1508         read.sequence = sequence;
1509         read.scores = qualScores;
1510
1511         return read;
1512     }
1513     catch(exception& e) {
1514         m->errorOut(e, "MakeContigsCommand", "readFastq");
1515         exit(1);
1516     }
1517 }
1518 //**********************************************************************************************************************
1519 bool MakeContigsCommand::checkReads(fastqRead& forward, fastqRead& reverse, string ffile, string rfile){
1520     try {
1521         bool good = true;
1522         
1523         //do sequence lengths match
1524         if (forward.sequence.length() != reverse.sequence.length()) {
1525             m->mothurOut("[WARNING]: For sequence " + forward.name + " I read a sequence of length " + toString(forward.sequence.length()) + " from " + ffile + ", but read a sequence of length " + toString(reverse.sequence.length()) + " from " + rfile + ", ignoring.\n");
1526             good = false; 
1527         }
1528         
1529         //do number of qual scores match 
1530         if (forward.scores.size() != reverse.scores.size()) {
1531             m->mothurOut("[WARNING]: For sequence " + forward.name + " I read " + toString(forward.scores.size()) + " quality scores from " + ffile + ", but read  " + toString(reverse.scores.size()) + " quality scores from " + rfile + ", ignoring.\n");
1532             good = false; 
1533         }
1534
1535         return good;
1536     }
1537     catch(exception& e) {
1538         m->errorOut(e, "MakeContigsCommand", "checkReads");
1539         exit(1);
1540     }
1541 }
1542 //***************************************************************************************************************
1543 vector< vector<string> > MakeContigsCommand::readFileNames(string filename){
1544         try {
1545         vector< vector<string> > files;
1546         string forward, reverse;
1547         
1548         ifstream in;
1549         m->openInputFile(filename, in);
1550         
1551         while(!in.eof()) {
1552             
1553             if (m->control_pressed) { return files; }
1554             
1555             in >> forward; m->gobble(in);
1556             in >> reverse; m->gobble(in);
1557             
1558             //check to make sure both are able to be opened
1559             ifstream in2;
1560             int openForward = m->openInputFile(forward, in2, "noerror");
1561             
1562             //if you can't open it, try default location
1563             if (openForward == 1) {
1564                 if (m->getDefaultPath() != "") { //default path is set
1565                     string tryPath = m->getDefaultPath() + m->getSimpleName(forward);
1566                     m->mothurOut("Unable to open " + forward + ". Trying default " + tryPath); m->mothurOutEndLine();
1567                     ifstream in3;
1568                     openForward = m->openInputFile(tryPath, in3, "noerror");
1569                     in3.close();
1570                     forward = tryPath;
1571                 }
1572             }
1573             
1574             //if you can't open it, try output location
1575             if (openForward == 1) {
1576                 if (m->getOutputDir() != "") { //default path is set
1577                     string tryPath = m->getOutputDir() + m->getSimpleName(forward);
1578                     m->mothurOut("Unable to open " + forward + ". Trying output directory " + tryPath); m->mothurOutEndLine();
1579                     ifstream in4;
1580                     openForward = m->openInputFile(tryPath, in4, "noerror");
1581                     forward = tryPath;
1582                     in4.close();
1583                 }
1584             }
1585             
1586             if (openForward == 1) { //can't find it
1587                 m->mothurOut("[WARNING]: can't find " + forward + ", ignoring pair.\n"); 
1588             }else{  in2.close();  }
1589             
1590             ifstream in3;
1591             int openReverse = m->openInputFile(reverse, in3, "noerror");
1592             
1593             //if you can't open it, try default location
1594             if (openReverse == 1) {
1595                 if (m->getDefaultPath() != "") { //default path is set
1596                     string tryPath = m->getDefaultPath() + m->getSimpleName(reverse);
1597                     m->mothurOut("Unable to open " + reverse + ". Trying default " + tryPath); m->mothurOutEndLine();
1598                     ifstream in3;
1599                     openReverse = m->openInputFile(tryPath, in3, "noerror");
1600                     in3.close();
1601                     reverse = tryPath;
1602                 }
1603             }
1604             
1605             //if you can't open it, try output location
1606             if (openReverse == 1) {
1607                 if (m->getOutputDir() != "") { //default path is set
1608                     string tryPath = m->getOutputDir() + m->getSimpleName(reverse);
1609                     m->mothurOut("Unable to open " + reverse + ". Trying output directory " + tryPath); m->mothurOutEndLine();
1610                     ifstream in4;
1611                     openReverse = m->openInputFile(tryPath, in4, "noerror");
1612                     reverse = tryPath;
1613                     in4.close();
1614                 }
1615             }
1616             
1617             if (openReverse == 1) { //can't find it
1618                 m->mothurOut("[WARNING]: can't find " + reverse + ", ignoring pair.\n"); 
1619             }else{  in3.close();  }
1620             
1621             if ((openForward != 1) && (openReverse != 1)) { //good pair
1622                 vector<string> pair;
1623                 pair.push_back(forward);
1624                 pair.push_back(reverse);
1625                 files.push_back(pair);
1626             }
1627             
1628         }
1629         in.close();
1630         
1631         return files;
1632     }
1633     catch(exception& e) {
1634         m->errorOut(e, "MakeContigsCommand", "checkReads");
1635         exit(1);
1636     }
1637 }
1638 //***************************************************************************************************************
1639 //illumina data requires paired forward and reverse data
1640 //BARCODE   atgcatgc   atgcatgc    groupName 
1641 //PRIMER   atgcatgc   atgcatgc    groupName  
1642 //PRIMER   atgcatgc   atgcatgc  
1643 bool MakeContigsCommand::getOligos(vector<vector<string> >& fastaFileNames, vector<vector<string> >& qualFileNames, string rootname){
1644         try {
1645                 ifstream in;
1646                 m->openInputFile(oligosfile, in);
1647                 
1648                 ofstream test;
1649                 
1650                 string type, foligo, roligo, group;
1651         
1652                 int indexPrimer = 0;
1653                 int indexBarcode = 0;
1654         set<string> uniquePrimers;
1655         set<string> uniqueBarcodes;
1656                 
1657                 while(!in.eof()){
1658             
1659                         in >> type; 
1660             
1661                         if (m->debug) { m->mothurOut("[DEBUG]: reading type - " + type + ".\n"); }      
1662             
1663                         if(type[0] == '#'){
1664                                 while (!in.eof())       {       char c = in.get();  if (c == 10 || c == 13){    break;  }       } // get rest of line if there's any crap there
1665                                 m->gobble(in);
1666                         }
1667                         else{
1668                                 m->gobble(in);
1669                                 //make type case insensitive
1670                                 for(int i=0;i<type.length();i++){       type[i] = toupper(type[i]);  }
1671                                 
1672                                 in >> foligo;
1673                 
1674                 if (m->debug) { m->mothurOut("[DEBUG]: reading - " + foligo + ".\n"); }
1675                                 
1676                                 for(int i=0;i<foligo.length();i++){
1677                                         foligo[i] = toupper(foligo[i]);
1678                                         if(foligo[i] == 'U')    {       foligo[i] = 'T';        }
1679                                 }
1680                                 
1681                                 if(type == "FORWARD"){
1682                                         m->gobble(in);
1683                                         
1684                     in >> roligo;
1685                     
1686                     for(int i=0;i<roligo.length();i++){
1687                         roligo[i] = toupper(roligo[i]);
1688                         if(roligo[i] == 'U')    {       roligo[i] = 'T';        }
1689                     }
1690                     //roligo = reverseOligo(roligo);
1691                     
1692                     group = "";
1693                     
1694                                         // get rest of line in case there is a primer name
1695                                         while (!in.eof())       {       
1696                                                 char c = in.get(); 
1697                                                 if (c == 10 || c == 13){        break;  }
1698                                                 else if (c == 32 || c == 9){;} //space or tab
1699                                                 else {  group += c;  }
1700                                         } 
1701                     
1702                     oligosPair newPrimer(foligo, roligo);
1703                                         
1704                                         //check for repeat barcodes
1705                     string tempPair = foligo+roligo;
1706                     if (uniquePrimers.count(tempPair) != 0) { m->mothurOut("primer pair " + newPrimer.forward + " " + newPrimer.reverse + " is in your oligos file already."); m->mothurOutEndLine();  }
1707                     else { uniquePrimers.insert(tempPair); }
1708                                         
1709                     if (m->debug) {  if (group != "") { m->mothurOut("[DEBUG]: reading group " + group + ".\n"); }else{ m->mothurOut("[DEBUG]: no group for primer pair " + newPrimer.forward + " " + newPrimer.reverse + ".\n"); }  }
1710                     
1711                                         primers[indexPrimer]=newPrimer; indexPrimer++;          
1712                                         primerNameVector.push_back(group);
1713                                 }else if(type == "BARCODE"){
1714                                         m->gobble(in);
1715                                         
1716                     in >> roligo;
1717                     
1718                     for(int i=0;i<roligo.length();i++){
1719                         roligo[i] = toupper(roligo[i]);
1720                         if(roligo[i] == 'U')    {       roligo[i] = 'T';        }
1721                     }
1722                     //roligo = reverseOligo(roligo);
1723                     
1724                     oligosPair newPair(foligo, roligo);
1725                     
1726                     group = "";
1727                     while (!in.eof())   {       
1728                                                 char c = in.get(); 
1729                                                 if (c == 10 || c == 13){        break;  }
1730                                                 else if (c == 32 || c == 9){;} //space or tab
1731                                                 else {  group += c;  }
1732                                         } 
1733                                         
1734                     if (m->debug) { m->mothurOut("[DEBUG]: barcode pair " + newPair.forward + " " + newPair.reverse + ", and group = " + group + ".\n"); }
1735                         
1736                     //check for repeat barcodes
1737                     string tempPair = foligo+roligo;
1738                     if (uniqueBarcodes.count(tempPair) != 0) { m->mothurOut("barcode pair " + newPair.forward + " " + newPair.reverse +  " is in your oligos file already, disregarding."); m->mothurOutEndLine();  }
1739                     else { uniqueBarcodes.insert(tempPair); }
1740                         
1741                     barcodes[indexBarcode]=newPair; indexBarcode++;
1742                                         barcodeNameVector.push_back(group);
1743                                 }else if(type == "LINKER"){
1744                                         linker.push_back(foligo);
1745                     m->mothurOut("[WARNING]: make.contigs is not setup to remove linkers, ignoring.\n");
1746                                 }else if(type == "SPACER"){
1747                                         spacer.push_back(foligo);
1748                     m->mothurOut("[WARNING]: make.contigs is not setup to remove spacers, ignoring.\n");
1749                                 }
1750                                 else{   m->mothurOut("[WARNING]: " + type + " is not recognized as a valid type. Choices are primer, barcode, linker and spacer. Ignoring " + foligo + "."); m->mothurOutEndLine(); }
1751                         }
1752                         m->gobble(in);
1753                 }       
1754                 in.close();
1755                 
1756                 if(barcodeNameVector.size() == 0 && primerNameVector[0] == ""){ allFiles = 0;   }
1757                 
1758                 //add in potential combos
1759                 if(barcodeNameVector.size() == 0){
1760             oligosPair temp("", "");
1761                         barcodes[0] = temp;
1762                         barcodeNameVector.push_back("");                        
1763                 }
1764                 
1765                 if(primerNameVector.size() == 0){
1766             oligosPair temp("", "");
1767                         primers[0] = temp;
1768                         primerNameVector.push_back("");                 
1769                 }
1770                 
1771                 fastaFileNames.resize(barcodeNameVector.size());
1772                 for(int i=0;i<fastaFileNames.size();i++){
1773                         fastaFileNames[i].assign(primerNameVector.size(), "");
1774                 }
1775                 qualFileNames = fastaFileNames; 
1776                 
1777                 if(allFiles){
1778                         set<string> uniqueNames; //used to cleanup outputFileNames
1779                         for(map<int, oligosPair>::iterator itBar = barcodes.begin();itBar != barcodes.end();itBar++){
1780                                 for(map<int, oligosPair>::iterator itPrimer = primers.begin();itPrimer != primers.end(); itPrimer++){
1781                                         
1782                                         string primerName = primerNameVector[itPrimer->first];
1783                                         string barcodeName = barcodeNameVector[itBar->first];
1784                                         
1785                                         string comboGroupName = "";
1786                                         string fastaFileName = "";
1787                                         string qualFileName = "";
1788                                         string nameFileName = "";
1789                     string countFileName = "";
1790                                         
1791                                         if(primerName == ""){
1792                                                 comboGroupName = barcodeNameVector[itBar->first];
1793                                         }
1794                                         else{
1795                                                 if(barcodeName == ""){
1796                                                         comboGroupName = primerNameVector[itPrimer->first];
1797                                                 }
1798                                                 else{
1799                                                         comboGroupName = barcodeNameVector[itBar->first] + "." + primerNameVector[itPrimer->first];
1800                                                 }
1801                                         }
1802                                         
1803                                         
1804                                         ofstream temp;
1805                                         fastaFileName = rootname + comboGroupName + ".fasta";
1806                                         if (uniqueNames.count(fastaFileName) == 0) {
1807                                                 outputNames.push_back(fastaFileName);
1808                                                 outputTypes["fasta"].push_back(fastaFileName);
1809                                                 uniqueNames.insert(fastaFileName);
1810                                         }
1811                                         
1812                                         fastaFileNames[itBar->first][itPrimer->first] = fastaFileName;
1813                                         m->openOutputFile(fastaFileName, temp);         temp.close();
1814                                         
1815                     if ((fqualfile != "") || (ffastqfile != "") || (file != "")) {
1816                         qualFileName = rootname + ".qual";
1817                         if (uniqueNames.count(qualFileName) == 0) {
1818                             outputNames.push_back(qualFileName);
1819                             outputTypes["qfile"].push_back(qualFileName);
1820                         }
1821                                                 
1822                         qualFileNames[itBar->first][itPrimer->first] = qualFileName;
1823                         m->openOutputFile(qualFileName, temp);          temp.close();
1824                     }
1825                                 }
1826                         }
1827                 }
1828                 
1829                 bool allBlank = true;
1830                 for (int i = 0; i < barcodeNameVector.size(); i++) {
1831                         if (barcodeNameVector[i] != "") {
1832                                 allBlank = false;
1833                                 break;
1834                         }
1835                 }
1836                 for (int i = 0; i < primerNameVector.size(); i++) {
1837                         if (primerNameVector[i] != "") {
1838                                 allBlank = false;
1839                                 break;
1840                         }
1841                 }
1842         
1843                 if (allBlank) {
1844                         m->mothurOut("[WARNING]: your oligos file does not contain any group names.  mothur will not create a groupfile."); m->mothurOutEndLine();
1845                         allFiles = false;
1846                         return false;
1847                 }
1848                 
1849                 return true;
1850                 
1851         }
1852         catch(exception& e) {
1853                 m->errorOut(e, "MakeContigsCommand", "getOligos");
1854                 exit(1);
1855         }
1856 }
1857 //********************************************************************/
1858 string MakeContigsCommand::reverseOligo(string oligo){
1859         try {
1860         string reverse = "";
1861         
1862         for(int i=oligo.length()-1;i>=0;i--){
1863             
1864             if(oligo[i] == 'A')         {       reverse += 'T'; }
1865             else if(oligo[i] == 'T'){   reverse += 'A'; }
1866             else if(oligo[i] == 'U'){   reverse += 'A'; }
1867             
1868             else if(oligo[i] == 'G'){   reverse += 'C'; }
1869             else if(oligo[i] == 'C'){   reverse += 'G'; }
1870             
1871             else if(oligo[i] == 'R'){   reverse += 'Y'; }
1872             else if(oligo[i] == 'Y'){   reverse += 'R'; }
1873             
1874             else if(oligo[i] == 'M'){   reverse += 'K'; }
1875             else if(oligo[i] == 'K'){   reverse += 'M'; }
1876             
1877             else if(oligo[i] == 'W'){   reverse += 'W'; }
1878             else if(oligo[i] == 'S'){   reverse += 'S'; }
1879             
1880             else if(oligo[i] == 'B'){   reverse += 'V'; }
1881             else if(oligo[i] == 'V'){   reverse += 'B'; }
1882             
1883             else if(oligo[i] == 'D'){   reverse += 'H'; }
1884             else if(oligo[i] == 'H'){   reverse += 'D'; }
1885             
1886             else                                                {       reverse += 'N'; }
1887         }
1888         
1889         
1890         return reverse;
1891     }
1892         catch(exception& e) {
1893                 m->errorOut(e, "MakeContigsCommand", "reverseOligo");
1894                 exit(1);
1895         }
1896 }
1897 //**********************************************************************************************************************
1898
1899
1900
1901