CommandParameter poligos("oligos", "InputTypes", "", "", "none", "none", "none","group",false,false,true); parameters.push_back(poligos);
CommandParameter ppdiffs("pdiffs", "Number", "", "0", "", "", "","",false,false,true); parameters.push_back(ppdiffs);
CommandParameter pbdiffs("bdiffs", "Number", "", "0", "", "", "","",false,false,true); parameters.push_back(pbdiffs);
-// CommandParameter pldiffs("ldiffs", "Number", "", "0", "", "", "",false,false); parameters.push_back(pldiffs);
-// CommandParameter psdiffs("sdiffs", "Number", "", "0", "", "", "",false,false); parameters.push_back(psdiffs);
CommandParameter ptdiffs("tdiffs", "Number", "", "0", "", "", "","",false,false); parameters.push_back(ptdiffs);
CommandParameter palign("align", "Multiple", "needleman-gotoh", "needleman", "", "", "","",false,false); parameters.push_back(palign);
CommandParameter pmismatch("mismatch", "Number", "", "-1.0", "", "", "","",false,false); parameters.push_back(pmismatch);
CommandParameter pgapopen("gapopen", "Number", "", "-2.0", "", "", "","",false,false); parameters.push_back(pgapopen);
CommandParameter pgapextend("gapextend", "Number", "", "-1.0", "", "", "","",false,false); parameters.push_back(pgapextend);
- CommandParameter pthreshold("insert", "Number", "", "25", "", "", "","",false,false); parameters.push_back(pthreshold);
+ CommandParameter pthreshold("insert", "Number", "", "20", "", "", "","",false,false); parameters.push_back(pthreshold);
CommandParameter pdeltaq("deltaq", "Number", "", "6", "", "", "","",false,false); parameters.push_back(pdeltaq);
CommandParameter pprocessors("processors", "Number", "", "1", "", "", "","",false,false,true); parameters.push_back(pprocessors);
CommandParameter pformat("format", "Multiple", "sanger-illumina-solexa-illumina1.8+", "illumina1.8+", "", "", "","",false,false,true); parameters.push_back(pformat);
helpString += "If an oligos file is provided barcodes and primers will be trimmed, and a group file will be created.\n";
helpString += "The make.contigs command parameters are file, ffastq, rfastq, ffasta, rfasta, fqfile, rqfile, oligos, format, tdiffs, bdiffs, pdiffs, align, match, mismatch, gapopen, gapextend, insert, deltaq, allfiles and processors.\n";
helpString += "The ffastq and rfastq, file, or ffasta and rfasta parameters are required.\n";
- helpString += "The file parameter is 2 column file containing the forward fastq files in the first column and their matching reverse fastq files in the second column. Mothur will process each pair and create a combined fasta and report file with all the sequences.\n";
+ helpString += "The file parameter is 2 or 3 column file containing the forward fastq files in the first column and their matching reverse fastq files in the second column, or a groupName then forward fastq file and reverse fastq file. Mothur will process each pair and create a combined fasta and report file with all the sequences.\n";
helpString += "The ffastq and rfastq parameters are used to provide a forward fastq and reverse fastq file to process. If you provide one, you must provide the other.\n";
helpString += "The ffasta and rfasta parameters are used to provide a forward fasta and reverse fasta file to process. If you provide one, you must provide the other.\n";
helpString += "The fqfile and rqfile parameters are used to provide a forward quality and reverse quality files to process with the ffasta and rfasta parameters. If you provide one, you must provide the other.\n";
helpString += "The deltaq parameter allows you to specify the delta allowed between quality scores of a mismatched base. For example in the overlap, if deltaq=5 and in the alignment seqA, pos 200 has a quality score of 30 and the same position in seqB has a quality score of 20, you take the base from seqA (30-20 >= 5). If the quality score in seqB is 28 then the base in the consensus will be an N (30-28<5) The default is 6.\n";
helpString += "The gapopen parameter allows you to specify the penalty for opening a gap in an alignment. The default is -2.0.\n";
helpString += "The gapextend parameter allows you to specify the penalty for extending a gap in an alignment. The default is -1.0.\n";
- helpString += "The insert parameter allows you to set a quality scores threshold. In the case where we are trying to decide whether to keep a base or remove it because the base is compared to a gap in the other fragment, if the base has a quality score below the threshold we eliminate it. Default=25.\n";
+ helpString += "The insert parameter allows you to set a quality scores threshold. In the case where we are trying to decide whether to keep a base or remove it because the base is compared to a gap in the other fragment, if the base has a quality score equal to or below the threshold we eliminate it. Default=20.\n";
helpString += "The processors parameter allows you to specify how many processors you would like to use. The default is 1. \n";
helpString += "The allfiles parameter will create separate group and fasta file for each grouping. The default is F.\n";
+
helpString += "The trimoverlap parameter allows you to trim the sequences to only the overlapping section. The default is F.\n";
helpString += "The make.contigs command should be in the following format: \n";
helpString += "make.contigs(ffastq=yourForwardFastqFile, rfastq=yourReverseFastqFile, align=yourAlignmentMethod) \n";
outputTypes["fasta"] = tempOutNames;
outputTypes["group"] = tempOutNames;
outputTypes["report"] = tempOutNames;
- }
+ }
catch(exception& e) {
m->errorOut(e, "MakeContigsCommand", "MakeContigsCommand");
exit(1);
//**********************************************************************************************************************
MakeContigsCommand::MakeContigsCommand(string option) {
try {
- abort = false; calledHelp = false;
+ abort = false; calledHelp = false;
+ createFileGroup = false; createOligosGroup = false;
//allow user to run help
if(option == "help") { help(); abort = true; calledHelp = true; }
m->mothurConvert(temp, gapExtend);
if (gapExtend > 0) { m->mothurOut("[ERROR]: gapextend must be negative.\n"); abort=true; }
- temp = validParameter.validFile(parameters, "insert", false); if (temp == "not found"){ temp = "25"; }
+ temp = validParameter.validFile(parameters, "insert", false); if (temp == "not found"){ temp = "20"; }
m->mothurConvert(temp, insert);
if ((insert < 0) || (insert > 40)) { m->mothurOut("[ERROR]: insert must be between 0 and 40.\n"); abort=true; }
temp = validParameter.validFile(parameters, "allfiles", false); if (temp == "not found") { temp = "F"; }
allFiles = m->isTrue(temp);
+
temp = validParameter.validFile(parameters, "trimoverlap", false); if (temp == "not found") { temp = "F"; }
trimOverlap = m->isTrue(temp);
outputNames.push_back(compositeScrapFastaFile); outputTypes["fasta"].push_back(compositeScrapFastaFile);
}
+ map<string, int> totalGroupCounts;
+
for (int l = 0; l < filesToProcess.size(); l++) {
m->mothurOut("\n>>>>>\tProcessing " + filesToProcess[l][0][0] + " (file " + toString(l+1) + " of " + toString(filesToProcess.size()) + ")\t<<<<<\n");
+ groupCounts.clear();
+ groupMap.clear();
vector<vector<string> > fastaFileNames;
- createGroup = false;
+ createOligosGroup = false;
string outputGroupFileName;
map<string, string> variables;
string thisOutputDir = outputDir;
if (outputDir == "") { thisOutputDir = m->hasPath(filesToProcess[l][0][0]); }
variables["[filename]"] = thisOutputDir + m->getRootName(m->getSimpleName(filesToProcess[l][0][0]));
variables["[tag]"] = "";
- if(oligosfile != ""){
- createGroup = getOligos(fastaFileNames, variables["[filename]"]);
- if (createGroup) {
- outputGroupFileName = getOutputFileName("group",variables);
- outputNames.push_back(outputGroupFileName); outputTypes["group"].push_back(outputGroupFileName);
- }
+ if(oligosfile != ""){ createOligosGroup = getOligos(fastaFileNames, variables["[filename]"]); }
+ if (createOligosGroup || createFileGroup) {
+ outputGroupFileName = getOutputFileName("group",variables);
}
+ //give group in file file precedence
+ if (createFileGroup) { createOligosGroup = false; }
+
variables["[tag]"] = "trim";
string outFastaFile = getOutputFileName("fasta",variables);
variables["[tag]"] = "scrap";
string outScrapFastaFile = getOutputFileName("fasta",variables);
variables["[tag]"] = "";
string outMisMatchFile = getOutputFileName("report",variables);
- outputNames.push_back(outFastaFile); outputTypes["fasta"].push_back(outFastaFile);
- outputNames.push_back(outScrapFastaFile); outputTypes["fasta"].push_back(outScrapFastaFile);
- outputNames.push_back(outMisMatchFile); outputTypes["report"].push_back(outMisMatchFile);
-
+
m->mothurOut("Making contigs...\n");
- createProcesses(filesToProcess[l], outFastaFile, outScrapFastaFile, outMisMatchFile, fastaFileNames);
- m->mothurOut("Done.\n");
+ createProcesses(filesToProcess[l], outFastaFile, outScrapFastaFile, outMisMatchFile, fastaFileNames, l);
+ m->mothurOut("Here...\n");
//remove temp fasta and qual files
for (int i = 0; i < processors; i++) { for(int j = 0; j < filesToProcess[l][i].size(); j++) { m->mothurRemove(filesToProcess[l][i][j]); } }
ofstream out;
string thisGroupName = thisOutputDir + m->getRootName(m->getSimpleName(it->first));
- thisGroupName += getOutputFileName("group",variables); outputNames.push_back(thisGroupName); outputTypes["group"].push_back(thisGroupName);
+ thisGroupName += getOutputFileName("group",variables); outputNames.push_back(thisGroupName); outputTypes["group"].push_back(thisGroupName);
m->openOutputFile(thisGroupName, out);
while (!in.eof()){
if (m->control_pressed) { break; }
Sequence currSeq(in); m->gobble(in);
- out << currSeq.getName() << '\t' << it->second << endl;
+ out << currSeq.getName() << '\t' << it->second << endl;
}
- in.close();
out.close();
+ in.close();
}
}
- if (createGroup) {
+ if (createFileGroup || createOligosGroup) {
ofstream outGroup;
m->openOutputFile(outputGroupFileName, outGroup);
for (map<string, string>::iterator itGroup = groupMap.begin(); itGroup != groupMap.end(); itGroup++) {
}
if (filesToProcess.size() > 1) { //merge into large combo files
- if (createGroup) {
- if (l == 0) {
+ if (createFileGroup || createOligosGroup) {
+ if (l == 0) {
ofstream outCGroup;
m->openOutputFile(compositeGroupFile, outCGroup); outCGroup.close();
outputNames.push_back(compositeGroupFile); outputTypes["group"].push_back(compositeGroupFile);
}
- m->appendFiles(outputGroupFileName, compositeGroupFile);
+ m->appendFiles(outputGroupFileName, compositeGroupFile);
+ if (!allFiles) { m->mothurRemove(outputGroupFileName); }
+ else { outputNames.push_back(outputGroupFileName); outputTypes["group"].push_back(outputGroupFileName); }
+
+ for (map<string, int>::iterator itGroups = groupCounts.begin(); itGroups != groupCounts.end(); itGroups++) {
+ map<string, int>::iterator itTemp = totalGroupCounts.find(itGroups->first);
+ if (itTemp == totalGroupCounts.end()) { totalGroupCounts[itGroups->first] = itGroups->second; } //new group create it in totalGroups
+ else { itTemp->second += itGroups->second; } //existing group, update total
+ }
}
- m->appendFiles(outMisMatchFile, compositeMisMatchFile);
+ if (l == 0) { m->appendFiles(outMisMatchFile, compositeMisMatchFile); }
+ else { m->appendFilesWithoutHeaders(outMisMatchFile, compositeMisMatchFile); }
m->appendFiles(outFastaFile, compositeFastaFile);
m->appendFiles(outScrapFastaFile, compositeScrapFastaFile);
+ if (!allFiles) {
+ m->mothurRemove(outMisMatchFile);
+ m->mothurRemove(outFastaFile);
+ m->mothurRemove(outScrapFastaFile);
+ }else {
+ outputNames.push_back(outFastaFile); outputTypes["fasta"].push_back(outFastaFile);
+ outputNames.push_back(outScrapFastaFile); outputTypes["fasta"].push_back(outScrapFastaFile);
+ outputNames.push_back(outMisMatchFile); outputTypes["report"].push_back(outMisMatchFile);
+ }
+ }else {
+ totalGroupCounts = groupCounts;
+ outputNames.push_back(outFastaFile); outputTypes["fasta"].push_back(outFastaFile);
+ outputNames.push_back(outScrapFastaFile); outputTypes["fasta"].push_back(outScrapFastaFile);
+ outputNames.push_back(outMisMatchFile); outputTypes["report"].push_back(outMisMatchFile);
+ if (createFileGroup || createOligosGroup) {
+ outputNames.push_back(outputGroupFileName); outputTypes["group"].push_back(outputGroupFileName);
+ }
}
+ m->mothurOut("Done.\n");
}
+
m->mothurOut("It took " + toString(time(NULL) - start) + " secs to process " + toString(numReads) + " sequences.\n");
if (m->control_pressed) { for (int i = 0; i < outputNames.size(); i++) { m->mothurRemove(outputNames[i]); } return 0; }
//output group counts
m->mothurOutEndLine();
int total = 0;
- if (groupCounts.size() != 0) { m->mothurOut("Group count: \n"); }
- for (map<string, int>::iterator it = groupCounts.begin(); it != groupCounts.end(); it++) {
+ if (totalGroupCounts.size() != 0) { m->mothurOut("Group count: \n"); }
+ for (map<string, int>::iterator it = totalGroupCounts.begin(); it != totalGroupCounts.end(); it++) {
total += it->second; m->mothurOut(it->first + "\t" + toString(it->second)); m->mothurOutEndLine();
}
if (total != 0) { m->mothurOut("Total of all groups is " + toString(total)); m->mothurOutEndLine(); }
}
}
//**********************************************************************************************************************
-int MakeContigsCommand::createProcesses(vector< vector<string> > files, string outputFasta, string outputScrapFasta, string outputMisMatches, vector<vector<string> > fastaFileNames) {
+int MakeContigsCommand::createProcesses(vector< vector<string> > files, string outputFasta, string outputScrapFasta, string outputMisMatches, vector<vector<string> > fastaFileNames, int index) {
try {
int num = 0;
vector<int> processIDS;
+ string group = "";
+ map<int, string>::iterator it = file2Group.find(index);
+ if (it != file2Group.end()) { group = it->second; }
+
#if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix)
int process = 0;
}
}
}
-
+
num = driver(files[process],
outputFasta + toString(getpid()) + ".temp",
outputScrapFasta + toString(getpid()) + ".temp",
outputMisMatches + toString(getpid()) + ".temp",
- tempFASTAFileNames, process);
+ tempFASTAFileNames, process, group);
//pass groupCounts to parent
ofstream out;
string tempFile = toString(getpid()) + ".num.temp";
m->openOutputFile(tempFile, out);
out << num << endl;
- if(createGroup){
+ if (createFileGroup || createOligosGroup) {
out << groupCounts.size() << endl;
for (map<string, int>::iterator it = groupCounts.begin(); it != groupCounts.end(); it++) {
m->openOutputFile(outputScrapFasta, temp); temp.close();
//do my part
- num = driver(files[processors-1], outputFasta, outputScrapFasta, outputMisMatches, fastaFileNames, processors-1);
+ num = driver(files[processors-1], outputFasta, outputScrapFasta, outputMisMatches, fastaFileNames, processors-1, group);
//force parent to wait until all the processes are done
for (int i=0;i<processIDS.size();i++) {
int tempNum;
in >> tempNum; num += tempNum; m->gobble(in);
- if(createGroup){
+ if (createFileGroup || createOligosGroup) {
string group;
in >> tempNum; m->gobble(in);
}
}
}
-
-
- contigsData* tempcontig = new contigsData(files[h], (outputFasta + extension), (outputScrapFasta + extension), (outputMisMatches + extension), align, m, match, misMatch, gapOpen, gapExtend, insert, deltaq, barcodes, primers, tempFASTAFileNames, barcodeNameVector, primerNameVector, pdiffs, bdiffs, tdiffs, createGroup, allFiles, trimOverlap, h);
+
+ contigsData* tempcontig = new contigsData(group, files[h], (outputFasta + extension), (outputScrapFasta + extension), (outputMisMatches + extension), align, m, match, misMatch, gapOpen, gapExtend, insert, deltaq, barcodes, primers, tempFASTAFileNames, barcodeNameVector, primerNameVector, pdiffs, bdiffs, tdiffs, createOligosGroup, createFileGroup, allFiles, trimOverlap, h);
pDataArray.push_back(tempcontig);
hThreadArray[h] = CreateThread(NULL, 0, MyContigsThreadFunction, pDataArray[h], 0, &dwThreadIdArray[h]);
//do my part
processIDS.push_back(processors-1);
- num = driver(files[processors-1], (outputFasta+ toString(processors-1) + ".temp"), (outputScrapFasta+ toString(processors-1) + ".temp"), (outputMisMatches+ toString(processors-1) + ".temp"), tempFASTAFileNames, processors-1);
+ num = driver(files[processors-1], (outputFasta+ toString(processors-1) + ".temp"), (outputScrapFasta+ toString(processors-1) + ".temp"), (outputMisMatches+ toString(processors-1) + ".temp"), tempFASTAFileNames, processors-1, group);
//Wait until all threads have terminated.
WaitForMultipleObjects(processors-1, hThreadArray, TRUE, INFINITE);
m->appendFiles((outputScrapFasta + toString(processIDS[i]) + ".temp"), outputScrapFasta);
m->mothurRemove((outputScrapFasta + toString(processIDS[i]) + ".temp"));
- m->appendFiles((outputMisMatches + toString(processIDS[i]) + ".temp"), outputMisMatches);
+ m->appendFilesWithoutHeaders((outputMisMatches + toString(processIDS[i]) + ".temp"), outputMisMatches);
m->mothurRemove((outputMisMatches + toString(processIDS[i]) + ".temp"));
if(allFiles){
}
}
//**********************************************************************************************************************
-int MakeContigsCommand::driver(vector<string> files, string outputFasta, string outputScrapFasta, string outputMisMatches, vector<vector<string> > fastaFileNames, int process){
+int MakeContigsCommand::driver(vector<string> files, string outputFasta, string outputScrapFasta, string outputMisMatches, vector<vector<string> > fastaFileNames, int process, string group){
try {
Alignment* alignment;
m->openOutputFile(outputFasta, outFasta);
m->openOutputFile(outputScrapFasta, outScrapFasta);
m->openOutputFile(outputMisMatches, outMisMatch);
- if (process == 0) { outMisMatch << "Name\tLength\tOverlap_Length\tOverlap_Start\tOverlap_End\tMisMatches\tNum_Ns\n"; }
+ outMisMatch << "Name\tLength\tOverlap_Length\tOverlap_Start\tOverlap_End\tMisMatches\tNum_Ns\n";
TrimOligos trimOligos(pdiffs, bdiffs, 0, 0, primers, barcodes);
contig += seq1[i];
}else if (((seq1[i] == '.') || (seq1[i] == '-')) && ((seq2[i] != '-') && (seq2[i] != '.'))) { //seq1 is a gap and seq2 is a base, choose seq2, unless quality score for base is below insert. In that case eliminate base
if (thisfqualfile != "") {
- if (scores2[BBaseMap[i]] < insert) { } //
+ if (scores2[BBaseMap[i]] <= insert) { } //
else { contig += seq2[i]; }
}else { contig += seq2[i]; } //with no quality info, then we keep it?
}else if (((seq2[i] == '.') || (seq2[i] == '-')) && ((seq1[i] != '-') && (seq1[i] != '.'))) { //seq2 is a gap and seq1 is a base, choose seq1, unless quality score for base is below insert. In that case eliminate base
if (thisfqualfile != "") {
- if (scores1[ABaseMap[i]] < insert) { } //
+ if (scores1[ABaseMap[i]] <= insert) { } //
else { contig += seq1[i]; }
}else { contig += seq1[i]; } //with no quality info, then we keep it?
}else if (((seq1[i] != '-') && (seq1[i] != '.')) && ((seq2[i] != '-') && (seq2[i] != '.'))) { //both bases choose one with better quality
if (m->debug) { m->mothurOut(fSeq.getName()); }
- if (createGroup) {
+ if (createOligosGroup) {
if(barcodes.size() != 0){
string thisGroup = barcodeNameVector[barcodeIndex];
if (primers.size() != 0) {
}else { ignore = true; }
}
+ }else if (createFileGroup) {
+ int pos = group.find("ignore");
+ if (pos == string::npos) {
+ groupMap[fSeq.getName()] = group;
+
+ map<string, int>::iterator it = groupCounts.find(group);
+ if (it == groupCounts.end()) { groupCounts[group] = 1; }
+ else { groupCounts[it->first] ++; }
+ }else { ignore = true; }
}
if (m->debug) { m->mothurOut("\n"); }
if (m->control_pressed) { return files; }
in >> forward; m->gobble(in);
- in >> reverse; m->gobble(in);
+ in >> reverse;
+
+ string group = "";
+ while (!in.eof()) { //do we have a group assigned to this pair
+ char c = in.get();
+ if (c == 10 || c == 13 || c == -1){ break; }
+ else if (c == 32 || c == 9){;} //space or tab
+ else { group += c; }
+ }
+
+ if (group != "") {
+ //line in file look like: group forward reverse
+ string temp = forward;
+ forward = reverse;
+ reverse = group;
+ group = temp;
+ createFileGroup = true;
+ }
+ m->gobble(in);
+
+ if (m->debug) { m->mothurOut("[DEBUG]: group = " + group + ", forward = " + forward + ", reverse = " + reverse + ".\n"); }
//check to make sure both are able to be opened
ifstream in2;
}else{ in3.close(); }
if ((openForward != 1) && (openReverse != 1)) { //good pair
+ file2Group[files.size()] = group;
vector<string> pair;
pair.push_back(forward);
pair.push_back(reverse);
files.push_back(pair);
}
-
}
in.close();
if(foligo[i] == 'U') { foligo[i] = 'T'; }
}
- if(type == "FORWARD"){
+ if(type == "PRIMER"){
m->gobble(in);
in >> roligo;
}
//roligo = reverseOligo(roligo);
+ if (m->debug) { m->mothurOut("[DEBUG]: reading - " + roligo + ".\n"); }
+
group = "";
// get rest of line in case there is a primer name
while (!in.eof()) {
char c = in.get();
- if (c == 10 || c == 13){ break; }
+ if (c == 10 || c == 13 || c == -1){ break; }
else if (c == 32 || c == 9){;} //space or tab
else { group += c; }
}
oligosPair newPrimer(foligo, roligo);
+
+ if (m->debug) { m->mothurOut("[DEBUG]: primer pair " + newPrimer.forward + " " + newPrimer.reverse + ", and group = " + group + ".\n"); }
//check for repeat barcodes
string tempPair = foligo+roligo;
group = "";
while (!in.eof()) {
char c = in.get();
- if (c == 10 || c == 13){ break; }
+ if (c == 10 || c == 13 || c == -1){ break; }
else if (c == 32 || c == 9){;} //space or tab
else { group += c; }
}
projects / mothur.git / blobdiff