//
#include "makecontigscommand.h"
+#include "counttable.h"
//**********************************************************************************************************************
vector<string> MakeContigsCommand::setParameters(){
CommandParameter poligos("oligos", "InputTypes", "", "", "none", "none", "none","group",false,false,true); parameters.push_back(poligos);
CommandParameter ppdiffs("pdiffs", "Number", "", "0", "", "", "","",false,false,true); parameters.push_back(ppdiffs);
CommandParameter pbdiffs("bdiffs", "Number", "", "0", "", "", "","",false,false,true); parameters.push_back(pbdiffs);
-// CommandParameter pldiffs("ldiffs", "Number", "", "0", "", "", "",false,false); parameters.push_back(pldiffs);
-// CommandParameter psdiffs("sdiffs", "Number", "", "0", "", "", "",false,false); parameters.push_back(psdiffs);
CommandParameter ptdiffs("tdiffs", "Number", "", "0", "", "", "","",false,false); parameters.push_back(ptdiffs);
CommandParameter palign("align", "Multiple", "needleman-gotoh", "needleman", "", "", "","",false,false); parameters.push_back(palign);
CommandParameter pallfiles("allfiles", "Boolean", "", "F", "", "", "","",false,false); parameters.push_back(pallfiles);
+ CommandParameter pmakecount("makecount", "Boolean", "", "F", "", "", "","",false,false); parameters.push_back(pmakecount);
CommandParameter ptrimoverlap("trimoverlap", "Boolean", "", "F", "", "", "","",false,false); parameters.push_back(ptrimoverlap);
CommandParameter pmatch("match", "Number", "", "1.0", "", "", "","",false,false); parameters.push_back(pmatch);
CommandParameter pmismatch("mismatch", "Number", "", "-1.0", "", "", "","",false,false); parameters.push_back(pmismatch);
string helpString = "";
helpString += "The make.contigs command reads a file, forward fastq file and a reverse fastq file or forward fasta and reverse fasta files and outputs new fasta. It will also provide new quality files if the fastq or file parameter is used.\n";
helpString += "If an oligos file is provided barcodes and primers will be trimmed, and a group file will be created.\n";
- helpString += "The make.contigs command parameters are file, ffastq, rfastq, ffasta, rfasta, fqfile, rqfile, oligos, format, tdiffs, bdiffs, pdiffs, align, match, mismatch, gapopen, gapextend, insert, deltaq, allfiles and processors.\n";
+ helpString += "The make.contigs command parameters are file, ffastq, rfastq, ffasta, rfasta, fqfile, rqfile, oligos, format, tdiffs, bdiffs, pdiffs, align, match, mismatch, gapopen, gapextend, insert, deltaq, allfiles, makecount and processors.\n";
helpString += "The ffastq and rfastq, file, or ffasta and rfasta parameters are required.\n";
- helpString += "The file parameter is 2 column file containing the forward fastq files in the first column and their matching reverse fastq files in the second column. Mothur will process each pair and create a combined fasta and report file with all the sequences.\n";
+ helpString += "The file parameter is 2 or 3 column file containing the forward fastq files in the first column and their matching reverse fastq files in the second column, or a groupName then forward fastq file and reverse fastq file. Mothur will process each pair and create a combined fasta and report file with all the sequences.\n";
helpString += "The ffastq and rfastq parameters are used to provide a forward fastq and reverse fastq file to process. If you provide one, you must provide the other.\n";
helpString += "The ffasta and rfasta parameters are used to provide a forward fasta and reverse fasta file to process. If you provide one, you must provide the other.\n";
helpString += "The fqfile and rqfile parameters are used to provide a forward quality and reverse quality files to process with the ffasta and rfasta parameters. If you provide one, you must provide the other.\n";
helpString += "The insert parameter allows you to set a quality scores threshold. In the case where we are trying to decide whether to keep a base or remove it because the base is compared to a gap in the other fragment, if the base has a quality score equal to or below the threshold we eliminate it. Default=20.\n";
helpString += "The processors parameter allows you to specify how many processors you would like to use. The default is 1. \n";
helpString += "The allfiles parameter will create separate group and fasta file for each grouping. The default is F.\n";
+ helpString += "The makecount parameter will create a count table file instead of a group file. The default is F.\n";
+
helpString += "The trimoverlap parameter allows you to trim the sequences to only the overlapping section. The default is F.\n";
helpString += "The make.contigs command should be in the following format: \n";
helpString += "make.contigs(ffastq=yourForwardFastqFile, rfastq=yourReverseFastqFile, align=yourAlignmentMethod) \n";
if (type == "fasta") { pattern = "[filename],[tag],contigs.fasta"; }
else if (type == "group") { pattern = "[filename],[tag],contigs.groups"; }
+ else if (type == "count") { pattern = "[filename],[tag],contigs.count_table"; }
else if (type == "report") { pattern = "[filename],[tag],contigs.report"; }
else { m->mothurOut("[ERROR]: No definition for type " + type + " output pattern.\n"); m->control_pressed = true; }
outputTypes["fasta"] = tempOutNames;
outputTypes["group"] = tempOutNames;
outputTypes["report"] = tempOutNames;
+ outputTypes["count"] = tempOutNames;
}
catch(exception& e) {
m->errorOut(e, "MakeContigsCommand", "MakeContigsCommand");
//**********************************************************************************************************************
MakeContigsCommand::MakeContigsCommand(string option) {
try {
- abort = false; calledHelp = false;
+ abort = false; calledHelp = false;
+ createFileGroup = false; createOligosGroup = false;
//allow user to run help
if(option == "help") { help(); abort = true; calledHelp = true; }
outputTypes["fasta"] = tempOutNames;
outputTypes["report"] = tempOutNames;
outputTypes["group"] = tempOutNames;
+ outputTypes["count"] = tempOutNames;
//if the user changes the input directory command factory will send this info to us in the output parameter
temp = validParameter.validFile(parameters, "allfiles", false); if (temp == "not found") { temp = "F"; }
allFiles = m->isTrue(temp);
+ temp = validParameter.validFile(parameters, "makecount", false); if (temp == "not found") { temp = "F"; }
+ makeCount = m->isTrue(temp);
+
temp = validParameter.validFile(parameters, "trimoverlap", false); if (temp == "not found") { temp = "F"; }
trimOverlap = m->isTrue(temp);
cvars["[filename]"] = compOutputDir + m->getRootName(m->getSimpleName(file));
cvars["[tag]"] = "";
string compositeGroupFile = getOutputFileName("group",cvars);
+ if (makeCount) { compositeGroupFile = getOutputFileName("count",cvars); }
cvars["[tag]"] = "trim";
string compositeFastaFile = getOutputFileName("fasta",cvars);
cvars["[tag]"] = "scrap";
outputNames.push_back(compositeScrapFastaFile); outputTypes["fasta"].push_back(compositeScrapFastaFile);
}
+ map<string, int> totalGroupCounts;
+ CountTable compositeCt;
for (int l = 0; l < filesToProcess.size(); l++) {
m->mothurOut("\n>>>>>\tProcessing " + filesToProcess[l][0][0] + " (file " + toString(l+1) + " of " + toString(filesToProcess.size()) + ")\t<<<<<\n");
+ groupCounts.clear();
+ groupMap.clear();
vector<vector<string> > fastaFileNames;
- createGroup = false;
+ createOligosGroup = false;
string outputGroupFileName;
map<string, string> variables;
string thisOutputDir = outputDir;
if (outputDir == "") { thisOutputDir = m->hasPath(filesToProcess[l][0][0]); }
variables["[filename]"] = thisOutputDir + m->getRootName(m->getSimpleName(filesToProcess[l][0][0]));
variables["[tag]"] = "";
- if(oligosfile != ""){
- createGroup = getOligos(fastaFileNames, variables["[filename]"]);
- if (createGroup) {
- outputGroupFileName = getOutputFileName("group",variables);
- outputNames.push_back(outputGroupFileName); outputTypes["group"].push_back(outputGroupFileName);
- }
+ if(oligosfile != ""){ createOligosGroup = getOligos(fastaFileNames, variables["[filename]"]); }
+ if (createOligosGroup || createFileGroup) {
+ outputGroupFileName = getOutputFileName("group",variables);
+ if (makeCount) { outputGroupFileName = getOutputFileName("count",variables); }
}
+ //give group in file file precedence
+ if (createFileGroup) { createOligosGroup = false; }
+
variables["[tag]"] = "trim";
string outFastaFile = getOutputFileName("fasta",variables);
variables["[tag]"] = "scrap";
string outScrapFastaFile = getOutputFileName("fasta",variables);
variables["[tag]"] = "";
string outMisMatchFile = getOutputFileName("report",variables);
- outputNames.push_back(outFastaFile); outputTypes["fasta"].push_back(outFastaFile);
- outputNames.push_back(outScrapFastaFile); outputTypes["fasta"].push_back(outScrapFastaFile);
- outputNames.push_back(outMisMatchFile); outputTypes["report"].push_back(outMisMatchFile);
-
+
m->mothurOut("Making contigs...\n");
- createProcesses(filesToProcess[l], outFastaFile, outScrapFastaFile, outMisMatchFile, fastaFileNames);
+ createProcesses(filesToProcess[l], outFastaFile, outScrapFastaFile, outMisMatchFile, fastaFileNames, l);
m->mothurOut("Done.\n");
//remove temp fasta and qual files
ofstream out;
string thisGroupName = thisOutputDir + m->getRootName(m->getSimpleName(it->first));
- thisGroupName += getOutputFileName("group",variables); outputNames.push_back(thisGroupName); outputTypes["group"].push_back(thisGroupName);
- m->openOutputFile(thisGroupName, out);
-
- while (!in.eof()){
- if (m->control_pressed) { break; }
+ if (!makeCount) { thisGroupName += getOutputFileName("group",variables); outputNames.push_back(thisGroupName); outputTypes["group"].push_back(thisGroupName);
+ m->openOutputFile(thisGroupName, out);
- Sequence currSeq(in); m->gobble(in);
- out << currSeq.getName() << '\t' << it->second << endl;
+ while (!in.eof()){
+ if (m->control_pressed) { break; }
+
+ Sequence currSeq(in); m->gobble(in);
+ out << currSeq.getName() << '\t' << it->second << endl;
+ }
+ out.close();
+ }
+ else {
+ thisGroupName += getOutputFileName("count",variables); outputNames.push_back(thisGroupName); outputTypes["count"].push_back(thisGroupName);
+ CountTable ct;
+ ct.addGroup(it->second);
+ while (!in.eof()){
+ if (m->control_pressed) { break; }
+
+ Sequence currSeq(in); m->gobble(in);
+ vector<int> tempGroupCount; tempGroupCount.push_back(1);
+ ct.push_back(currSeq.getName(), tempGroupCount);
+ }
+ ct.printTable(thisGroupName);
}
in.close();
- out.close();
}
}
- if (createGroup) {
- ofstream outGroup;
- m->openOutputFile(outputGroupFileName, outGroup);
- for (map<string, string>::iterator itGroup = groupMap.begin(); itGroup != groupMap.end(); itGroup++) {
- outGroup << itGroup->first << '\t' << itGroup->second << endl;
+ if (createFileGroup || createOligosGroup) {
+ if (makeCount) {
+ if ((allFiles) || (filesToProcess.size() == 1)) {
+ CountTable ct;
+ for (map<string, int>::iterator itGroups = groupCounts.begin(); itGroups != groupCounts.end(); itGroups++) {
+ ct.addGroup(itGroups->first);
+ }
+ vector<string> groups = ct.getNamesOfGroups();
+ for (map<string, string>::iterator itGroup = groupMap.begin(); itGroup != groupMap.end(); itGroup++) {
+ vector<int> tempGroupCounts; tempGroupCounts.resize(groups.size(), 0);
+ ct.push_back(itGroup->first, tempGroupCounts);
+ ct.setAbund(itGroup->first, itGroup->second, 1);
+ }
+ ct.printTable(outputGroupFileName);
+ }
+ }else {
+ ofstream outGroup;
+ m->openOutputFile(outputGroupFileName, outGroup);
+ for (map<string, string>::iterator itGroup = groupMap.begin(); itGroup != groupMap.end(); itGroup++) {
+ outGroup << itGroup->first << '\t' << itGroup->second << endl;
+ }
+ outGroup.close();
}
- outGroup.close();
}
if (filesToProcess.size() > 1) { //merge into large combo files
- if (createGroup) {
- if (l == 0) {
- ofstream outCGroup;
- m->openOutputFile(compositeGroupFile, outCGroup); outCGroup.close();
- outputNames.push_back(compositeGroupFile); outputTypes["group"].push_back(compositeGroupFile);
+ if (createFileGroup || createOligosGroup) {
+ if (makeCount) {
+ for (map<string, string>::iterator itGroup = groupMap.begin(); itGroup != groupMap.end(); itGroup++) {
+ vector<string> groups = compositeCt.getNamesOfGroups();
+ if (m->inUsersGroups(itGroup->second, groups)) {
+ vector<int> tempGroupCounts; tempGroupCounts.resize(groups.size(), 0);
+ compositeCt.push_back(itGroup->first, tempGroupCounts);
+ compositeCt.setAbund(itGroup->first, itGroup->second, 1);
+ }else{
+ compositeCt.addGroup(itGroup->second);
+ vector<int> tempGroupCounts; tempGroupCounts.resize(groups.size()+1, 0);
+ compositeCt.push_back(itGroup->first, tempGroupCounts);
+ compositeCt.setAbund(itGroup->first, itGroup->second, 1);
+ }
+ }
+ }else {
+ if (l == 0) {
+ ofstream outCGroup;
+ m->openOutputFile(compositeGroupFile, outCGroup); outCGroup.close();
+ outputNames.push_back(compositeGroupFile); outputTypes["group"].push_back(compositeGroupFile);
+ }
+ m->appendFiles(outputGroupFileName, compositeGroupFile);
+ if (!allFiles) { m->mothurRemove(outputGroupFileName); }
+ else { outputNames.push_back(outputGroupFileName); outputTypes["group"].push_back(outputGroupFileName); }
+ }
+ for (map<string, int>::iterator itGroups = groupCounts.begin(); itGroups != groupCounts.end(); itGroups++) {
+ map<string, int>::iterator itTemp = totalGroupCounts.find(itGroups->first);
+ if (itTemp == totalGroupCounts.end()) { totalGroupCounts[itGroups->first] = itGroups->second; } //new group create it in totalGroups
+ else { itTemp->second += itGroups->second; } //existing group, update total
}
- m->appendFiles(outputGroupFileName, compositeGroupFile);
}
- m->appendFiles(outMisMatchFile, compositeMisMatchFile);
+ if (l == 0) { m->appendFiles(outMisMatchFile, compositeMisMatchFile); }
+ else { m->appendFilesWithoutHeaders(outMisMatchFile, compositeMisMatchFile); }
m->appendFiles(outFastaFile, compositeFastaFile);
m->appendFiles(outScrapFastaFile, compositeScrapFastaFile);
+ if (!allFiles) {
+ m->mothurRemove(outMisMatchFile);
+ m->mothurRemove(outFastaFile);
+ m->mothurRemove(outScrapFastaFile);
+ }else {
+ outputNames.push_back(outFastaFile); outputTypes["fasta"].push_back(outFastaFile);
+ outputNames.push_back(outScrapFastaFile); outputTypes["fasta"].push_back(outScrapFastaFile);
+ outputNames.push_back(outMisMatchFile); outputTypes["report"].push_back(outMisMatchFile);
+ }
+ }else {
+ totalGroupCounts = groupCounts;
+ outputNames.push_back(outFastaFile); outputTypes["fasta"].push_back(outFastaFile);
+ outputNames.push_back(outScrapFastaFile); outputTypes["fasta"].push_back(outScrapFastaFile);
+ outputNames.push_back(outMisMatchFile); outputTypes["report"].push_back(outMisMatchFile);
+ if (createFileGroup || createOligosGroup) {
+ if (makeCount) { outputNames.push_back(outputGroupFileName); outputTypes["count"].push_back(outputGroupFileName); }
+ else { outputNames.push_back(outputGroupFileName); outputTypes["group"].push_back(outputGroupFileName); }
+ }
}
}
+
+ if ((filesToProcess.size() > 1) && makeCount) { //merge into large combo files
+ compositeCt.printTable(compositeGroupFile);
+ outputNames.push_back(compositeGroupFile); outputTypes["count"].push_back(compositeGroupFile);
+ }
m->mothurOut("It took " + toString(time(NULL) - start) + " secs to process " + toString(numReads) + " sequences.\n");
if (m->control_pressed) { for (int i = 0; i < outputNames.size(); i++) { m->mothurRemove(outputNames[i]); } return 0; }
//output group counts
m->mothurOutEndLine();
int total = 0;
- if (groupCounts.size() != 0) { m->mothurOut("Group count: \n"); }
- for (map<string, int>::iterator it = groupCounts.begin(); it != groupCounts.end(); it++) {
+ if (totalGroupCounts.size() != 0) { m->mothurOut("Group count: \n"); }
+ for (map<string, int>::iterator it = totalGroupCounts.begin(); it != totalGroupCounts.end(); it++) {
total += it->second; m->mothurOut(it->first + "\t" + toString(it->second)); m->mothurOutEndLine();
}
if (total != 0) { m->mothurOut("Total of all groups is " + toString(total)); m->mothurOutEndLine(); }
if (itTypes != outputTypes.end()) {
if ((itTypes->second).size() != 0) { currentGroup = (itTypes->second)[0]; m->setGroupFile(currentGroup); }
}
+
+ string currentCount = "";
+ itTypes = outputTypes.find("count");
+ if (itTypes != outputTypes.end()) {
+ if ((itTypes->second).size() != 0) { currentCount = (itTypes->second)[0]; m->setCountTableFile(currentCount); }
+ }
//output files created by command
m->mothurOutEndLine();
}
}
//**********************************************************************************************************************
-int MakeContigsCommand::createProcesses(vector< vector<string> > files, string outputFasta, string outputScrapFasta, string outputMisMatches, vector<vector<string> > fastaFileNames) {
+int MakeContigsCommand::createProcesses(vector< vector<string> > files, string outputFasta, string outputScrapFasta, string outputMisMatches, vector<vector<string> > fastaFileNames, int index) {
try {
int num = 0;
vector<int> processIDS;
+ string group = "";
+ map<int, string>::iterator it = file2Group.find(index);
+ if (it != file2Group.end()) { group = it->second; }
+
#if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix)
int process = 0;
}
}
}
-
+
num = driver(files[process],
outputFasta + toString(getpid()) + ".temp",
outputScrapFasta + toString(getpid()) + ".temp",
outputMisMatches + toString(getpid()) + ".temp",
- tempFASTAFileNames, process);
+ tempFASTAFileNames, process, group);
//pass groupCounts to parent
ofstream out;
string tempFile = toString(getpid()) + ".num.temp";
m->openOutputFile(tempFile, out);
out << num << endl;
- if(createGroup){
+ if (createFileGroup || createOligosGroup) {
out << groupCounts.size() << endl;
for (map<string, int>::iterator it = groupCounts.begin(); it != groupCounts.end(); it++) {
m->openOutputFile(outputScrapFasta, temp); temp.close();
//do my part
- num = driver(files[processors-1], outputFasta, outputScrapFasta, outputMisMatches, fastaFileNames, processors-1);
+ num = driver(files[processors-1], outputFasta, outputScrapFasta, outputMisMatches, fastaFileNames, processors-1, group);
//force parent to wait until all the processes are done
for (int i=0;i<processIDS.size();i++) {
int tempNum;
in >> tempNum; num += tempNum; m->gobble(in);
- if(createGroup){
+ if (createFileGroup || createOligosGroup) {
string group;
in >> tempNum; m->gobble(in);
}
}
}
-
-
- contigsData* tempcontig = new contigsData(files[h], (outputFasta + extension), (outputScrapFasta + extension), (outputMisMatches + extension), align, m, match, misMatch, gapOpen, gapExtend, insert, deltaq, barcodes, primers, tempFASTAFileNames, barcodeNameVector, primerNameVector, pdiffs, bdiffs, tdiffs, createGroup, allFiles, trimOverlap, h);
+
+ contigsData* tempcontig = new contigsData(group, files[h], (outputFasta + extension), (outputScrapFasta + extension), (outputMisMatches + extension), align, m, match, misMatch, gapOpen, gapExtend, insert, deltaq, barcodes, primers, tempFASTAFileNames, barcodeNameVector, primerNameVector, pdiffs, bdiffs, tdiffs, createOligosGroup, createFileGroup, allFiles, trimOverlap, h);
pDataArray.push_back(tempcontig);
hThreadArray[h] = CreateThread(NULL, 0, MyContigsThreadFunction, pDataArray[h], 0, &dwThreadIdArray[h]);
//do my part
processIDS.push_back(processors-1);
- num = driver(files[processors-1], (outputFasta+ toString(processors-1) + ".temp"), (outputScrapFasta+ toString(processors-1) + ".temp"), (outputMisMatches+ toString(processors-1) + ".temp"), tempFASTAFileNames, processors-1);
+ num = driver(files[processors-1], (outputFasta+ toString(processors-1) + ".temp"), (outputScrapFasta+ toString(processors-1) + ".temp"), (outputMisMatches+ toString(processors-1) + ".temp"), tempFASTAFileNames, processors-1, group);
//Wait until all threads have terminated.
WaitForMultipleObjects(processors-1, hThreadArray, TRUE, INFINITE);
m->appendFiles((outputScrapFasta + toString(processIDS[i]) + ".temp"), outputScrapFasta);
m->mothurRemove((outputScrapFasta + toString(processIDS[i]) + ".temp"));
- m->appendFiles((outputMisMatches + toString(processIDS[i]) + ".temp"), outputMisMatches);
+ m->appendFilesWithoutHeaders((outputMisMatches + toString(processIDS[i]) + ".temp"), outputMisMatches);
m->mothurRemove((outputMisMatches + toString(processIDS[i]) + ".temp"));
if(allFiles){
}
}
//**********************************************************************************************************************
-int MakeContigsCommand::driver(vector<string> files, string outputFasta, string outputScrapFasta, string outputMisMatches, vector<vector<string> > fastaFileNames, int process){
+int MakeContigsCommand::driver(vector<string> files, string outputFasta, string outputScrapFasta, string outputMisMatches, vector<vector<string> > fastaFileNames, int process, string group){
try {
Alignment* alignment;
m->openOutputFile(outputFasta, outFasta);
m->openOutputFile(outputScrapFasta, outScrapFasta);
m->openOutputFile(outputMisMatches, outMisMatch);
- if (process == 0) { outMisMatch << "Name\tLength\tOverlap_Length\tOverlap_Start\tOverlap_End\tMisMatches\tNum_Ns\n"; }
+ outMisMatch << "Name\tLength\tOverlap_Length\tOverlap_Start\tOverlap_End\tMisMatches\tNum_Ns\n";
TrimOligos trimOligos(pdiffs, bdiffs, 0, 0, primers, barcodes);
if (m->debug) { m->mothurOut(fSeq.getName()); }
- if (createGroup) {
+ if (createOligosGroup) {
if(barcodes.size() != 0){
string thisGroup = barcodeNameVector[barcodeIndex];
if (primers.size() != 0) {
}else { ignore = true; }
}
+ }else if (createFileGroup) {
+ int pos = group.find("ignore");
+ if (pos == string::npos) {
+ groupMap[fSeq.getName()] = group;
+
+ map<string, int>::iterator it = groupCounts.find(group);
+ if (it == groupCounts.end()) { groupCounts[group] = 1; }
+ else { groupCounts[it->first] ++; }
+ }else { ignore = true; }
}
if (m->debug) { m->mothurOut("\n"); }
if (m->control_pressed) { return files; }
in >> forward; m->gobble(in);
- in >> reverse; m->gobble(in);
+ in >> reverse;
+
+ string group = "";
+ while (!in.eof()) { //do we have a group assigned to this pair
+ char c = in.get();
+ if (c == 10 || c == 13 || c == -1){ break; }
+ else if (c == 32 || c == 9){;} //space or tab
+ else { group += c; }
+ }
+
+ if (group != "") {
+ //line in file look like: group forward reverse
+ string temp = forward;
+ forward = reverse;
+ reverse = group;
+ group = temp;
+ createFileGroup = true;
+ }
+ m->gobble(in);
+
//check to make sure both are able to be opened
ifstream in2;
}else{ in3.close(); }
if ((openForward != 1) && (openReverse != 1)) { //good pair
+ file2Group[files.size()] = group;
vector<string> pair;
pair.push_back(forward);
pair.push_back(reverse);
files.push_back(pair);
}
-
}
in.close();
if(foligo[i] == 'U') { foligo[i] = 'T'; }
}
- if(type == "FORWARD"){
+ if(type == "PRIMER"){
m->gobble(in);
in >> roligo;