2 * chimeracheckcommand.cpp
5 * Created by westcott on 3/31/10.
6 * Copyright 2010 Schloss Lab. All rights reserved.
10 #include "chimeracheckcommand.h"
11 #include "referencedb.h"
13 //**********************************************************************************************************************
14 vector<string> ChimeraCheckCommand::setParameters(){
16 CommandParameter ptemplate("reference", "InputTypes", "", "", "none", "none", "none",false,true); parameters.push_back(ptemplate);
17 CommandParameter pfasta("fasta", "InputTypes", "", "", "none", "none", "none",false,true); parameters.push_back(pfasta);
18 CommandParameter pname("name", "InputTypes", "", "", "none", "none", "none",false,false); parameters.push_back(pname);
19 CommandParameter psvg("svg", "Boolean", "", "F", "", "", "",false,false); parameters.push_back(psvg);
20 CommandParameter pincrement("increment", "Number", "", "10", "", "", "",false,false); parameters.push_back(pincrement);
21 CommandParameter pksize("ksize", "Number", "", "7", "", "", "",false,false); parameters.push_back(pksize);
22 CommandParameter pprocessors("processors", "Number", "", "1", "", "", "",false,false); parameters.push_back(pprocessors);
23 CommandParameter pinputdir("inputdir", "String", "", "", "", "", "",false,false); parameters.push_back(pinputdir);
24 CommandParameter poutputdir("outputdir", "String", "", "", "", "", "",false,false); parameters.push_back(poutputdir);
25 CommandParameter psave("save", "Boolean", "", "F", "", "", "",false,false); parameters.push_back(psave);
27 vector<string> myArray;
28 for (int i = 0; i < parameters.size(); i++) { myArray.push_back(parameters[i].name); }
32 m->errorOut(e, "ChimeraCheckCommand", "setParameters");
36 //**********************************************************************************************************************
37 string ChimeraCheckCommand::getHelpString(){
39 string helpString = "";
40 helpString += "The chimera.check command reads a fastafile and referencefile and outputs potentially chimeric sequences.\n";
41 helpString += "This command was created using the algorythms described in CHIMERA_CHECK version 2.7 written by Niels Larsen. \n";
42 helpString += "The chimera.check command parameters are fasta, reference, processors, ksize, increment, svg and name.\n";
43 helpString += "The fasta parameter allows you to enter the fasta file containing your potentially chimeric sequences, and is required unless you have a valid current fasta file. \n";
44 helpString += "You may enter multiple fasta files by separating their names with dashes. ie. fasta=abrecovery.fasta-amzon.fasta \n";
45 helpString += "The reference parameter allows you to enter a reference file containing known non-chimeric sequences, and is required. \n";
46 helpString += "The processors parameter allows you to specify how many processors you would like to use. The default is 1. \n";
48 helpString += "When using MPI, the processors parameter is set to the number of MPI processes running. \n";
50 helpString += "The increment parameter allows you to specify how far you move each window while finding chimeric sequences, default is 10.\n";
51 helpString += "The ksize parameter allows you to input kmersize, default is 7. \n";
52 helpString += "The svg parameter allows you to specify whether or not you would like a svg file outputted for each query sequence, default is False.\n";
53 helpString += "The name parameter allows you to enter a file containing names of sequences you would like .svg files for.\n";
54 helpString += "You may enter multiple name files by separating their names with dashes. ie. fasta=abrecovery.svg.names-amzon.svg.names \n";
55 helpString += "If the save parameter is set to true the reference sequences will be saved in memory, to clear them later you can use the clear.memory command. Default=f.";
56 helpString += "The chimera.check command should be in the following format: \n";
57 helpString += "chimera.check(fasta=yourFastaFile, reference=yourTemplateFile, processors=yourProcessors, ksize=yourKmerSize) \n";
58 helpString += "Example: chimera.check(fasta=AD.fasta, reference=core_set_aligned,imputed.fasta, processors=4, ksize=8) \n";
59 helpString += "Note: No spaces between parameter labels (i.e. fasta), '=' and parameters (i.e.yourFastaFile).\n";
63 m->errorOut(e, "ChimeraCheckCommand", "getHelpString");
67 //**********************************************************************************************************************
68 ChimeraCheckCommand::ChimeraCheckCommand(){
70 abort = true; calledHelp = true;
72 vector<string> tempOutNames;
73 outputTypes["chimera"] = tempOutNames;
76 m->errorOut(e, "ChimeraCheckCommand", "ChimeraCheckCommand");
80 //***************************************************************************************************************
81 ChimeraCheckCommand::ChimeraCheckCommand(string option) {
83 abort = false; calledHelp = false;
84 ReferenceDB* rdb = ReferenceDB::getInstance();
86 //allow user to run help
87 if(option == "help") { help(); abort = true; calledHelp = true; }
88 else if(option == "citation") { citation(); abort = true; calledHelp = true;}
91 vector<string> myArray = setParameters();
93 OptionParser parser(option);
94 map<string,string> parameters = parser.getParameters();
96 ValidParameters validParameter("chimera.check");
97 map<string,string>::iterator it;
99 //check to make sure all parameters are valid for command
100 for (it = parameters.begin(); it != parameters.end(); it++) {
101 if (validParameter.isValidParameter(it->first, myArray, it->second) != true) { abort = true; }
104 vector<string> tempOutNames;
105 outputTypes["chimera"] = tempOutNames;
107 //if the user changes the input directory command factory will send this info to us in the output parameter
108 string inputDir = validParameter.validFile(parameters, "inputdir", false);
109 if (inputDir == "not found"){ inputDir = ""; }
111 it = parameters.find("reference");
112 //user has given a template file
113 if(it != parameters.end()){
114 string path = m->hasPath(it->second);
115 //if the user has not given a path then, add inputdir. else leave path alone.
116 if (path == "") { parameters["reference"] = inputDir + it->second; }
120 //check for required parameters
121 fastafile = validParameter.validFile(parameters, "fasta", false);
122 if (fastafile == "not found") {
123 //if there is a current fasta file, use it
124 string filename = m->getFastaFile();
125 if (filename != "") { fastaFileNames.push_back(filename); m->mothurOut("Using " + filename + " as input file for the fasta parameter."); m->mothurOutEndLine(); }
126 else { m->mothurOut("You have no current fastafile and the fasta parameter is required."); m->mothurOutEndLine(); abort = true; }
128 m->splitAtDash(fastafile, fastaFileNames);
130 //go through files and make sure they are good, if not, then disregard them
131 for (int i = 0; i < fastaFileNames.size(); i++) {
134 if (fastaFileNames[i] == "current") {
135 fastaFileNames[i] = m->getFastaFile();
136 if (fastaFileNames[i] != "") { m->mothurOut("Using " + fastaFileNames[i] + " as input file for the fasta parameter where you had given current."); m->mothurOutEndLine(); }
138 m->mothurOut("You have no current fastafile, ignoring current."); m->mothurOutEndLine(); ignore=true;
139 //erase from file list
140 fastaFileNames.erase(fastaFileNames.begin()+i);
148 if (inputDir != "") {
149 string path = m->hasPath(fastaFileNames[i]);
150 //if the user has not given a path then, add inputdir. else leave path alone.
151 if (path == "") { fastaFileNames[i] = inputDir + fastaFileNames[i]; }
157 ableToOpen = m->openInputFile(fastaFileNames[i], in, "noerror");
159 //if you can't open it, try default location
160 if (ableToOpen == 1) {
161 if (m->getDefaultPath() != "") { //default path is set
162 string tryPath = m->getDefaultPath() + m->getSimpleName(fastaFileNames[i]);
163 m->mothurOut("Unable to open " + fastaFileNames[i] + ". Trying default " + tryPath); m->mothurOutEndLine();
165 ableToOpen = m->openInputFile(tryPath, in2, "noerror");
167 fastaFileNames[i] = tryPath;
171 //if you can't open it, try default location
172 if (ableToOpen == 1) {
173 if (m->getOutputDir() != "") { //default path is set
174 string tryPath = m->getOutputDir() + m->getSimpleName(fastaFileNames[i]);
175 m->mothurOut("Unable to open " + fastaFileNames[i] + ". Trying output directory " + tryPath); m->mothurOutEndLine();
177 ableToOpen = m->openInputFile(tryPath, in2, "noerror");
179 fastaFileNames[i] = tryPath;
185 if (ableToOpen == 1) {
186 m->mothurOut("Unable to open " + fastaFileNames[i] +". It will be disregarded."); m->mothurOutEndLine();
187 //erase from file list
188 fastaFileNames.erase(fastaFileNames.begin()+i);
191 m->setFastaFile(fastaFileNames[i]);
196 //make sure there is at least one valid file left
197 if (fastaFileNames.size() == 0) { m->mothurOut("no valid files."); m->mothurOutEndLine(); abort = true; }
200 //if the user changes the output directory command factory will send this info to us in the output parameter
201 outputDir = validParameter.validFile(parameters, "outputdir", false); if (outputDir == "not found"){ outputDir = ""; }
203 namefile = validParameter.validFile(parameters, "name", false);
204 if (namefile == "not found") { namefile = ""; }
206 m->splitAtDash(namefile, nameFileNames);
208 //go through files and make sure they are good, if not, then disregard them
209 for (int i = 0; i < nameFileNames.size(); i++) {
212 if (nameFileNames[i] == "current") {
213 nameFileNames[i] = m->getNameFile();
214 if (nameFileNames[i] != "") { m->mothurOut("Using " + nameFileNames[i] + " as input file for the name parameter where you had given current."); m->mothurOutEndLine(); }
216 m->mothurOut("You have no current namefile, ignoring current."); m->mothurOutEndLine(); ignore=true;
217 //erase from file list
218 nameFileNames.erase(nameFileNames.begin()+i);
225 if (inputDir != "") {
226 string path = m->hasPath(nameFileNames[i]);
227 //if the user has not given a path then, add inputdir. else leave path alone.
228 if (path == "") { nameFileNames[i] = inputDir + nameFileNames[i]; }
234 ableToOpen = m->openInputFile(nameFileNames[i], in, "noerror");
236 //if you can't open it, try default location
237 if (ableToOpen == 1) {
238 if (m->getDefaultPath() != "") { //default path is set
239 string tryPath = m->getDefaultPath() + m->getSimpleName(nameFileNames[i]);
240 m->mothurOut("Unable to open " + nameFileNames[i] + ". Trying default " + tryPath); m->mothurOutEndLine();
242 ableToOpen = m->openInputFile(tryPath, in2, "noerror");
244 nameFileNames[i] = tryPath;
248 //if you can't open it, try default location
249 if (ableToOpen == 1) {
250 if (m->getOutputDir() != "") { //default path is set
251 string tryPath = m->getOutputDir() + m->getSimpleName(nameFileNames[i]);
252 m->mothurOut("Unable to open " + nameFileNames[i] + ". Trying output directory " + tryPath); m->mothurOutEndLine();
254 ableToOpen = m->openInputFile(tryPath, in2, "noerror");
256 nameFileNames[i] = tryPath;
262 if (ableToOpen == 1) {
263 m->mothurOut("Unable to open " + nameFileNames[i] + ". It will be disregarded."); m->mothurOutEndLine();
264 //erase from file list
265 nameFileNames.erase(nameFileNames.begin()+i);
268 m->setNameFile(nameFileNames[i]);
273 //make sure there is at least one valid file left
274 if (nameFileNames.size() != 0) {
275 if (nameFileNames.size() != fastaFileNames.size()) {
276 m->mothurOut("Different number of valid name files and fasta files, aborting command."); m->mothurOutEndLine();
282 string temp = validParameter.validFile(parameters, "processors", false); if (temp == "not found"){ temp = m->getProcessors(); }
283 m->setProcessors(temp);
284 m->mothurConvert(temp, processors);
286 temp = validParameter.validFile(parameters, "save", false); if (temp == "not found"){ temp = "f"; }
287 save = m->isTrue(temp);
289 if (save) { //clear out old references
293 //this has to go after save so that if the user sets save=t and provides no reference we abort
294 templatefile = validParameter.validFile(parameters, "reference", true);
295 if (templatefile == "not found") {
296 //check for saved reference sequences
297 if (rdb->referenceSeqs.size() != 0) {
298 templatefile = "saved";
300 m->mothurOut("[ERROR]: You don't have any saved reference sequences and the reference parameter is a required.");
301 m->mothurOutEndLine();
304 }else if (templatefile == "not open") { abort = true; }
305 else { if (save) { rdb->setSavedReference(templatefile); } }
308 temp = validParameter.validFile(parameters, "ksize", false); if (temp == "not found") { temp = "7"; }
309 m->mothurConvert(temp, ksize);
311 temp = validParameter.validFile(parameters, "svg", false); if (temp == "not found") { temp = "F"; }
312 svg = m->isTrue(temp);
313 if (nameFileNames.size() != 0) { svg = true; }
315 temp = validParameter.validFile(parameters, "increment", false); if (temp == "not found") { temp = "10"; }
316 m->mothurConvert(temp, increment);
319 catch(exception& e) {
320 m->errorOut(e, "ChimeraCheckCommand", "ChimeraCheckCommand");
324 //***************************************************************************************************************
326 int ChimeraCheckCommand::execute(){
329 if (abort == true) { if (calledHelp) { return 0; } return 2; }
331 for (int i = 0; i < fastaFileNames.size(); i++) {
333 m->mothurOut("Checking sequences from " + fastaFileNames[i] + " ..." ); m->mothurOutEndLine();
335 int start = time(NULL);
337 string thisNameFile = "";
338 if (nameFileNames.size() != 0) { thisNameFile = nameFileNames[i]; }
340 chimera = new ChimeraCheckRDP(fastaFileNames[i], templatefile, thisNameFile, svg, increment, ksize, outputDir);
342 if (m->control_pressed) { delete chimera; return 0; }
344 if (outputDir == "") { outputDir = m->hasPath(fastaFileNames[i]); }//if user entered a file with a path then preserve it
345 string outputFileName = outputDir + m->getRootName(m->getSimpleName(fastaFileNames[i])) + "chimeracheck.chimeras";
346 outputNames.push_back(outputFileName); outputTypes["chimera"].push_back(outputFileName);
350 int pid, numSeqsPerProcessor;
352 vector<unsigned long long> MPIPos;
355 MPI_Comm_rank(MPI_COMM_WORLD, &pid); //find out who we are
356 MPI_Comm_size(MPI_COMM_WORLD, &processors);
361 int outMode=MPI_MODE_CREATE|MPI_MODE_WRONLY;
362 int inMode=MPI_MODE_RDONLY;
364 char outFilename[1024];
365 strcpy(outFilename, outputFileName.c_str());
367 char inFileName[1024];
368 strcpy(inFileName, fastaFileNames[i].c_str());
370 MPI_File_open(MPI_COMM_WORLD, inFileName, inMode, MPI_INFO_NULL, &inMPI); //comm, filename, mode, info, filepointer
371 MPI_File_open(MPI_COMM_WORLD, outFilename, outMode, MPI_INFO_NULL, &outMPI);
373 if (m->control_pressed) { MPI_File_close(&inMPI); MPI_File_close(&outMPI); for (int j = 0; j < outputNames.size(); j++) { m->mothurRemove(outputNames[j]); } outputTypes.clear(); delete chimera; return 0; }
375 if (pid == 0) { //you are the root process
376 MPIPos = m->setFilePosFasta(fastaFileNames[i], numSeqs); //fills MPIPos, returns numSeqs
378 //send file positions to all processes
379 for(int j = 1; j < processors; j++) {
380 MPI_Send(&numSeqs, 1, MPI_INT, j, tag, MPI_COMM_WORLD);
381 MPI_Send(&MPIPos[0], (numSeqs+1), MPI_LONG, j, tag, MPI_COMM_WORLD);
384 //figure out how many sequences you have to align
385 numSeqsPerProcessor = numSeqs / processors;
386 int startIndex = pid * numSeqsPerProcessor;
387 if(pid == (processors - 1)){ numSeqsPerProcessor = numSeqs - pid * numSeqsPerProcessor; }
391 driverMPI(startIndex, numSeqsPerProcessor, inMPI, outMPI, MPIPos);
393 if (m->control_pressed) { MPI_File_close(&inMPI); MPI_File_close(&outMPI); for (int j = 0; j < outputNames.size(); j++) { m->mothurRemove(outputNames[j]); } outputTypes.clear(); delete chimera; return 0; }
396 for(int j = 1; j < processors; j++) {
398 MPI_Recv(buf, 5, MPI_CHAR, j, tag, MPI_COMM_WORLD, &status);
400 }else{ //you are a child process
401 MPI_Recv(&numSeqs, 1, MPI_INT, 0, tag, MPI_COMM_WORLD, &status);
402 MPIPos.resize(numSeqs+1);
403 MPI_Recv(&MPIPos[0], (numSeqs+1), MPI_LONG, 0, tag, MPI_COMM_WORLD, &status);
405 //figure out how many sequences you have to align
406 numSeqsPerProcessor = numSeqs / processors;
407 int startIndex = pid * numSeqsPerProcessor;
408 if(pid == (processors - 1)){ numSeqsPerProcessor = numSeqs - pid * numSeqsPerProcessor; }
411 driverMPI(startIndex, numSeqsPerProcessor, inMPI, outMPI, MPIPos);
413 if (m->control_pressed) { MPI_File_close(&inMPI); MPI_File_close(&outMPI); for (int j = 0; j < outputNames.size(); j++) { m->mothurRemove(outputNames[j]); } outputTypes.clear(); delete chimera; return 0; }
415 //tell parent you are done.
418 MPI_Send(buf, 5, MPI_CHAR, 0, tag, MPI_COMM_WORLD);
422 MPI_File_close(&inMPI);
423 MPI_File_close(&outMPI);
424 MPI_Barrier(MPI_COMM_WORLD); //make everyone wait - just in case
430 #if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix)
431 vector<unsigned long long> positions = m->divideFile(fastaFileNames[i], processors);
433 for (int s = 0; s < (positions.size()-1); s++) {
434 lines.push_back(new linePair(positions[s], positions[(s+1)]));
438 numSeqs = driver(lines[0], outputFileName, fastaFileNames[i]);
440 if (m->control_pressed) { for (int j = 0; j < outputNames.size(); j++) { m->mothurRemove(outputNames[j]); } for (int j = 0; j < lines.size(); j++) { delete lines[j]; } outputTypes.clear(); lines.clear(); delete chimera; return 0; }
443 processIDS.resize(0);
445 numSeqs = createProcesses(outputFileName, fastaFileNames[i]);
447 rename((outputFileName + toString(processIDS[0]) + ".temp").c_str(), outputFileName.c_str());
449 //append output files
450 for(int j=1;j<processors;j++){
451 m->appendFiles((outputFileName + toString(processIDS[j]) + ".temp"), outputFileName);
452 m->mothurRemove((outputFileName + toString(processIDS[j]) + ".temp"));
455 if (m->control_pressed) {
456 for (int j = 0; j < outputNames.size(); j++) { m->mothurRemove(outputNames[j]); } outputTypes.clear();
457 for (int j = 0; j < lines.size(); j++) { delete lines[j]; } lines.clear();
464 lines.push_back(new linePair(0, 1000));
465 numSeqs = driver(lines[0], outputFileName, fastaFileNames[i]);
467 if (m->control_pressed) { for (int j = 0; j < lines.size(); j++) { delete lines[j]; } lines.clear(); for (int j = 0; j < outputNames.size(); j++) { m->mothurRemove(outputNames[j]); } outputTypes.clear(); delete chimera; return 0; }
471 for (int j = 0; j < lines.size(); j++) { delete lines[j]; } lines.clear();
473 m->mothurOutEndLine(); m->mothurOut("This method does not determine if a sequence is chimeric, but allows you to make that determination based on the IS values."); m->mothurOutEndLine();
474 m->mothurOutEndLine(); m->mothurOut("It took " + toString(time(NULL) - start) + " secs to check " + toString(numSeqs) + " sequences."); m->mothurOutEndLine(); m->mothurOutEndLine();
478 m->mothurOutEndLine();
479 m->mothurOut("Output File Names: "); m->mothurOutEndLine();
480 for (int i = 0; i < outputNames.size(); i++) { m->mothurOut(outputNames[i]); m->mothurOutEndLine(); }
481 m->mothurOutEndLine();
486 catch(exception& e) {
487 m->errorOut(e, "ChimeraCheckCommand", "execute");
491 //**********************************************************************************************************************
493 int ChimeraCheckCommand::driver(linePair* filePos, string outputFName, string filename){
496 m->openOutputFile(outputFName, out);
501 m->openInputFile(filename, inFASTA);
503 inFASTA.seekg(filePos->start);
510 if (m->control_pressed) { return 1; }
512 Sequence* candidateSeq = new Sequence(inFASTA); m->gobble(inFASTA);
514 if (candidateSeq->getName() != "") { //incase there is a commented sequence at the end of a file
516 chimera->getChimeras(candidateSeq);
518 if (m->control_pressed) { delete candidateSeq; return 1; }
521 chimera->print(out, out2);
525 #if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix)
526 unsigned long long pos = inFASTA.tellg();
527 if ((pos == -1) || (pos >= filePos->end)) { break; }
529 if (inFASTA.eof()) { break; }
533 if((count) % 100 == 0){ m->mothurOut("Processing sequence: " + toString(count)); m->mothurOutEndLine(); }
536 if((count) % 100 != 0){ m->mothurOut("Processing sequence: " + toString(count)); m->mothurOutEndLine(); }
543 catch(exception& e) {
544 m->errorOut(e, "ChimeraCheckCommand", "driver");
548 //**********************************************************************************************************************
550 int ChimeraCheckCommand::driverMPI(int start, int num, MPI_File& inMPI, MPI_File& outMPI, vector<unsigned long long>& MPIPos){
555 MPI_Comm_rank(MPI_COMM_WORLD, &pid); //find out who we are
557 for(int i=0;i<num;i++){
559 if (m->control_pressed) { return 0; }
562 int length = MPIPos[start+i+1] - MPIPos[start+i];
564 char* buf4 = new char[length];
565 MPI_File_read_at(inMPI, MPIPos[start+i], buf4, length, MPI_CHAR, &status);
567 string tempBuf = buf4;
568 if (tempBuf.length() > length) { tempBuf = tempBuf.substr(0, length); }
569 istringstream iss (tempBuf,istringstream::in);
572 Sequence* candidateSeq = new Sequence(iss); m->gobble(iss);
574 if (candidateSeq->getName() != "") { //incase there is a commented sequence at the end of a file
576 chimera->getChimeras(candidateSeq);
579 chimera->print(outMPI, outAccMPI);
584 if((i+1) % 100 == 0){ cout << "Processing sequence: " << (i+1) << endl; m->mothurOutJustToLog("Processing sequence: " + toString(i+1) + "\n"); }
587 if(num % 100 != 0){ cout << "Processing sequence: " << num << endl; m->mothurOutJustToLog("Processing sequence: " + toString(num) + "\n"); }
591 catch(exception& e) {
592 m->errorOut(e, "ChimeraCheckCommand", "driverMPI");
598 /**************************************************************************************************/
600 int ChimeraCheckCommand::createProcesses(string outputFileName, string filename) {
602 #if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix)
606 //loop through and create all the processes you want
607 while (process != processors) {
611 processIDS.push_back(pid); //create map from line number to pid so you can append files in correct order later
614 num = driver(lines[process], outputFileName + toString(getpid()) + ".temp", filename);
616 //pass numSeqs to parent
618 string tempFile = outputFileName + toString(getpid()) + ".num.temp";
619 m->openOutputFile(tempFile, out);
625 m->mothurOut("[ERROR]: unable to spawn the necessary processes."); m->mothurOutEndLine();
626 for (int i = 0; i < processIDS.size(); i++) { kill (processIDS[i], SIGINT); }
631 //force parent to wait until all the processes are done
632 for (int i=0;i<processors;i++) {
633 int temp = processIDS[i];
637 for (int i = 0; i < processIDS.size(); i++) {
639 string tempFile = outputFileName + toString(processIDS[i]) + ".num.temp";
640 m->openInputFile(tempFile, in);
641 if (!in.eof()) { int tempNum = 0; in >> tempNum; num += tempNum; }
642 in.close(); m->mothurRemove(tempFile);
648 catch(exception& e) {
649 m->errorOut(e, "ChimeraCheckCommand", "createProcesses");
653 /**************************************************************************************************/