makecontigscommand.h

   1 #ifndef Mothur_makecontigscommand_h
   2 #define Mothur_makecontigscommand_h
   3
   4 //
   5 //  makecontigscommand.h
   6 //  Mothur
   7 //
   8 //  Created by Sarah Westcott on 5/15/12.
   9 //  Copyright (c) 2012 Schloss Lab. All rights reserved.
  10 //
  11
  12 #include "command.hpp"
  13 #include "sequence.hpp"
  14 #include "qualityscores.h"
  15 #include "alignment.hpp"
  16 #include "gotohoverlap.hpp"
  17 #include "needlemanoverlap.hpp"
  18 #include "blastalign.hpp"
  19 #include "noalign.hpp"
  20 #include "trimoligos.h"
  21
  22 struct fastqRead {
  23         vector<int> scores;
  24         string name;
  25         string sequence;
  26
  27         fastqRead() { name = ""; sequence = ""; scores.clear(); };
  28         fastqRead(string n, string s, vector<int> sc) : name(n), sequence(s), scores(sc) {};
  29         ~fastqRead() {};
  30 };
  31
  32 struct pairFastqRead {
  33         fastqRead forward;
  34     fastqRead reverse;
  35
  36         pairFastqRead() {};
  37         pairFastqRead(fastqRead f, fastqRead r) : forward(f), reverse(r){};
  38         ~pairFastqRead() {};
  39 };
  40 /**************************************************************************************************/
  41
  42 class MakeContigsCommand : public Command {
  43 public:
  44     MakeContigsCommand(string);
  45     MakeContigsCommand();
  46     ~MakeContigsCommand(){}
  47
  48     vector<string> setParameters();
  49     string getCommandName()                     { return "make.contigs";                        }
  50     string getCommandCategory()         { return "Sequence Processing";         }
  51     //commmand category choices: Sequence Processing, OTU-Based Approaches, Hypothesis Testing, Phylotype Analysis, General, Clustering and Hidden
  52
  53         string getHelpString();
  54     string getOutputPattern(string);
  55     string getCitation() { return "http://www.mothur.org/wiki/Make.contigs"; }
  56     string getDescription()             { return "description"; }
  57
  58     int execute();
  59     void help() { m->mothurOut(getHelpString()); }
  60
  61 private:
  62     bool abort, allFiles, trimOverlap, createFileGroup, createOligosGroup, makeCount;
  63     string outputDir, ffastqfile, rfastqfile, align, oligosfile, rfastafile, ffastafile, rqualfile, fqualfile, file, format;
  64         float match, misMatch, gapOpen, gapExtend;
  65         int processors, longestBase, insert, tdiffs, bdiffs, pdiffs, ldiffs, sdiffs, deltaq;
  66     vector<string> outputNames;
  67
  68     map<int, oligosPair> barcodes;
  69         map<int, oligosPair> primers;
  70     vector<string>  linker;
  71     vector<string>  spacer;
  72         vector<string> primerNameVector;
  73         vector<string> barcodeNameVector;
  74         vector<char> convertTable;
  75
  76         map<string, int> groupCounts;
  77     map<string, string> groupMap;
  78     map<int, string> file2Group;
  79
  80     vector<int> convertQual(string);
  81     fastqRead readFastq(ifstream&, bool&);
  82     vector< vector< vector<string> > > preProcessData(unsigned long int&);
  83     vector< vector<string> > readFileNames(string);
  84     vector< vector<string> > readFastqFiles(unsigned long int&, string, string);
  85     vector< vector<string> > readFastaFiles(unsigned long int&, string, string);
  86     //bool checkReads(fastqRead&, fastqRead&, string, string);
  87     int createProcesses(vector< vector<string> >, string, string, string, vector<vector<string> >, int);
  88     int driver(vector<string>, string, string, string, vector<vector<string> >, int, string);
  89     bool getOligos(vector<vector<string> >&, string);
  90     string reverseOligo(string);
  91     vector<pairFastqRead> getReads(bool ignoref, bool ignorer, fastqRead forward, fastqRead reverse, map<string, fastqRead>& uniques);
  92 };
  93
  94 /**************************************************************************************************/
  95
  96 /**************************************************************************************************/
  97 //custom data structure for threads to use.
  98 // This is passed by void pointer so it can be any data type
  99 // that can be passed using a single void pointer (LPVOID).
 100 struct contigsData {
 101         string outputFasta;
 102     string outputScrapFasta;
 103         string outputMisMatches;
 104         string align, group;
 105     vector<string> files;
 106     vector<vector<string> > fastaFileNames;
 107         MothurOut* m;
 108         float match, misMatch, gapOpen, gapExtend;
 109         int count, insert, threadID, pdiffs, bdiffs, tdiffs, deltaq;
 110     bool allFiles, createOligosGroup, createFileGroup, done, trimOverlap;
 111     map<string, int> groupCounts;
 112     map<string, string> groupMap;
 113     vector<string> primerNameVector;
 114         vector<string> barcodeNameVector;
 115     map<int, oligosPair> barcodes;
 116         map<int, oligosPair> primers;
 117
 118         contigsData(){}
 119         contigsData(string g, vector<string> f, string of, string osf, string om, string al, MothurOut* mout, float ma, float misMa, float gapO, float gapE, int thr, int delt, map<int, oligosPair> br, map<int, oligosPair> pr, vector<vector<string> > ffn, vector<string>bnv, vector<string> pnv, int pdf, int bdf, int tdf, bool cg, bool cfg, bool all, bool to, int tid) {
 120         files = f;
 121                 outputFasta = of;
 122         outputMisMatches = om;
 123         m = mout;
 124                 match = ma;
 125                 misMatch = misMa;
 126                 gapOpen = gapO;
 127                 gapExtend = gapE;
 128         insert = thr;
 129                 align = al;
 130         group = g;
 131                 count = 0;
 132         outputScrapFasta = osf;
 133         fastaFileNames = ffn;
 134         barcodes = br;
 135         primers = pr;
 136         barcodeNameVector = bnv;
 137         primerNameVector = pnv;
 138         pdiffs = pdf;
 139         bdiffs = bdf;
 140         tdiffs = tdf;
 141         allFiles = all;
 142         trimOverlap = to;
 143         createOligosGroup = cg;
 144         createFileGroup = cfg;
 145                 threadID = tid;
 146         deltaq = delt;
 147         done=false;
 148         }
 149 };
 150
 151 /**************************************************************************************************/
 152 #if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix)
 153 #else
 154 static DWORD WINAPI MyContigsThreadFunction(LPVOID lpParam){
 155         contigsData* pDataArray;
 156         pDataArray = (contigsData*)lpParam;
 157
 158         try {
 159         int longestBase = 1000;
 160         Alignment* alignment;
 161         if(pDataArray->align == "gotoh")                        {       alignment = new GotohOverlap(pDataArray->gapOpen, pDataArray->gapExtend, pDataArray->match, pDataArray->misMatch, longestBase);                 }
 162                 else if(pDataArray->align == "needleman")       {       alignment = new NeedlemanOverlap(pDataArray->gapOpen, pDataArray->match, pDataArray->misMatch, longestBase);                            }
 163
 164         pDataArray->count = 0;
 165         string thisffastafile = pDataArray->files[0];
 166         string thisfqualfile = pDataArray->files[1];
 167         string thisrfastafile = pDataArray->files[2];
 168         string thisrqualfile = pDataArray->files[3];
 169
 170         if (pDataArray->m->debug) {  pDataArray->m->mothurOut("[DEBUG]: ffasta = " + thisffastafile + ".\n[DEBUG]: fqual = " + thisfqualfile + ".\n[DEBUG]: rfasta = " + thisrfastafile + ".\n[DEBUG]: rqual = " + thisrqualfile + ".\n"); }
 171
 172                 if(pDataArray->allFiles){
 173                         for (int i = 0; i < pDataArray->fastaFileNames.size(); i++) { //clears old file
 174                                 for (int j = 0; j < pDataArray->fastaFileNames[i].size(); j++) { //clears old file
 175                                         if (pDataArray->fastaFileNames[i][j] != "") {
 176                                                 ofstream temp;
 177                                                 pDataArray->m->openOutputFile(pDataArray->fastaFileNames[i][j], temp);                  temp.close();
 178                                         }
 179                                 }
 180                         }
 181                 }
 182
 183         ifstream inFFasta, inRFasta, inFQual, inRQual;
 184         ofstream outFasta, outMisMatch, outScrapFasta;
 185         pDataArray->m->openInputFile(thisffastafile, inFFasta);
 186         pDataArray->m->openInputFile(thisrfastafile, inRFasta);
 187         if (thisfqualfile != "") {
 188             pDataArray->m->openInputFile(thisfqualfile, inFQual);
 189             pDataArray->m->openInputFile(thisrqualfile, inRQual);
 190         }
 191         pDataArray->m->openOutputFile(pDataArray->outputFasta, outFasta);
 192         pDataArray->m->openOutputFile(pDataArray->outputMisMatches, outMisMatch);
 193         pDataArray->m->openOutputFile(pDataArray->outputScrapFasta, outScrapFasta);
 194
 195         outMisMatch << "Name\tLength\tOverlap_Length\tOverlap_Start\tOverlap_End\tMisMatches\tNum_Ns\n";
 196
 197         TrimOligos trimOligos(pDataArray->pdiffs, pDataArray->bdiffs, 0, 0, pDataArray->primers, pDataArray->barcodes);
 198
 199         while ((!inFFasta.eof()) && (!inRFasta.eof())) {
 200
 201             if (pDataArray->m->control_pressed) { break; }
 202
 203             int success = 1;
 204             string trashCode = "";
 205             int currentSeqsDiffs = 0;
 206
 207             //read seqs and quality info
 208             Sequence fSeq(inFFasta); pDataArray->m->gobble(inFFasta);
 209             Sequence rSeq(inRFasta); pDataArray->m->gobble(inRFasta);
 210             QualityScores* fQual = NULL; QualityScores* rQual = NULL;
 211             if (thisfqualfile != "") {
 212                 fQual = new QualityScores(inFQual); pDataArray->m->gobble(inFQual);
 213                 rQual = new QualityScores(inRQual); pDataArray->m->gobble(inRQual);
 214             }
 215
 216             int barcodeIndex = 0;
 217             int primerIndex = 0;
 218
 219             if(pDataArray->barcodes.size() != 0){
 220                 if (thisfqualfile != "") {
 221                     success = trimOligos.stripBarcode(fSeq, rSeq, *fQual, *rQual, barcodeIndex);
 222                 }else {
 223                     success = trimOligos.stripBarcode(fSeq, rSeq, barcodeIndex);
 224                 }
 225                 if(success > pDataArray->bdiffs)                {       trashCode += 'b';       }
 226                 else{ currentSeqsDiffs += success;  }
 227             }
 228
 229             if(pDataArray->primers.size() != 0){
 230                 if (thisfqualfile != "") {
 231                     success = trimOligos.stripForward(fSeq, rSeq, *fQual, *rQual, primerIndex);
 232                 }else {
 233                     success = trimOligos.stripForward(fSeq, rSeq, primerIndex);
 234                 }
 235                 if(success > pDataArray->pdiffs)                {       trashCode += 'f';       }
 236                 else{ currentSeqsDiffs += success;  }
 237             }
 238
 239             if (currentSeqsDiffs > pDataArray->tdiffs)  {       trashCode += 't';   }
 240
 241             //flip the reverse reads
 242             rSeq.reverseComplement();
 243             if (thisfqualfile != "") { rQual->flipQScores(); }
 244
 245             //pairwise align
 246             alignment->align(fSeq.getUnaligned(), rSeq.getUnaligned());
 247             map<int, int> ABaseMap = alignment->getSeqAAlnBaseMap();
 248             map<int, int> BBaseMap = alignment->getSeqBAlnBaseMap();
 249             fSeq.setAligned(alignment->getSeqAAln());
 250             rSeq.setAligned(alignment->getSeqBAln());
 251             int length = fSeq.getAligned().length();
 252
 253             //traverse alignments merging into one contiguous seq
 254             string contig = "";
 255             int numMismatches = 0;
 256             string seq1 = fSeq.getAligned();
 257             string seq2 = rSeq.getAligned();
 258             vector<int> scores1, scores2;
 259             if (thisfqualfile != "") {
 260                 scores1 = fQual->getQualityScores();
 261                 scores2 = rQual->getQualityScores();
 262                 delete fQual; delete rQual;
 263             }
 264
 265             int overlapStart = fSeq.getStartPos();
 266             int seq2Start = rSeq.getStartPos();
 267             //bigger of the 2 starting positions is the location of the overlapping start
 268             if (overlapStart < seq2Start) { //seq2 starts later so take from 0 to seq2Start from seq1
 269                 overlapStart = seq2Start;
 270                 for (int i = 0; i < overlapStart; i++) { contig += seq1[i];  }
 271             }else { //seq1 starts later so take from 0 to overlapStart from seq2
 272                 for (int i = 0; i < overlapStart; i++) {  contig += seq2[i]; }
 273             }
 274
 275             int seq1End = fSeq.getEndPos();
 276             int seq2End = rSeq.getEndPos();
 277             int overlapEnd = seq1End;
 278             if (seq2End < overlapEnd) { overlapEnd = seq2End; }  //smallest end position is where overlapping ends
 279
 280             int oStart = contig.length();
 281             for (int i = overlapStart; i < overlapEnd; i++) {
 282                 if (seq1[i] == seq2[i]) { //match, add base and choose highest score
 283                     contig += seq1[i];
 284                 }else if (((seq1[i] == '.') || (seq1[i] == '-')) && ((seq2[i] != '-') && (seq2[i] != '.'))) { //seq1 is a gap and seq2 is a base, choose seq2, unless quality score for base is below insert. In that case eliminate base
 285                     if (thisfqualfile != "") {
 286                         if (scores2[BBaseMap[i]] < pDataArray->insert) { } //
 287                         else { contig += seq2[i];  }
 288                     }else { contig += seq2[i]; } //with no quality info, then we keep it?
 289                 }else if (((seq2[i] == '.') || (seq2[i] == '-')) && ((seq1[i] != '-') && (seq1[i] != '.'))) { //seq2 is a gap and seq1 is a base, choose seq1, unless quality score for base is below insert. In that case eliminate base
 290                     if (thisfqualfile != "") {
 291                         if (scores1[ABaseMap[i]] < pDataArray->insert) { } //
 292                         else { contig += seq1[i];  }
 293                     }else { contig += seq1[i]; } //with no quality info, then we keep it?
 294                 }else if (((seq1[i] != '-') && (seq1[i] != '.')) && ((seq2[i] != '-') && (seq2[i] != '.'))) { //both bases choose one with better quality
 295                     if (thisfqualfile != "") {
 296                         if (abs(scores1[ABaseMap[i]] - scores2[BBaseMap[i]]) >= pDataArray->deltaq) { //is the difference in qual scores >= deltaq, if yes choose base with higher score
 297                             char c = seq1[i];
 298                             if (scores1[ABaseMap[i]] < scores2[BBaseMap[i]]) { c = seq2[i]; }
 299                             contig += c;
 300                         }else { //if no, base becomes n
 301                             contig += 'N';
 302                         }
 303                         numMismatches++;
 304                     }else { numMismatches++; } //cant decide, so eliminate and mark as mismatch
 305                 }else { //should never get here
 306                     pDataArray->m->mothurOut("[ERROR]: case I didn't think of seq1 = " + toString(seq1[i]) + " and seq2 = " + toString(seq2[i]) + "\n");
 307                 }
 308             }
 309             int oend = contig.length();
 310
 311             if (seq1End < seq2End) { //seq1 ends before seq2 so take from overlap to length from seq2
 312                 for (int i = overlapEnd; i < length; i++) { contig += seq2[i];  }
 313             }else { //seq2 ends before seq1 so take from overlap to length from seq1
 314                 for (int i = overlapEnd; i < length; i++) {  contig += seq1[i]; }
 315             }
 316
 317             if (pDataArray->trimOverlap) { contig = contig.substr(overlapStart-1, oend-oStart); if (contig.length() == 0) { trashCode += "l"; } }
 318
 319             if(trashCode.length() == 0){
 320                 bool ignore = false;
 321                 if (pDataArray->createOligosGroup) {
 322                     if(pDataArray->barcodes.size() != 0){
 323                         string thisGroup = pDataArray->barcodeNameVector[barcodeIndex];
 324                         if (pDataArray->primers.size() != 0) {
 325                             if (pDataArray->primerNameVector[primerIndex] != "") {
 326                                 if(thisGroup != "") {
 327                                     thisGroup += "." + pDataArray->primerNameVector[primerIndex];
 328                                 }else {
 329                                     thisGroup = pDataArray->primerNameVector[primerIndex];
 330                                 }
 331                             }
 332                         }
 333
 334                         if (pDataArray->m->debug) { pDataArray->m->mothurOut(", group= " + thisGroup + "\n"); }
 335
 336                         int pos = thisGroup.find("ignore");
 337                         if (pos == string::npos) {
 338                             pDataArray->groupMap[fSeq.getName()] = thisGroup;
 339
 340                             map<string, int>::iterator it = pDataArray->groupCounts.find(thisGroup);
 341                             if (it == pDataArray->groupCounts.end()) {  pDataArray->groupCounts[thisGroup] = 1; }
 342                             else { pDataArray->groupCounts[it->first] ++; }
 343                         }else { ignore = true; }
 344                     }
 345                 }else if (pDataArray->createFileGroup) {
 346                     int pos = pDataArray->group.find("ignore");
 347                     if (pos == string::npos) {
 348                         pDataArray->groupMap[fSeq.getName()] = pDataArray->group;
 349
 350                         map<string, int>::iterator it = pDataArray->groupCounts.find(pDataArray->group);
 351                         if (it == pDataArray->groupCounts.end()) {      pDataArray->groupCounts[pDataArray->group] = 1; }
 352                         else { pDataArray->groupCounts[it->first]++; }
 353                     }else { ignore = true; }
 354                 }
 355
 356
 357                 if(pDataArray->allFiles && !ignore){
 358                     ofstream output;
 359                     pDataArray->m->openOutputFileAppend(pDataArray->fastaFileNames[barcodeIndex][primerIndex], output);
 360                     output << ">" << fSeq.getName() << endl << contig << endl;
 361                     output.close();
 362                 }
 363
 364                 //output
 365                 outFasta << ">" << fSeq.getName() << endl << contig << endl;
 366                 int numNs = 0;
 367                 for (int i = 0; i < contig.length(); i++) { if (contig[i] == 'N') { numNs++; }  }
 368                 outMisMatch << fSeq.getName() << '\t' << contig.length() << '\t' << (oend-oStart) << '\t' << oStart << '\t' << oend << '\t' << numMismatches << '\t' << numNs << endl;
 369             }else {
 370                 //output
 371                 outScrapFasta << ">" << fSeq.getName() << " | " << trashCode << endl << contig << endl;
 372             }
 373             pDataArray->count++;
 374
 375                         //report progress
 376                         if((pDataArray->count) % 1000 == 0){    pDataArray->m->mothurOut(toString(pDataArray->count)); pDataArray->m->mothurOutEndLine();               }
 377                 }
 378
 379                 //report progress
 380                 if((pDataArray->count) % 1000 != 0){    pDataArray->m->mothurOut(toString(pDataArray->count)); pDataArray->m->mothurOutEndLine();               }
 381
 382         inFFasta.close();
 383         inRFasta.close();
 384         outFasta.close();
 385         outMisMatch.close();
 386         outScrapFasta.close();
 387         if (thisfqualfile != "") {
 388             inFQual.close();
 389             inRQual.close();
 390         }
 391         delete alignment;
 392
 393         pDataArray->done = true;
 394         if (pDataArray->m->control_pressed) {  pDataArray->m->mothurRemove(pDataArray->outputFasta);  pDataArray->m->mothurRemove(pDataArray->outputMisMatches);  pDataArray->m->mothurRemove(pDataArray->outputScrapFasta); }
 395
 396         return 0;
 397
 398         }
 399         catch(exception& e) {
 400                 pDataArray->m->errorOut(e, "AlignCommand", "MyContigsThreadFunction");
 401                 exit(1);
 402         }
 403 }
 404 #endif
 405
 406
 407 #endif