5 * Created by Sarah Westcott on 7/22/09.
6 * Copyright 2009 Schloss Lab UMASS Amherst. All rights reserved.
12 //***************************************************************************************************************
13 void DeCalculator::setMask(string m) {
17 if (seqMask.length() != 0) {
18 //whereever there is a base in the mask, save that value is query and subject
19 for (int i = 0; i < seqMask.length(); i++) {
20 if (isalpha(seqMask[i])) {
25 for (int i = 0; i < alignLength; i++) { h.insert(i); }
29 errorOut(e, "DeCalculator", "setMask");
33 //***************************************************************************************************************
34 void DeCalculator::runMask(Sequence* seq) {
37 string q = seq->getAligned();
38 string tempQuery = "";
40 //whereever there is a base in the mask, save that value is query and subject
41 set<int>::iterator setit;
42 for ( setit=h.begin() ; setit != h.end(); setit++ ) {
43 tempQuery += q[*setit];
47 seq->setAligned(tempQuery);
48 seq->setUnaligned(tempQuery);
51 errorOut(e, "DeCalculator", "runMask");
55 //***************************************************************************************************************
56 //num is query's spot in querySeqs
57 void DeCalculator::trimSeqs(Sequence* query, Sequence* subject, map<int, int>& trim) {
60 string q = query->getAligned();
61 string s = subject->getAligned();
64 for (int i = 0; i < q.length(); i++) {
65 //cout << "query = " << q[i] << " subject = " << s[i] << endl;
66 if (isalpha(q[i]) && isalpha(s[i])) { front = i; break; }
68 //cout << endl << endl;
70 for (int i = q.length(); i >= 0; i--) {
71 //cout << "query = " << q[i] << " subject = " << s[i] << endl;
72 if (isalpha(q[i]) && isalpha(s[i])) { back = i; break; }
79 errorOut(e, "DeCalculator", "trimSeqs");
83 //***************************************************************************************************************
84 vector<int> DeCalculator::findWindows(Sequence* query, int front, int back, int& size, int increment) {
89 int cutoff = back - front; //back - front
91 //if window is set to default
92 if (size == 0) { if (cutoff > 1200) { size = 300; }
93 else{ size = (cutoff / 4); } }
94 else if (size > (cutoff / 4)) {
95 mothurOut("You have selected to large a window size for sequence " + query->getName() + ". I will choose an appropriate window size."); mothurOutEndLine();
99 /* string seq = query->getAligned().substr(front, cutoff);
103 for (int l = 0; l < seq.length(); l++) { if (isalpha(seq[l])) { numBases++; } }
104 //cout << "num Bases = " << numBases << endl;
106 win.push_back(front);
107 //cout << front << '\t';
108 //move ahead increment bases at a time until all bases are in a window
110 int totalBases = 0; //used to eliminate window of blanks at end of sequence
112 seq = query->getAligned();
113 for (int m = front; m < (back - size) ; m++) {
115 //count number of bases you see
116 if (isalpha(seq[m])) { countBases++; }
118 //if you have seen enough bases to make a new window
119 if (countBases >= increment) {
120 //total bases is the number of bases in a window already.
121 totalBases += countBases;
122 //cout << "total bases = " << totalBases << endl;
123 win.push_back(m); //save spot in alignment
125 countBases = 0; //reset bases you've seen in this window
128 //no need to continue if all your bases are in a window
129 if (totalBases == numBases) { break; }
133 //get last window if needed
134 if (totalBases < numBases) { win.push_back(back-size); }
135 //cout << endl << endl;
138 //this follows wigeon, but we may want to consider that it chops off the end values if the sequence cannot be evenly divided into steps
139 for (int m = front; m < (back - size) ; m+=increment) { win.push_back(m); }
146 catch(exception& e) {
147 errorOut(e, "DeCalculator", "findWindows");
152 //***************************************************************************************************************
153 vector<float> DeCalculator::calcObserved(Sequence* query, Sequence* subject, vector<int> window, int size) {
158 for (int m = 0; m < window.size(); m++) {
160 string seqFrag = query->getAligned().substr(window[m], size);
161 string seqFragsub = subject->getAligned().substr(window[m], size);
164 for (int b = 0; b < seqFrag.length(); b++) {
165 //if at least one is a base and they are not equal
166 if( (isalpha(seqFrag[b]) || isalpha(seqFragsub[b])) && (seqFrag[b] != seqFragsub[b]) ) { diff++; }
169 //if((!isalpha(seqFrag[b])) && (!isalpha(seqFragsub[b]))) { gaps++; }
172 //percentage of mismatched bases
175 //if the whole fragment is 0 distance = 0
176 //if ((seqFrag.length()-gaps) == 0) { dist = 0.0; }
178 //percentage of mismatched bases
179 //else { dist = diff / (float) (seqFrag.length()-gaps) * 100; }
181 dist = diff / (float) (seqFrag.length()) * 100;
183 temp.push_back(dist);
188 catch(exception& e) {
189 errorOut(e, "DeCalculator", "calcObserved");
193 //***************************************************************************************************************
194 float DeCalculator::calcDist(Sequence* query, Sequence* subject, int front, int back) {
197 //so you only look at the trimmed part of the sequence
198 int cutoff = back - front;
201 //from first startpoint with length back-front
202 string seqFrag = query->getAligned().substr(front, cutoff);
203 string seqFragsub = subject->getAligned().substr(front, cutoff);
206 for (int b = 0; b < seqFrag.length(); b++) {
208 if((!isalpha(seqFrag[b])) && (!isalpha(seqFragsub[b]))) { gaps++; }
209 if (seqFrag[b] != seqFragsub[b]) { diff++; }
212 //if the whole fragment is 0 distance = 0
213 if ((seqFrag.length()-gaps) == 0) { return 0.0; }
215 //percentage of mismatched bases
216 float dist = diff / (float) (seqFrag.length()-gaps) * 100;
220 catch(exception& e) {
221 errorOut(e, "DeCalculator", "calcDist");
226 //***************************************************************************************************************
227 vector<float> DeCalculator::calcExpected(vector<float> qav, float coef) {
231 vector<float> queryExpected;
233 for (int m = 0; m < qav.size(); m++) {
235 float expected = qav[m] * coef;
237 queryExpected.push_back(expected);
240 return queryExpected;
243 catch(exception& e) {
244 errorOut(e, "DeCalculator", "calcExpected");
248 //***************************************************************************************************************
249 float DeCalculator::calcDE(vector<float> obs, vector<float> exp) {
253 float sum = 0.0; //sum = sum from 1 to m of (oi-ei)^2
254 for (int m = 0; m < obs.size(); m++) { sum += ((obs[m] - exp[m]) * (obs[m] - exp[m])); }
256 float de = sqrt((sum / (obs.size() - 1)));
260 catch(exception& e) {
261 errorOut(e, "DeCalculator", "calcDE");
266 //***************************************************************************************************************
268 vector<float> DeCalculator::calcFreq(vector<Sequence*> seqs, string filename) {
272 string freqfile = getRootName(filename) + "freq";
275 openOutputFile(freqfile, outFreq);
277 string length = toString(seqs.size()); //if there are 5000 seqs in the template then set precision to 3
278 int precision = length.length() - 1;
281 outFreq.setf(ios::fixed, ios::floatfield); outFreq.setf(ios::showpoint);
283 //at each position in the sequence
284 for (int i = 0; i < seqs[0]->getAligned().length(); i++) {
286 vector<int> freq; freq.resize(4,0);
289 //find the frequency of each nucleotide
290 for (int j = 0; j < seqs.size(); j++) {
292 char value = seqs[j]->getAligned()[i];
294 if(toupper(value) == 'A') { freq[0]++; }
295 else if(toupper(value) == 'T' || toupper(value) == 'U') { freq[1]++; }
296 else if(toupper(value) == 'G') { freq[2]++; }
297 else if(toupper(value) == 'C') { freq[3]++; }
301 //find base with highest frequency
303 for (int m = 0; m < freq.size(); m++) { if (freq[m] > highest) { highest = freq[m]; } }
305 float highFreq = highest / (float) (seqs.size());
308 Pi = (highFreq - 0.25) / 0.75;
310 //cannot have probability less than 0.
311 if (Pi < 0) { Pi = 0.0; }
313 //saves this for later
314 outFreq << setprecision(precision) << i << '\t' << highFreq << endl;
316 if (h.count(i) > 0) {
326 catch(exception& e) {
327 errorOut(e, "DeCalculator", "calcFreq");
331 //***************************************************************************************************************
332 vector<float> DeCalculator::findQav(vector<int> window, int size, vector<float> probabilityProfile) {
334 vector<float> averages;
336 //for each window find average
337 for (int m = 0; m < window.size(); m++) {
341 //while you are in the window for this sequence
343 for (int j = window[m]; j < (window[m]+size); j++) {
344 average += probabilityProfile[j];
348 average = average / count;
350 //save this windows average
351 averages.push_back(average);
356 catch(exception& e) {
357 errorOut(e, "DeCalculator", "findQav");
361 //********************************************************************************************************************
362 //sorts lowest to highest
363 inline bool compareQuanMembers(quanMember left, quanMember right){
364 return (left.score < right.score);
366 //***************************************************************************************************************
367 //seqs have already been masked
368 vector< vector<quanMember> > DeCalculator::getQuantiles(vector<Sequence*> seqs, vector<int> windowSizesTemplate, int window, vector<float> probProfile, int increment, int start, int end, vector<float>& highestDE) {
370 vector< vector<quanMember> > quan;
372 //percentage of mismatched pairs 1 to 100
376 for(int i = start; i < end; i++){
378 mothurOut("Processing sequence " + toString(i)); mothurOutEndLine();
379 Sequence* query = new Sequence(seqs[i]->getName(), seqs[i]->getAligned());
381 //compare to every other sequence in template
382 for(int j = 0; j < i; j++){
384 Sequence* subject = new Sequence(seqs[j]->getName(), seqs[j]->getAligned());
387 map<int, int>::iterator it;
389 trimSeqs(query, subject, trim);
392 int front = it->first; int back = it->second;
394 //reset window for each new comparison
395 windowSizesTemplate[i] = window;
397 vector<int> win = findWindows(query, front, back, windowSizesTemplate[i], increment);
399 vector<float> obsi = calcObserved(query, subject, win, windowSizesTemplate[i]);
401 vector<float> q = findQav(win, windowSizesTemplate[i], probProfile);
403 float alpha = getCoef(obsi, q);
405 vector<float> exp = calcExpected(q, alpha);
407 float de = calcDE(obsi, exp);
409 float dist = calcDist(query, subject, front, back);
413 quanMember newScore(de, i, j);
415 //dist-1 because vector indexes start at 0.
416 quan[dist-1].push_back(newScore);
419 if (de > highestDE[i]) { highestDE[i] = de; }
420 if(de > highestDE[j]) { highestDE[j] = de; }
431 catch(exception& e) {
432 errorOut(e, "DeCalculator", "getQuantiles");
437 //***************************************************************************************************************
438 //this was going to be used by pintail to increase the sensitivity of the chimera detection, but it wasn't quite right. may want to revisit in the future...
439 vector< vector<float> > DeCalculator::removeObviousOutliers(vector< vector<quanMember> >& quantiles, int num) {
441 vector< vector<float> > quan;
444 /*vector<quanMember> contributions;
445 vector<int> seen; //seen[0] is the number of outliers that template seqs[0] was part of.
449 for (int i = 0; i < quantiles.size(); i++) {
451 //find mean of this quantile score
452 sort(quantiles[i].begin(), quantiles[i].end(), compareQuanMembers);
454 float high = quantiles[i][int(quantiles[i].size() * 0.99)].score;
455 float low = quantiles[i][int(quantiles[i].size() * 0.01)].score;
457 //look at each value in quantiles to see if it is an outlier
458 for (int j = 0; j < quantiles[i].size(); j++) {
460 //is this score between 1 and 99%
461 if ((quantiles[i][j].score > low) && (quantiles[i][j].score < high)) {
464 //add to contributions
465 contributions.push_back(quantiles[i][j]);
466 seen[quantiles[i][j].member1]++;
467 seen[quantiles[i][j].member2]++;
472 //find contributer with most offending score related to it
473 int largestContrib = findLargestContrib(seen);
475 //while you still have guys to eliminate
476 while (contributions.size() > 0) {
478 mothurOut("Removing scores contributed by sequence " + toString(largestContrib) + " in your template file."); mothurOutEndLine();
480 //remove from quantiles all scores that were made using this largestContrib
481 for (int i = 0; i < quantiles.size(); i++) {
482 //cout << "before remove " << quantiles[i].size() << '\t';
483 removeContrib(largestContrib, quantiles[i]);
484 //cout << "after remove " << quantiles[i].size() << endl;
486 //cout << count << " largest contrib = " << largestContrib << endl; count++;
487 //remove from contributions all scores that were made using this largestContrib
488 removeContrib(largestContrib, contributions);
490 //"erase" largestContrib
491 seen[largestContrib] = -1;
493 //get next largestContrib
494 largestContrib = findLargestContrib(seen);
497 **************************************************************************************************
499 vector<int> marked = returnObviousOutliers(quantiles, num);
500 vector<int> copyMarked = marked;
502 //find the 99% of marked
503 sort(copyMarked.begin(), copyMarked.end());
504 int high = copyMarked[int(copyMarked.size() * 0.99)];
505 cout << "high = " << high << endl;
507 for(int i = 0; i < marked.size(); i++) {
508 if (marked[i] > high) {
509 mothurOut("Removing scores contributed by sequence " + toString(marked[i]) + " in your template file."); mothurOutEndLine();
510 for (int i = 0; i < quantiles.size(); i++) {
511 removeContrib(marked[i], quantiles[i]);
519 for (int i = 0; i < quantiles.size(); i++) {
522 if (quantiles[i].size() == 0) {
523 //in case this is not a distance found in your template files
524 for (int g = 0; g < 6; g++) {
529 sort(quantiles[i].begin(), quantiles[i].end(), compareQuanMembers);
532 temp.push_back(quantiles[i][int(quantiles[i].size() * 0.10)].score);
534 temp.push_back(quantiles[i][int(quantiles[i].size() * 0.25)].score);
536 temp.push_back(quantiles[i][int(quantiles[i].size() * 0.5)].score);
538 temp.push_back(quantiles[i][int(quantiles[i].size() * 0.75)].score);
540 temp.push_back(quantiles[i][int(quantiles[i].size() * 0.95)].score);
542 temp.push_back(quantiles[i][int(quantiles[i].size() * 0.99)].score);
552 catch(exception& e) {
553 errorOut(e, "DeCalculator", "removeObviousOutliers");
557 //***************************************************************************************************************
558 //follows Mallard algorythn in paper referenced from mallard class
559 vector<int> DeCalculator::returnObviousOutliers(vector< vector<quanMember> > quantiles, int num) {
561 vector< vector<float> > quan;
564 map<quanMember*, float> contributions; //map of quanMember to distance from high or low - how bad is it.
565 vector<int> marked; //marked[0] is the penalty of template seqs[0]. the higher the penalty the more likely the sequence is chimeric
566 marked.resize(num,0);
569 for (int i = 0; i < quantiles.size(); i++) {
571 //find mean of this quantile score
572 sort(quantiles[i].begin(), quantiles[i].end(), compareQuanMembers);
574 float high = quantiles[i][int(quantiles[i].size() * 0.99)].score;
576 //look at each value in quantiles to see if it is an outlier
577 for (int j = 0; j < quantiles[i].size(); j++) {
579 //is this score between above 99%
580 if (quantiles[i][j].score > high) {
581 //find out how "bad" of an outlier you are - so you can rank the outliers
582 float dist = quantiles[i][j].score - high;
583 contributions[&(quantiles[i][j])] = dist;
585 //penalizing sequences for being in multiple outliers
586 marked[quantiles[i][j].member1]++;
587 marked[quantiles[i][j].member2]++;
592 //find contributer with most offending score related to it
593 vector<quanMember> outliers = sortContrib(contributions);
595 //go through the outliers marking the potential chimeras
596 for (int i = 0; i < outliers.size(); i++) {
598 //who is responsible for this outlying score?
599 //if member1 has greater score mark him
600 //if member2 has greater score mark her
601 //if they are the same mark both
602 if (marked[outliers[i].member1] > marked[outliers[i].member2]) { marked[outliers[i].member1]++; }
603 else if (marked[outliers[i].member2] > marked[outliers[i].member1]) { marked[outliers[i].member2]++; }
604 else if (marked[outliers[i].member2] == marked[outliers[i].member1]) { marked[outliers[i].member2]++; marked[outliers[i].member1]++; }
609 catch(exception& e) {
610 errorOut(e, "DeCalculator", "removeObviousOutliers");
614 //***************************************************************************************************************
615 //put quanMember in the vector based on how far they are from the 99% or 1%. Biggest offenders in front.
616 vector<quanMember> DeCalculator::sortContrib(map<quanMember*, float> quan) {
619 vector<quanMember> newQuan;
620 map<quanMember*, float>::iterator it;
622 while (quan.size() > 0) {
624 map<quanMember*, float>::iterator largest = quan.begin();
626 //find biggest member
627 for (it = quan.begin(); it != quan.end(); it++) {
628 if (it->second > largest->second) { largest = it; }
630 cout << largest->second << '\t' << largest->first->score << '\t' << largest->first->member1 << '\t' << largest->first->member2 << endl;
632 newQuan.push_back(*(largest->first));
641 catch(exception& e) {
642 errorOut(e, "DeCalculator", "sortContrib");
647 //***************************************************************************************************************
648 //used by removeObviousOutliers which was attempt to increase sensitivity of chimera detection...not currently used...
649 /*int DeCalculator::findLargestContrib(vector<int> seen) {
656 for (int i = 0; i < seen.size(); i++) {
658 if (seen[i] > largest) {
664 return largestContribs;
667 catch(exception& e) {
668 errorOut(e, "DeCalculator", "findLargestContribs");
672 //***************************************************************************************************************
673 void DeCalculator::removeContrib(int bad, vector<quanMember>& quan) {
676 vector<quanMember> newQuan;
677 for (int i = 0; i < quan.size(); i++) {
678 if ((quan[i].member1 != bad) && (quan[i].member2 != bad) ) {
679 newQuan.push_back(quan[i]);
686 catch(exception& e) {
687 errorOut(e, "DeCalculator", "removeContrib");
692 //***************************************************************************************************************
693 float DeCalculator::findAverage(vector<float> myVector) {
697 for (int i = 0; i < myVector.size(); i++) { total += myVector[i]; }
699 float average = total / (float) myVector.size();
704 catch(exception& e) {
705 errorOut(e, "DeCalculator", "findAverage");
710 //***************************************************************************************************************
711 float DeCalculator::getCoef(vector<float> obs, vector<float> qav) {
714 //find average prob for this seqs windows
715 float probAverage = findAverage(qav);
717 //find observed average
718 float obsAverage = findAverage(obs);
720 float coef = obsAverage / probAverage;
724 catch(exception& e) {
725 errorOut(e, "DeCalculator", "getCoef");
729 //***************************************************************************************************************