%\usepackage{libertine}
\usepackage{fontspec}
\setmainfont[Ligatures=TeX]{FreeSerif}
\setsansfont[Ligatures=TeX]{FreeSans}
\setmonofont{FreeMono}
\usepackage[Symbols]{ucharclasses}
\newfontfamily{\defaultfont}{FreeSerif}
%\newfontfamily{\symbolfont}{FreeSerif}
\newfontfamily{\sansfont}{FreeSans}
\setDefaultTransitions{\bgroup\defaultfont}{\egroup}
\usepackage{url}
\usepackage{fancyhdr}
\usepackage{graphicx}
\usepackage[bf]{caption}
\usepackage{rotating}
\usepackage{wrapfig}
\usepackage{fancybox}
\usepackage{booktabs}
\usepackage{minted}
\usepackage{tcolorbox}
\usepackage{etoolbox}
\BeforeBeginEnvironment{minted}{\begin{tcolorbox}}%
\AfterEndEnvironment{minted}{\end{tcolorbox}}%
% \usepackage{multirow}
\usepackage{acronym}
\usepackage{qrcode}
\usepackage[backend=biber,natbib=true,hyperref=true,style=nature]{biblatex}
\addbibresource{references.bib}
% \usepackage[nomargin,inline,draft]{fixme}
% \newcommand{\DLA}[1]{\textcolor{red}{\fxnote{DLA: #1}}}
% \usepackage[hyperfigures,bookmarks,colorlinks,citecolor=black,filecolor=black,linkcolor=black,urlcolor=black]{hyperref}
\usepackage{texshade}
\usepackage{tikz}
\usepackage{nameref}
\usepackage{zref-xr,zref-user}
\renewcommand*{\bibfont}{\tiny}
\usepackage[absolute,overlay]{textpos}

\mode<presentation>
{
  \usetheme{Montpellier}
  %\useoutertheme[hideallsubsections]{debiansidebar}
%  \setbeamercovered{transparent}  
}

\setbeamertemplate{navigation symbols}{}%remove navigation symbols

\usepackage[nomargin,inline,draft]{fixme}
\usepackage{listings}


\logo{\begin{tikzpicture}% Pale figure
    {\node[opacity=0.3] {\includegraphics[width=2cm,keepaspectratio]{figures/openlogo-crop.pdf}};}%
    \end{tikzpicture}}
\author{Don Armstrong}
\title{Debbugs}
\subtitle{22 Years of Bugs}
\date{August 10th, 2017}
\titlegraphic{\includegraphics[height=0.2\textheight,keepaspectratio]{figures/openlogo-crop.pdf}}
\subject{BTS}

\setbeamercolor{postit}{fg=black,bg=yellow}

% State of the BTS: new features, changes and tips
% 
% The bug tracking system is where all bugs that affect Debian are
% filed. New features, including the affects, summary, and local debbugs
% mirror will be presented and demonstrated. Some of the underlying
% architectural changes to the codebase in debbugs to make it more
% maintainable and deployable are also outlined. Finally, a set of tips
% and tricks that can be used to make the BTS easier to use are
% presented.
% 
%  - New Features
%    - affects
%    - summary
%    - local debbugs mirror for offline work
%    - New SOAP
%      - full text search
%      - filtering bugs
%    - multi-distribution/arch status
%    - control changes at submit@ time
%  - Code Architecture Changes
%    - Modularization
%    - Control abstraction
%    - release to experimental


<<load.libraries,echo=FALSE,results="hide",error=FALSE,message=FALSE,cache=FALSE>>=
library(lattice)
library(xtable)
library(ggplot2)
library("scales")
library(reshape2)
library("data.table")
opts_chunk$set(dev="cairo_pdf",out.width="\\textwidth",out.height="0.8\\textheight",out.extra="keepaspectratio",fig.width=6/1.2,fig.height=4/1.2)
opts_chunk$set(cache=TRUE, autodep=TRUE)
options(device = function(file, width = 6/1.2, height = 4/1.2, ...) {
  cairo_pdf(tempfile(), width = width, height = height, ...)
})

@ 

<<scientific_format,echo=FALSE>>=
scientific_10 <- function(x) {
  parse(text=gsub("e", " %*% 10^", scientific_format()(x)))
}
@ 

\begin{document}

%\setbeamercolor{frametitle}[bg=-red!90!green!10,fg=black]

\frame[plain]{\titlepage
  Code and slides are here: 
  \qrcode[padding]{https://dla2.us/p/debbugs2017}
  \url{https://dla2.us/p/debbugs2017}

}

% \begin{frame}{Debbugs}
%   \tableofcontents[subsectionstyle=hide]
% \end{frame}

\section{Introduction to Debbugs}

\subsection{Introduction}
\begin{frame}{Goals of this talk}
  \begin{columns}
    \column{0.5\textwidth}
    \begin{itemize}
    \item Bug Statistics
    \item New features
    \item How you can help
    \item Answer questions/discussion
    \end{itemize}
    \column{0.4\textwidth}
    \includegraphics[width=\textwidth,keepaspectratio]{images/rc_buggy}
  \end{columns}
\end{frame}
\begin{frame}{Goals of the BTS}
  \begin{columns}
  \column{0.4\textwidth}
    \includegraphics[width=\textwidth,keepaspectratio]{images/not_a_bug}
  \column{0.6\textwidth}
  \begin{itemize}
  \item Reporting of \only<1>{Features}\only<2>{Bugs}
  \item Evolution of \only<1>{Features}\only<2>{Bugs}
  \item Fixing of \only<1>{Features}\only<2>{Bugs}
  \item Reducing impact of \only<1>{Features}\only<2>{Bugs}
  \end{itemize}
\end{columns}
\end{frame}

\section{Bugs in Debian}
\subsection{Bug Reporting Rate}
\begin{frame}{Bugs from the beginning of time}
  \begin{center}
<<bug.growth,fig=TRUE,echo=FALSE,cache.extra=file.info("data/sorted_bug_growth_for_r_every_500.txt")[,"mtime"]>>=
bug.growth <- fread("data/sorted_bug_growth_for_r_every_500.txt")
colnames(bug.growth) <- c("time","bugs")
bug.growth <- bug.growth[pmax(bugs) <= as.numeric(bugs),]
bug.growth <- bug.growth[order(time),]
while (bug.growth[,any(c(0,diff(bugs)) < 0)]) {
       bug.growth <-
           bug.growth[c(0,diff(bugs))>=0,]
}
bug.growth[,date:=
                as.POSIXct(ISOdatetime(1970,1,1,0,0,0)+
                           as.numeric(bug.growth$time))]
bug.growth[,`bugs per day`:=c(0,diff(bugs)/
                                as.double(diff(date),"days"))
           ]
print(ggplot(bug.growth,#[date > as.POSIXct(ISOdatetime(2005,1,1,0,0,0)),],
             aes(x=date,y=bugs))+
      xlab("Time")+ylab("Bugs filed in Debian")+
      scale_y_continuous(labels = scientific_10)+
      geom_point())
@ 
\end{center}
\end{frame}

\begin{frame}[fragile]{Bug Growth Rate}
<<bug_growth_rate,echo=FALSE>>=
print(ggplot(bug.growth[date > as.POSIXct(ISOdatetime(2014,1,1,0,0,0)),],
             aes(x=date,y=`bugs per day`))+
      xlab("Time")+ylab("Bugs filed per day")
      + scale_y_log10(breaks=c(60,100,200,400,800))
      + geom_line()+stat_smooth(method="lm")
      + geom_label(data=data.table(date=as.POSIXct(ISOdatetime(2015,04,25,0,0,0)),"bugs per day"=400),label="Jessie",color="red")
      + geom_label(data=data.table(date=as.POSIXct(ISOdatetime(2017,06,17,0,0,0)),"bugs per day"=400),label="Stretch",color="purple")
      )
@ 
\end{frame}

\begin{frame}[fragile]{Is the bug filing rate decreasing?}
\tiny
<<bug_growth_rate_over_time>>=
summary(lm(log(`bugs per day`)~date,bug.growth[date > "2014-01-01 PST",]))
@ 
\normalsize
Not significantly decreasing.
\end{frame}

\begin{frame}{My entries into Christian's game}
  \begin{columns}
    \column{0.5\textwidth}
  \begin{center}
<<bugs.filed,fig=TRUE,echo=FALSE,warning=FALSE,fig.width=3,fig.height=3>>=
print(ggplot(bug.growth[date > as.POSIXct(ISOdatetime(2015,1,1,0,0,0)),],
             aes(x=date,y=bugs))+
      xlab("Time")+ylab("Bugs filed in Debian")+
      scale_y_continuous(labels = scientific_10)+
      geom_point()+stat_smooth(method="lm"))
bugs.filed.per.day <-
  lm(bugs~date,bug.growth)$coeff[2]*3600*24
temp.lm <- lm(date~bugs,bug.growth[date > as.POSIXct(ISOdatetime(2015,1,1,0,0,0)),])
bug.880000 <- 
    as.POSIXct(ISOdatetime(1970,1,1,0,0,0)+as.numeric(predict(temp.lm,data.frame(bugs=880000,date=NA))))
bug.900000 <- 
    as.POSIXct(ISOdatetime(1970,1,1,0,0,0)+as.numeric(predict(temp.lm,data.frame(bugs=900000,date=NA))))
bug.1000000 <- 
    as.POSIXct(ISOdatetime(1970,1,1,0,0,0)+as.numeric(predict(temp.lm,data.frame(bugs=1000000,date=NA))))
@ 
\end{center}
\column{0.5\textwidth}
Roughly \Sexpr{format(bugs.filed.per.day,digits=1)} bugs are filed per
day. The 880000th bug will be filed at \Sexpr{bug.880000}, the
900000th bug will be filed at \Sexpr{bug.900000}, and bug one million will be filled 
\Sexpr{bug.1000000}.
\end{columns}
\end{frame}


\subsection{Bug Fixing Rate}
\begin{frame}
  \begin{center}
<<bugs.closed,fig=TRUE,echo=FALSE,width=7,height=5,warning=FALSE,error=FALSE,cache.extra=file.info("data/bug_closed_time_series.txt")[,"mtime"]>>=
bug.closed.series <- fread(file="data/bug_closed_time_series.txt")
colnames(bug.closed.series) <- c("archived.bugs","year.week")
bug.closed.series[,week:=
    gsub("(\\d+)-(\\d+)","\\2",year.week)]
bug.closed.series[,year:=
    gsub("(\\d+)-(\\d+)","\\1",year.week)]
bug.closed.series[,doy:= as.numeric(week)*7]
bug.closed.series[,year.doy:=
                       paste(sep="-",year,doy)]
bug.closed.series[,date:=
                       as.POSIXct(strptime(year.doy,
                                           format="%Y-%j"))]

# bug.closed.ts <- ts(bug.closed.series[,1],start=1,frequency=7)

print(ggplot(bug.closed.series[date > as.POSIXct(ISOdatetime(2008,1,1,0,0,0)),],
             aes(x=date,y=archived.bugs/7))+geom_line()+stat_smooth(method="lm")+
      ylab("Bugs archived Per Day")+xlab("Time"))
bugs.closed.per.day <-
    bug.closed.series[,sum(archived.bugs)]/
    as.numeric(bug.closed.series[nrow(bug.closed.series),date]-
               bug.closed.series[1,date])
@ 
  \end{center}
Roughly \Sexpr{format(bugs.closed.per.day,digits=1)} bugs are closed per day.
\end{frame}

\subsection{RC Bugs}
\begin{frame}{RC Bugs in the Past Year}
  \begin{center}
<<rc.bugs,fig=TRUE,echo=FALSE,warning=FALSE,cache.extra=file.info("data/rc_bugs.txt")[,"mtime"]>>=
rc.bugs <-
    data.table(read.table(file="data/rc_bugs.txt",
                          header=TRUE,fill=TRUE))
rc.bugs[,date:=
             as.POSIXct(strptime(date,
                                 format="%Y%m%d%H%M"))]
rc.bugs[,unknown:=NULL]
rc.bugs[,unknown.1:=NULL]
for (i in 1:10) {
    rc.bugs <- rc.bugs[c(0,diff(total)) > -1000,]
}
rc.bugs.long <-
    data.table(melt(rc.bugs,id="date"))
print(ggplot(rc.bugs.long[date > 
                          as.POSIXct(ISOdatetime(2016,08,1,0,0,0)),]
            ,aes(x=date,y=value,color=variable))+
      geom_line()+
      ylab("# of Release Critical Bugs")+xlab("Time")+
      theme(legend.position="top")+
      scale_color_discrete("Measure"))
@ 
\end{center}
\end{frame}

\section{Debbugs Structure and Infrastructure}

\subsection{BTS System Diagram}
\begin{frame}{BTS system Diagram}
  \begin{center}
    \includegraphics[width=\textwidth,height=\textheight,keepaspectratio]{figures/bug_global_layout}
  \end{center}
\end{frame}

\subsection{Debbugs Box Diagram}
\begin{frame}{Debbugs Box Diagram}
  \begin{center}
    \input{debbugs_layout}
  \end{center}
\end{frame}

\section{New Features}

\subsection{Database}

\subsubsection{Perl Database Infrastucture}
\begin{frame}{Perl Database Infrastructure}
  \begin{itemize}
  \item DBIx::Class
  \item DBIx::Class::Schema::Loader -- SQL schema to DBIx::Class
  \item DBIx::Class::DeploymentHandler -- Upgrades of Schema
  \item Debbugs::DB -- Debbugs SQL Module
  \end{itemize}
\end{frame}


\subsubsection{SQL Loading}
\begin{frame}[fragile]{SQL Loading}
  % SQL loading
  \begin{itemize}
  \item Load bugs
\begin{minted}{sh}
debbugs-loadsql bugs;
debbugs-loadsql bugs archive;
\end{minted}
  \item Load Versioning information
\begin{minted}{sh}
debbugs-loadsql versions;
debbugs-loadsql debinfo;
\end{minted}
  \end{itemize}
\end{frame}

\subsubsection{Fun Queries}

\begin{frame}[fragile]{Messages to bugs}
\small
\begin{minted}{sql}
SELECT count(*),c.addr FROM 
message_correspondent mc 
JOIN correspondent c ON mc.correspondent=c.id 
WHERE correspondent_type='from' 
GROUP BY mc.correspondent,c.addr 
ORDER BY count DESC LIMIT 5;
\end{minted}
\begin{verbatim}
 count |              addr
-------+---------------------------------
  5123 | bubulle@debian.org
  4346 | joeyh@debian.org
  4214 | biebl@debian.org
  3875 | tbm@cyrius.com
  3632 | ftpmaster@ftp-master.debian.org
\end{verbatim}
\end{frame}

\begin{frame}[fragile]{Single-message correspondents}
\small
\begin{minted}{sql}
SELECT count (*) FROM 
(SELECT count(*),c.addr 
 FROM message_correspondent mc 
 JOIN correspondent c ON 
 mc.correspondent=c.id 
 WHERE correspondent_type='from' 
 GROUP BY mc.correspondent,
 c.addr 
 HAVING count(*) = 1) AS foo;
\end{minted}
\begin{verbatim}
 count
-------
 30215
\end{verbatim}
\end{frame}



\begin{frame}[fragile]{More fun queries}
\begin{minted}{sql}
SELECT count(*),t.tag 
FROM bug b 
JOIN bug_tag bt ON b.id=bt.bug 
JOIN tag t on bt.tag=t.id 
GROUP by bt.tag,t.tag 
ORDER by count DESC LIMIT 5;
\end{minted}
\begin{verbatim}
 count  |      tag
--------+----------------
 141184 | patch
  42288 | upstream
  39466 | sid
  33640 | l10n
  25566 | fixed-upstream
\end{verbatim}
\end{frame}

\subsection{Database status caching}
\begin{frame}[fragile]{Status caching ``working''}
\begin{minted}{sql}
SELECT count(*),status 
FROM bug_status_cache bsc 
JOIN suite s ON bsc.suite=s.id 
WHERE s.suite_name='unstable' 
GROUP by status;
\end{minted}
\begin{verbatim}
 count  | status
--------+--------
 514488 | fixed
 234155 | absent
   3923 | found
\end{verbatim}
\end{frame}

\subsection{Work still needed for SQL}
\begin{frame}{Work Still needed for SQL}
\begin{itemize}
\item Integration into CGI (\texttt{bugreport.cgi} and
  \texttt{pkgreport.cgi})
\item Database loading/update is slow in some cases
\item Some corrupted bugs have been exposed by the loading (#167851
  and #280931); likely others
\item Testing (Mainly of queries)
\item Deployment
\end{itemize}
\end{frame}

\section{New(ish) Features}

\subsection{New Changes}
\begin{frame}{New Changes}
  \begin{columns}
    \column{0.5\textwidth}
    \begin{itemize}
  \item Format-flowed messages are wrapped
  \item https links everywhere
  \item etag caching support everywhere
  \item a11y (accessibility, ♿) tag
  \item newcomer (⎈) tag
  \item Usercategories now selectable (Categorize/order using)
  \end{itemize}
    \column{0.5\textwidth}
    \includegraphics[width=\textwidth,keepaspectratio]{images/telegraph_bug}
\end{columns}
\end{frame}

\section{Contributing to Debbugs}

\begin{frame}{7 tasks seeking contributors}
  \begin{itemize}
  \item Documentation of Usercategories
  \item Implementation of rss feeds for packages and bugs
  \item CGI options on pkgreport.cgi for usertags et al. \#536378)
  \item Documentation of multiple-package reassign and when it or
    affects should be used
  \item Documentation for local-debbugs configuration file (behaves
    like bts select)
  \end{itemize}
\end{frame}

\begin{frame}{Getting started on these tasks}
  \begin{itemize}
  \item Upstream branches: \url{https://bugs.debian.org/debbugs-source/}
    \begin{itemize}
    \item master (upstream debbugs): git clone \url{https://bugs.debian.org/debbugs-source/debbugs.git}
    \item debian (debian branch): git clone -b debian \url{https://bugs.debian.org/debbugs-source/debbugs.git}
    \item Checked out branches are also available at
      \url{https://bugs.debian.org/debbugs-source/master} and
      \url{https://bugs.debian.org/debbugs-source/debian}
    \end{itemize}
  \item My branches: \url{https://git.donarmstrong.com/debbugs.git}
  \item Mailing list debian-debbugs@lists.debian.org
  \item IRC \#debbugs or \#debian-bugs on irc.debian.org; I'm dondelelcaro
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Using \texttt{local-debbugs} to work on these tasks}
\footnotesize
\begin{minted}{sh}
git clone \
 https://bugs.debian.org/debbugs-source/debbugs.git
cd debbugs
# get some bugs
./bin/local-debbugs --mirror
# start a web daemon
./bin/local-debbugs --daemon
# show some of my bugs (use your own search here)
./bin/local-debbugs --search bugs.debian.org
# stop the daemon
./bin/local-debbugs --stop
# hack and repeat
./bin/local-debbugs --daemon
./bin/local-debbugs --show 614351
\end{minted}
\end{frame}

\begin{frame}{The Debbugs Team}
  \begin{itemize}
  \item Current team
    \begin{itemize}
    \item Don Armstrong
    \item Blars Blarson
    \item Colin Watson
    \end{itemize}
  \item Emeritus Developers
    \begin{itemize}
    \item Steiner Gunderson
    \item Adam Heath
    \item Josip Rodin
    \item Anthony Towns
    \end{itemize}
  \item You!
  \end{itemize}
\end{frame}

\begin{frame}{Questions?}
\begin{center}
\includegraphics[height=0.8\textheight,keepaspectratio]{images/relevant_xkcd}
\end{center}
\end{frame}

\end{document}