debbugs.Rnw

   1
   2 \usepackage{fontspec}
   3 \setmainfont{FreeSerif}
   4 \setsansfont{FreeSans}
   5 \setmonofont{FreeMono}
   6 \usepackage{url}
   7 \usepackage{fancyhdr}
   8 \usepackage{graphicx}
   9 \usepackage[bf]{caption}
  10 \usepackage{rotating}
  11 \usepackage{wrapfig}
  12 \usepackage{fancybox}
  13 \usepackage{booktabs}
  14 % \usepackage{multirow}
  15 \usepackage{acronym}
  16 \usepackage{qrcode}
  17 \usepackage[backend=biber,natbib=true,hyperref=true,style=nature]{biblatex}
  18 \addbibresource{references.bib}
  19 % \usepackage[nomargin,inline,draft]{fixme}
  20 % \newcommand{\DLA}[1]{\textcolor{red}{\fxnote{DLA: #1}}}
  21 % \usepackage[hyperfigures,bookmarks,colorlinks,citecolor=black,filecolor=black,linkcolor=black,urlcolor=black]{hyperref}
  22 \usepackage{texshade}
  23 \usepackage{tikz}
  24 \usepackage{nameref}
  25 \usepackage{zref-xr,zref-user}
  26 \renewcommand*{\bibfont}{\tiny}
  27 \usepackage[absolute,overlay]{textpos}
  28
  29 \mode<presentation>
  30 {
  31   \usetheme{Montpellier}
  32   %\useoutertheme[hideallsubsections]{debiansidebar}
  33 %  \setbeamercovered{transparent}
  34 }
  35
  36
  37 \usepackage[nomargin,inline,draft]{fixme}
  38 \usepackage{listings}
  39
  40
  41 \logo{\begin{tikzpicture}% Pale figure
  42     {\node[opacity=0.3] {\includegraphics[width=2cm,keepaspectratio]{figures/openlogo-crop.pdf}};}%
  43     \end{tikzpicture}}
  44 \author{Don Armstrong}
  45 \title{Debbugs}
  46 \subtitle{22 Years of Bugs}
  47 \date{August 10th, 2017}
  48 \titlegraphic{\includegraphics[height=0.2\textheight,keepaspectratio]{figures/openlogo-crop.pdf}}
  49 \subject{BTS}
  50
  51 % State of the BTS: new features, changes and tips
  52 %
  53 % The bug tracking system is where all bugs that affect Debian are
  54 % filed. New features, including the affects, summary, and local debbugs
  55 % mirror will be presented and demonstrated. Some of the underlying
  56 % architectural changes to the codebase in debbugs to make it more
  57 % maintainable and deployable are also outlined. Finally, a set of tips
  58 % and tricks that can be used to make the BTS easier to use are
  59 % presented.
  60 %
  61 %  - New Features
  62 %    - affects
  63 %    - summary
  64 %    - local debbugs mirror for offline work
  65 %    - New SOAP
  66 %      - full text search
  67 %      - filtering bugs
  68 %    - multi-distribution/arch status
  69 %    - control changes at submit@ time
  70 %  - Code Architecture Changes
  71 %    - Modularization
  72 %    - Control abstraction
  73 %    - release to experimental
  74
  75
  76 <<load.libraries,echo=FALSE,results="hide",error=FALSE,message=FALSE>>=
  77 library(lattice)
  78 library(xtable)
  79 library(ggplot2)
  80 library("scales")
  81 library(reshape2)
  82 library("data.table")
  83 opts_chunk$set(dev="cairo_pdf",out.width="\\textwidth",out.height="0.8\\textheight",out.extra="keepaspectratio",fig.width=6,fig.height=4)
  84 opts_chunk$set(cache=TRUE, autodep=TRUE)
  85 options(device = function(file, width = 6, height = 4, ...) {
  86   cairo_pdf(tempfile(), width = width, height = height, ...)
  87 })
  88
  89 @
  90
  91 <<scientific_format,echo=FALSE>>=
  92 scientific_10 <- function(x) {
  93   parse(text=gsub("e", " %*% 10^", scientific_format()(x)))
  94 }
  95 @
  96
  97 \begin{document}
  98
  99 %\setbeamercolor{frametitle}[bg=-red!90!green!10,fg=black]
 100
 101 \frame[plain]{\titlepage
 102   Code and slides are here:
 103   \qrcode[padding]{https://dla2.us/p/debbugs2017}
 104   \url{https://dla2.us/p/debbugs2017}
 105
 106 }
 107
 108 % \begin{frame}{Debbugs}
 109 %   \tableofcontents[subsectionstyle=hide]
 110 % \end{frame}
 111
 112 \section{Introduction to Debbugs}
 113
 114 \subsection{Introduction}
 115 \begin{frame}{Goals of this talk}
 116   \begin{columns}
 117     \column{0.5\textwidth}
 118     \begin{itemize}
 119     \item Bug Statistics
 120     \item New features
 121     \item Planned features
 122     \item Places you can help
 123     \end{itemize}
 124     \column{0.5\textwidth}
 125     \includegraphics[width=\textwidth,keepaspectratio]{images/rc_buggy}
 126   \end{columns}
 127 \end{frame}
 128 \begin{frame}{Goals of the BTS}
 129   \begin{columns}
 130   \column{0.4\textwidth}
 131     \includegraphics[width=\textwidth,keepaspectratio]{images/not_a_bug}
 132   \column{0.6\textwidth}
 133   \begin{itemize}
 134   \item Reporting of \only<1>{Features}\only<2>{Bugs}
 135   \item Evolution of \only<1>{Features}\only<2>{Bugs}
 136   \item Fixing of \only<1>{Features}\only<2>{Bugs}
 137   \item Reducing impact of \only<1>{Features}\only<2>{Bugs}
 138   \end{itemize}
 139 \end{columns}
 140 \end{frame}
 141
 142 \section{Bugs in Debian}
 143 \subsection{Bug Reporting Rate}
 144 \begin{frame}{How many bugs do we have?}
 145   \begin{center}
 146 <<bug.growth,fig=TRUE,echo=FALSE,cache.extra=file.info("data/sorted_bug_growth_for_r_every_500.txt")[,"mtime"]>>=
 147 bug.growth <- fread("data/sorted_bug_growth_for_r_every_500.txt")
 148 colnames(bug.growth) <- c("time","bugs")
 149 bug.growth <- bug.growth[pmax(bugs) <= as.numeric(bugs),]
 150 bug.growth <- bug.growth[order(time),]
 151 bug.growth <-
 152     bug.growth[c(0,diff(bugs))>=0,]
 153 bug.growth[,date:=
 154                 as.POSIXct(ISOdatetime(1970,1,1,0,0,0)+
 155                            as.numeric(bug.growth$time))]
 156 bug.growth[,`bugs per day`:=c(0,diff(bugs)/
 157                                 as.double(diff(date),"days"))
 158            ]
 159 print(ggplot(bug.growth,#[date > as.POSIXct(ISOdatetime(2005,1,1,0,0,0)),],
 160              aes(x=date,y=bugs))+
 161       xlab("Time")+ylab("Bugs filed in Debian")+
 162       scale_y_continuous(labels = scientific_10)+
 163       geom_point()+stat_smooth(method="lm"))
 164 @
 165 \end{center}
 166 \end{frame}
 167
 168 \begin{frame}{Bug Growth Rate}
 169 <<bug_growth_rate,echo=FALSE>>=
 170 print(ggplot(bug.growth[date > as.POSIXct(ISOdatetime(2014,1,1,0,0,0)),],
 171              aes(x=date,y=`bugs per day`))+
 172       xlab("Time")+ylab("Bugs filed per day")
 173       + scale_y_log10(breaks=c(60,100,200,400,800))
 174       + geom_line()+stat_smooth(method="lm")
 175       + geom_label(data=data.table(date=as.POSIXct(ISOdatetime(2015,04,25,0,0,0)),"bugs per day"=400),label="Jessie",color="red")
 176       + geom_label(data=data.table(date=as.POSIXct(ISOdatetime(2017,06,17,0,0,0)),"bugs per day"=400),label="Stretch",color="purple")
 177       )
 178 @
 179 \end{frame}
 180
 181 \begin{frame}{My entries into Cristian's game}
 182   \begin{columns}
 183     \column{0.5\textwidth}
 184   \begin{center}
 185 <<bugs.filed,fig=TRUE,echo=FALSE,warning=FALSE,fig.width=3,fig.height=3>>=
 186 print(ggplot(bug.growth[date > as.POSIXct(ISOdatetime(2015,1,1,0,0,0)),],
 187              aes(x=date,y=bugs))+
 188       xlab("Time")+ylab("Bugs filed in Debian")+
 189       scale_y_continuous(labels = scientific_10)+
 190       geom_point()+stat_smooth(method="lm"))
 191 bugs.filed.per.day <-
 192   lm(bugs~date,bug.growth)$coeff[2]*3600*24
 193 temp.lm <- lm(date~bugs,bug.growth[date > as.POSIXct(ISOdatetime(2015,1,1,0,0,0)),])
 194 bug.880000 <-
 195     as.POSIXct(ISOdatetime(1970,1,1,0,0,0)+as.numeric(predict(temp.lm,data.frame(bugs=880000,date=NA))))
 196 bug.900000 <-
 197     as.POSIXct(ISOdatetime(1970,1,1,0,0,0)+as.numeric(predict(temp.lm,data.frame(bugs=900000,date=NA))))
 198 bug.1000000 <-
 199     as.POSIXct(ISOdatetime(1970,1,1,0,0,0)+as.numeric(predict(temp.lm,data.frame(bugs=1000000,date=NA))))
 200 @
 201 \end{center}
 202 \column{0.5\textwidth}
 203 Roughly \Sexpr{format(bugs.filed.per.day,digits=1)} bugs are filed per
 204 day. The 880000th bug will be filed at \Sexpr{bug.880000}, the
 205 900000th bug will be filed at \Sexpr{bug.900000}, and bug one million will be filled
 206 \Sexpr{bug.1000000}.
 207 \end{columns}
 208 \end{frame}
 209
 210
 211 \subsection{Bug Fixing Rate}
 212 \begin{frame}
 213   \begin{center}
 214 <<bugs.closed,fig=TRUE,echo=FALSE,width=7,height=5,warning=FALSE,error=FALSE,cache.extra=file.info("data/bug_closed_time_series.txt")[,"mtime"]>>=
 215 bug.closed.series <- fread(file="data/bug_closed_time_series.txt")
 216 colnames(bug.closed.series) <- c("archived.bugs","year.week")
 217 bug.closed.series[,week:=
 218     gsub("(\\d+)-(\\d+)","\\2",year.week)]
 219 bug.closed.series[,year:=
 220     gsub("(\\d+)-(\\d+)","\\1",year.week)]
 221 bug.closed.series[,doy:= as.numeric(week)*7]
 222 bug.closed.series[,year.doy:=
 223                        paste(sep="-",year,doy)]
 224 bug.closed.series[,date:=
 225                        as.POSIXct(strptime(year.doy,
 226                                            format="%Y-%j"))]
 227
 228 # bug.closed.ts <- ts(bug.closed.series[,1],start=1,frequency=7)
 229
 230 print(ggplot(bug.closed.series[date > as.POSIXct(ISOdatetime(2008,1,1,0,0,0)),],
 231              aes(x=date,y=archived.bugs/7))+geom_line()+stat_smooth(method="lm")+
 232       ylab("Bugs archived Per Day")+xlab("Time"))
 233 bugs.closed.per.day <-
 234     bug.closed.series[,sum(archived.bugs)]/
 235     as.numeric(bug.closed.series[nrow(bug.closed.series),date]-
 236                bug.closed.series[1,date])
 237 @
 238   \end{center}
 239 Roughly \Sexpr{format(bugs.closed.per.day,digits=1)} bugs are closed per day.
 240 \end{frame}
 241
 242 \subsection{RC Bugs}
 243 \begin{frame}{RC Bugs in the Past Year}
 244   \begin{center}
 245 <<rc.bugs,fig=TRUE,echo=FALSE,warning=FALSE,cache.extra=file.info("data/rc_bugs.txt")[,"mtime"]>>=
 246 rc.bugs <-
 247     data.table(read.table(file="data/rc_bugs.txt",
 248                           header=TRUE,fill=TRUE))
 249 rc.bugs[,date:=
 250              as.POSIXct(strptime(date,
 251                                  format="%Y%m%d%H%M"))]
 252 rc.bugs[,unknown:=NULL]
 253 rc.bugs[,unknown.1:=NULL]
 254 for (i in 1:10) {
 255     rc.bugs <- rc.bugs[c(0,diff(total)) > -1000,]
 256 }
 257 rc.bugs.long <-
 258     data.table(melt(rc.bugs,id="date"))
 259 print(ggplot(rc.bugs.long[date >
 260                           as.POSIXct(ISOdatetime(2016,08,1,0,0,0)),]
 261             ,aes(x=date,y=value,color=variable))+
 262       geom_line()+
 263       ylab("# of Release Critical Bugs")+xlab("Time")+
 264       theme(legend.position="top")+
 265       scale_color_discrete("Measure"))
 266 @
 267 \end{center}
 268   \setbeamercolor{postit}{fg=black,bg=yellow}
 269   \begin{textblock}{4}(6,4)
 270     \begin{onlyenv}<2>
 271       \begin{beamercolorbox}[sep=1em,wd=5cm]{postit}
 272         \centering \huge Too many RC bugs!
 273       \end{beamercolorbox}
 274     \end{onlyenv}
 275   \end{textblock}
 276
 277 \end{frame}
 278
 279 \section{Debbugs Structure and Infrastructure}
 280
 281 \subsection{BTS System Diagram}
 282 \begin{frame}{BTS system Diagram}
 283   \begin{center}
 284     \includegraphics[width=\textwidth,height=\textheight,keepaspectratio]{figures/bug_global_layout}
 285   \end{center}
 286 \end{frame}
 287
 288 \subsection{Debbugs Box Diagram}
 289 \begin{frame}{Debbugs Box Diagram}
 290   \begin{center}
 291     \input{debbugs_layout}
 292   \end{center}
 293 \end{frame}
 294
 295 \section{Database Ho!}
 296
 297 \subsection{Overall Database Design}
 298
 299 \begin{frame}{Overall Database Design}
 300   \begin{center}
 301     \input{debbugs_layout_db}
 302   \end{center}
 303 \end{frame}
 304
 305 \subsection{Perl Database Infrastucture}
 306 \begin{frame}{Perl Database Infrastructure}
 307   \begin{itemize}
 308   \item DBIx::Class
 309   \item DBIx::Class::Schema::Loader -- SQL schema to DBIx::Class
 310   \item DBIx::Class::DeploymentHandler -- Upgrades of Schema
 311   \item Debbugs::DB -- Debbugs SQL Module
 312   \end{itemize}
 313 \end{frame}
 314
 315
 316 \subsection{SQL Design}
 317 \begin{frame}{SQL Schema}
 318   \includegraphics[width=\textwidth,keepaspectratio]{figures/schema.png}
 319   \begin{itemize}
 320   \item Current Debbugs SQL Schema
 321   \end{itemize}
 322 \end{frame}
 323
 324 \subsubsection{dak SQL Design}
 325 \begin{frame}{dak SQL Schema}
 326   \includegraphics[width=\textwidth,keepaspectratio]{figures/dak_schema.png}
 327   \begin{itemize}
 328   \item Inspiration taken from dak SQL Schema where appropriate
 329   \end{itemize}
 330 \end{frame}
 331
 332 \subsection{SQL Loading}
 333 \begin{frame}[fragile]{SQL Loading}
 334   % SQL loading
 335   \begin{itemize}
 336   \item Load bugs
 337 \begin{lstlisting}[language=sh]
 338 debbugs-loadsql bugs;
 339 debbugs-loadsql bugs archive;
 340 \end{lstlisting}
 341   \item Load Versioning information
 342 \begin{lstlisting}[language=sh]
 343 debbugs-loadsql versions;
 344 debbugs-loadsql debinfo;
 345 \end{lstlisting}
 346   \end{itemize}
 347 \end{frame}
 348
 349 \subsection{SQL Working}
 350 \begin{frame}[fragile]{SQL Working}
 351   % example SQL query
 352 \begin{lstlisting}[language=SQL]
 353 SELECT count(*) FROM bug
 354 WHERE last_modified > '2014-07-01'
 355 AND done IS NOT NULL
 356 AND owner IS NOT NULL;
 357 \end{lstlisting}
 358 \begin{visibleenv}<2>
 359 \begin{lstlisting}
 360  count
 361 -------
 362    521
 363 \end{lstlisting}
 364 \end{visibleenv}
 365 \end{frame}
 366
 367 \subsection{Work still needed for SQL}
 368 \begin{frame}{Work Still needed for SQL}
 369 \begin{itemize}
 370 \item Logfile loading (for full text searching)
 371 \item Status Caching (for faster page loading)
 372 \item Testing
 373 \item Deployment
 374 \end{itemize}
 375 \end{frame}
 376
 377 \section{New(ish) Features}
 378
 379 \subsection{New Changes}
 380 \begin{frame}{New Changes}
 381   \begin{columns}
 382     \column{0.5\textwidth}
 383     \begin{itemize}
 384   \item mailto links (demo)
 385   \item forcemerge does the right thing
 386   \item control at submit time
 387   \end{itemize}
 388     \column{0.5\textwidth}
 389     \includegraphics[width=\textwidth,keepaspectratio]{images/telegraph_bug}
 390 \end{columns}
 391 \end{frame}
 392
 393 \section{Future features}
 394
 395 \subsection{Status Caching}
 396 \begin{frame}{Status Caching}
 397   \begin{itemize}
 398   \item Cache status of bugs (presence/absence at all distributions)
 399   \item Speed up display of \url{http://bugs.debian.org/src:linux-2.6}
 400   \item Allows for reverse status lookup
 401     \begin{itemize}
 402     \item Enables finding bugs which are present in testing or unstable
 403     \end{itemize}
 404   \end{itemize}
 405 \end{frame}
 406
 407 \subsection{Statistics}
 408 \begin{frame}{Statistics}
 409   \begin{columns}
 410     \column{0.5\textwidth}
 411     \begin{itemize}
 412     \item Track status changes over time
 413     \item Bugs found/fixed/absent in stable/testing/unstable over time
 414     \item Bug submission times
 415     \item Bug closure times
 416     \item Bug mail rates
 417     \item Per package, severity, maintainer tracking of the above
 418     \end{itemize}
 419     \column{0.5\textwidth}
 420     \begin{center}
 421       \includegraphics[width=\textwidth,height=0.35\textheight,keepaspectratio]{images/bug}
 422     \end{center}
 423   \end{columns}
 424 \end{frame}
 425
 426
 427 \subsection{Additional Planned Features}
 428 \begin{frame}{More planned features}
 429   \begin{columns}
 430     \column{0.5\textwidth}
 431     \begin{itemize}
 432     \item HTTP based reporting system with e-mail roundtrip
 433     \item Release Debbugs again
 434     \item Bug mailing lists in Debbugs
 435       \begin{itemize}
 436       \item Avoid duplicate mail
 437       \item Opt in/out of mail
 438       \end{itemize}
 439     \item Merging of merged bug reports
 440     \item Threading in report
 441     \end{itemize}
 442     \column{0.5\textwidth}
 443     \includegraphics[width=\textwidth,keepaspectratio]{images/fixed_bug}
 444   \end{columns}
 445 \end{frame}
 446
 447 \begin{frame}{Even More planned features}
 448   \begin{columns}
 449     \column{0.5\textwidth}
 450     \begin{itemize}
 451     \item Usercategory duplication and replay
 452     \item Remote Attachments
 453     \item New Spool Storage Format
 454     \item Usertags visibility
 455     \item Smarter CGI options
 456     \end{itemize}
 457     \column{0.5\textwidth}
 458     \includegraphics[width=\textwidth,keepaspectratio]{images/fixed_bug}
 459   \end{columns}
 460 \end{frame}
 461
 462
 463
 464
 465
 466 \section{Places you can help}
 467
 468 \begin{frame}{7 tasks seeking contributors}
 469   \begin{itemize}
 470   \item Documentation of Usercategories
 471   \item Implementation of rss feeds for packages and bugs
 472   \item CGI options on pkgreport.cgi for usertags et al. \#536378)
 473   \item Documentation of multipe-package reassign and when it or
 474     affects should be used
 475   \item Mailto link with subject, references, etc. all filled in
 476   \item Documentation for local-debbugs configuration file (behaves
 477     like bts select)
 478   \end{itemize}
 479 \end{frame}
 480
 481 \begin{frame}{Getting started on these tasks}
 482   \begin{itemize}
 483   \item Upstream branches: \url{http://bugs.debian.org/debbugs-source/}
 484     \begin{itemize}
 485     \item master (upstream debbugs): git clone \url{http://bugs.debian.org/debbugs-source/debbugs.git}
 486     \item debian (debian branch): git clone -b debian \url{http://bugs.debian.org/debbugs-source/debbugs.git}
 487     \item Checked out branches are also available at
 488       \url{http://bugs.debian.org/debbugs-source/master} and
 489       \url{http://bugs.debian.org/debbugs-source/debian}
 490     \end{itemize}
 491   \item My branches: \url{http://git.donarmstrong.com/debbugs.git}
 492   \item Mailing list debian-debbugs@lists.debian.org
 493   \item IRC \#debbugs or \#debian-bugs on irc.debian.org; I'm dondelelcaro
 494   \end{itemize}
 495 \end{frame}
 496
 497 \begin{frame}{The Debbugs Team}
 498   \begin{itemize}
 499   \item Current team
 500     \begin{itemize}
 501     \item Don Armstrong
 502     \item Blars Blarson
 503     \item Colin Watson
 504     \end{itemize}
 505   \item Emeritus Developers
 506     \begin{itemize}
 507     \item Steiner Gunderson
 508     \item Adam Heath
 509     \item Josip Rodin
 510     \item Anthony Towns
 511     \end{itemize}
 512   \item You!
 513   \end{itemize}
 514 \end{frame}
 515
 516 \begin{frame}{Questions?}
 517   \includegraphics[width=0.8\textwidth,height=0.8\textheight,keepaspectratio]{images/don_at_burning_man.jpg}
 518 \end{frame}
 519
 520 \end{document}
 521
 522