debbugs.Rnw

   1
   2 \usepackage{fontspec}
   3 \setmainfont{FreeSerif}
   4 \setsansfont{FreeSans}
   5 \setmonofont{FreeMono}
   6 \usepackage{url}
   7 \usepackage{fancyhdr}
   8 \usepackage{graphicx}
   9 \usepackage[bf]{caption}
  10 \usepackage{rotating}
  11 \usepackage{wrapfig}
  12 \usepackage{fancybox}
  13 \usepackage{booktabs}
  14 \usepackage{minted}
  15 \usepackage{tcolorbox}
  16 \usepackage{etoolbox}
  17 \BeforeBeginEnvironment{minted}{\begin{tcolorbox}}%
  18 \AfterEndEnvironment{minted}{\end{tcolorbox}}%
  19 % \usepackage{multirow}
  20 \usepackage{acronym}
  21 \usepackage{qrcode}
  22 \usepackage[backend=biber,natbib=true,hyperref=true,style=nature]{biblatex}
  23 \addbibresource{references.bib}
  24 % \usepackage[nomargin,inline,draft]{fixme}
  25 % \newcommand{\DLA}[1]{\textcolor{red}{\fxnote{DLA: #1}}}
  26 % \usepackage[hyperfigures,bookmarks,colorlinks,citecolor=black,filecolor=black,linkcolor=black,urlcolor=black]{hyperref}
  27 \usepackage{texshade}
  28 \usepackage{tikz}
  29 \usepackage{nameref}
  30 \usepackage{zref-xr,zref-user}
  31 \renewcommand*{\bibfont}{\tiny}
  32 \usepackage[absolute,overlay]{textpos}
  33
  34 \mode<presentation>
  35 {
  36   \usetheme{Montpellier}
  37   %\useoutertheme[hideallsubsections]{debiansidebar}
  38 %  \setbeamercovered{transparent}
  39 }
  40
  41
  42 \usepackage[nomargin,inline,draft]{fixme}
  43 \usepackage{listings}
  44
  45
  46 \logo{\begin{tikzpicture}% Pale figure
  47     {\node[opacity=0.3] {\includegraphics[width=2cm,keepaspectratio]{figures/openlogo-crop.pdf}};}%
  48     \end{tikzpicture}}
  49 \author{Don Armstrong}
  50 \title{Debbugs}
  51 \subtitle{22 Years of Bugs}
  52 \date{August 10th, 2017}
  53 \titlegraphic{\includegraphics[height=0.2\textheight,keepaspectratio]{figures/openlogo-crop.pdf}}
  54 \subject{BTS}
  55
  56 % State of the BTS: new features, changes and tips
  57 %
  58 % The bug tracking system is where all bugs that affect Debian are
  59 % filed. New features, including the affects, summary, and local debbugs
  60 % mirror will be presented and demonstrated. Some of the underlying
  61 % architectural changes to the codebase in debbugs to make it more
  62 % maintainable and deployable are also outlined. Finally, a set of tips
  63 % and tricks that can be used to make the BTS easier to use are
  64 % presented.
  65 %
  66 %  - New Features
  67 %    - affects
  68 %    - summary
  69 %    - local debbugs mirror for offline work
  70 %    - New SOAP
  71 %      - full text search
  72 %      - filtering bugs
  73 %    - multi-distribution/arch status
  74 %    - control changes at submit@ time
  75 %  - Code Architecture Changes
  76 %    - Modularization
  77 %    - Control abstraction
  78 %    - release to experimental
  79
  80
  81 <<load.libraries,echo=FALSE,results="hide",error=FALSE,message=FALSE,cache=FALSE>>=
  82 library(lattice)
  83 library(xtable)
  84 library(ggplot2)
  85 library("scales")
  86 library(reshape2)
  87 library("data.table")
  88 opts_chunk$set(dev="cairo_pdf",out.width="\\textwidth",out.height="0.8\\textheight",out.extra="keepaspectratio",fig.width=6/1.2,fig.height=4/1.2)
  89 opts_chunk$set(cache=TRUE, autodep=TRUE)
  90 options(device = function(file, width = 6/1.2, height = 4/1.2, ...) {
  91   cairo_pdf(tempfile(), width = width, height = height, ...)
  92 })
  93
  94 @
  95
  96 <<scientific_format,echo=FALSE>>=
  97 scientific_10 <- function(x) {
  98   parse(text=gsub("e", " %*% 10^", scientific_format()(x)))
  99 }
 100 @
 101
 102 \begin{document}
 103
 104 %\setbeamercolor{frametitle}[bg=-red!90!green!10,fg=black]
 105
 106 \frame[plain]{\titlepage
 107   Code and slides are here:
 108   \qrcode[padding]{https://dla2.us/p/debbugs2017}
 109   \url{https://dla2.us/p/debbugs2017}
 110
 111 }
 112
 113 % \begin{frame}{Debbugs}
 114 %   \tableofcontents[subsectionstyle=hide]
 115 % \end{frame}
 116
 117 \section{Introduction to Debbugs}
 118
 119 \subsection{Introduction}
 120 \begin{frame}{Goals of this talk}
 121   \begin{columns}
 122     \column{0.5\textwidth}
 123     \begin{itemize}
 124     \item Bug Statistics
 125     \item New features
 126     \item Planned features
 127     \item Places you can help
 128     \end{itemize}
 129     \column{0.5\textwidth}
 130     \includegraphics[width=\textwidth,keepaspectratio]{images/rc_buggy}
 131   \end{columns}
 132 \end{frame}
 133 \begin{frame}{Goals of the BTS}
 134   \begin{columns}
 135   \column{0.4\textwidth}
 136     \includegraphics[width=\textwidth,keepaspectratio]{images/not_a_bug}
 137   \column{0.6\textwidth}
 138   \begin{itemize}
 139   \item Reporting of \only<1>{Features}\only<2>{Bugs}
 140   \item Evolution of \only<1>{Features}\only<2>{Bugs}
 141   \item Fixing of \only<1>{Features}\only<2>{Bugs}
 142   \item Reducing impact of \only<1>{Features}\only<2>{Bugs}
 143   \end{itemize}
 144 \end{columns}
 145 \end{frame}
 146
 147 \section{Bugs in Debian}
 148 \subsection{Bug Reporting Rate}
 149 \begin{frame}{How many bugs do we have?}
 150   \begin{center}
 151 <<bug.growth,fig=TRUE,echo=FALSE,cache.extra=file.info("data/sorted_bug_growth_for_r_every_500.txt")[,"mtime"]>>=
 152 bug.growth <- fread("data/sorted_bug_growth_for_r_every_500.txt")
 153 colnames(bug.growth) <- c("time","bugs")
 154 bug.growth <- bug.growth[pmax(bugs) <= as.numeric(bugs),]
 155 bug.growth <- bug.growth[order(time),]
 156 bug.growth <-
 157     bug.growth[c(0,diff(bugs))>=0,]
 158 bug.growth[,date:=
 159                 as.POSIXct(ISOdatetime(1970,1,1,0,0,0)+
 160                            as.numeric(bug.growth$time))]
 161 bug.growth[,`bugs per day`:=c(0,diff(bugs)/
 162                                 as.double(diff(date),"days"))
 163            ]
 164 print(ggplot(bug.growth,#[date > as.POSIXct(ISOdatetime(2005,1,1,0,0,0)),],
 165              aes(x=date,y=bugs))+
 166       xlab("Time")+ylab("Bugs filed in Debian")+
 167       scale_y_continuous(labels = scientific_10)+
 168       geom_point()+stat_smooth(method="lm"))
 169 @
 170 \end{center}
 171 \end{frame}
 172
 173 \begin{frame}{Bug Growth Rate}
 174 <<bug_growth_rate,echo=FALSE>>=
 175 print(ggplot(bug.growth[date > as.POSIXct(ISOdatetime(2014,1,1,0,0,0)),],
 176              aes(x=date,y=`bugs per day`))+
 177       xlab("Time")+ylab("Bugs filed per day")
 178       + scale_y_log10(breaks=c(60,100,200,400,800))
 179       + geom_line()+stat_smooth(method="lm")
 180       + geom_label(data=data.table(date=as.POSIXct(ISOdatetime(2015,04,25,0,0,0)),"bugs per day"=400),label="Jessie",color="red")
 181       + geom_label(data=data.table(date=as.POSIXct(ISOdatetime(2017,06,17,0,0,0)),"bugs per day"=400),label="Stretch",color="purple")
 182       )
 183 @
 184 \end{frame}
 185
 186 \begin{frame}{My entries into Cristian's game}
 187   \begin{columns}
 188     \column{0.5\textwidth}
 189   \begin{center}
 190 <<bugs.filed,fig=TRUE,echo=FALSE,warning=FALSE,fig.width=3,fig.height=3>>=
 191 print(ggplot(bug.growth[date > as.POSIXct(ISOdatetime(2015,1,1,0,0,0)),],
 192              aes(x=date,y=bugs))+
 193       xlab("Time")+ylab("Bugs filed in Debian")+
 194       scale_y_continuous(labels = scientific_10)+
 195       geom_point()+stat_smooth(method="lm"))
 196 bugs.filed.per.day <-
 197   lm(bugs~date,bug.growth)$coeff[2]*3600*24
 198 temp.lm <- lm(date~bugs,bug.growth[date > as.POSIXct(ISOdatetime(2015,1,1,0,0,0)),])
 199 bug.880000 <-
 200     as.POSIXct(ISOdatetime(1970,1,1,0,0,0)+as.numeric(predict(temp.lm,data.frame(bugs=880000,date=NA))))
 201 bug.900000 <-
 202     as.POSIXct(ISOdatetime(1970,1,1,0,0,0)+as.numeric(predict(temp.lm,data.frame(bugs=900000,date=NA))))
 203 bug.1000000 <-
 204     as.POSIXct(ISOdatetime(1970,1,1,0,0,0)+as.numeric(predict(temp.lm,data.frame(bugs=1000000,date=NA))))
 205 @
 206 \end{center}
 207 \column{0.5\textwidth}
 208 Roughly \Sexpr{format(bugs.filed.per.day,digits=1)} bugs are filed per
 209 day. The 880000th bug will be filed at \Sexpr{bug.880000}, the
 210 900000th bug will be filed at \Sexpr{bug.900000}, and bug one million will be filled
 211 \Sexpr{bug.1000000}.
 212 \end{columns}
 213 \end{frame}
 214
 215
 216 \subsection{Bug Fixing Rate}
 217 \begin{frame}
 218   \begin{center}
 219 <<bugs.closed,fig=TRUE,echo=FALSE,width=7,height=5,warning=FALSE,error=FALSE,cache.extra=file.info("data/bug_closed_time_series.txt")[,"mtime"]>>=
 220 bug.closed.series <- fread(file="data/bug_closed_time_series.txt")
 221 colnames(bug.closed.series) <- c("archived.bugs","year.week")
 222 bug.closed.series[,week:=
 223     gsub("(\\d+)-(\\d+)","\\2",year.week)]
 224 bug.closed.series[,year:=
 225     gsub("(\\d+)-(\\d+)","\\1",year.week)]
 226 bug.closed.series[,doy:= as.numeric(week)*7]
 227 bug.closed.series[,year.doy:=
 228                        paste(sep="-",year,doy)]
 229 bug.closed.series[,date:=
 230                        as.POSIXct(strptime(year.doy,
 231                                            format="%Y-%j"))]
 232
 233 # bug.closed.ts <- ts(bug.closed.series[,1],start=1,frequency=7)
 234
 235 print(ggplot(bug.closed.series[date > as.POSIXct(ISOdatetime(2008,1,1,0,0,0)),],
 236              aes(x=date,y=archived.bugs/7))+geom_line()+stat_smooth(method="lm")+
 237       ylab("Bugs archived Per Day")+xlab("Time"))
 238 bugs.closed.per.day <-
 239     bug.closed.series[,sum(archived.bugs)]/
 240     as.numeric(bug.closed.series[nrow(bug.closed.series),date]-
 241                bug.closed.series[1,date])
 242 @
 243   \end{center}
 244 Roughly \Sexpr{format(bugs.closed.per.day,digits=1)} bugs are closed per day.
 245 \end{frame}
 246
 247 \subsection{RC Bugs}
 248 \begin{frame}{RC Bugs in the Past Year}
 249   \begin{center}
 250 <<rc.bugs,fig=TRUE,echo=FALSE,warning=FALSE,cache.extra=file.info("data/rc_bugs.txt")[,"mtime"]>>=
 251 rc.bugs <-
 252     data.table(read.table(file="data/rc_bugs.txt",
 253                           header=TRUE,fill=TRUE))
 254 rc.bugs[,date:=
 255              as.POSIXct(strptime(date,
 256                                  format="%Y%m%d%H%M"))]
 257 rc.bugs[,unknown:=NULL]
 258 rc.bugs[,unknown.1:=NULL]
 259 for (i in 1:10) {
 260     rc.bugs <- rc.bugs[c(0,diff(total)) > -1000,]
 261 }
 262 rc.bugs.long <-
 263     data.table(melt(rc.bugs,id="date"))
 264 print(ggplot(rc.bugs.long[date >
 265                           as.POSIXct(ISOdatetime(2016,08,1,0,0,0)),]
 266             ,aes(x=date,y=value,color=variable))+
 267       geom_line()+
 268       ylab("# of Release Critical Bugs")+xlab("Time")+
 269       theme(legend.position="top")+
 270       scale_color_discrete("Measure"))
 271 @
 272 \end{center}
 273   \setbeamercolor{postit}{fg=black,bg=yellow}
 274   \begin{textblock}{4}(6,4)
 275     \begin{onlyenv}<2>
 276       \begin{beamercolorbox}[sep=1em,wd=5cm]{postit}
 277         \centering \huge Too many RC bugs!
 278       \end{beamercolorbox}
 279     \end{onlyenv}
 280   \end{textblock}
 281
 282 \end{frame}
 283
 284 \section{Debbugs Structure and Infrastructure}
 285
 286 \subsection{BTS System Diagram}
 287 \begin{frame}{BTS system Diagram}
 288   \begin{center}
 289     \includegraphics[width=\textwidth,height=\textheight,keepaspectratio]{figures/bug_global_layout}
 290   \end{center}
 291 \end{frame}
 292
 293 \subsection{Debbugs Box Diagram}
 294 \begin{frame}{Debbugs Box Diagram}
 295   \begin{center}
 296     \input{debbugs_layout}
 297   \end{center}
 298 \end{frame}
 299
 300 \section{Database Ho!}
 301
 302 \subsection{Overall Database Design}
 303
 304 \begin{frame}{Overall Database Design}
 305   \begin{center}
 306     \input{debbugs_layout_db}
 307   \end{center}
 308 \end{frame}
 309
 310 \subsection{Perl Database Infrastucture}
 311 \begin{frame}{Perl Database Infrastructure}
 312   \begin{itemize}
 313   \item DBIx::Class
 314   \item DBIx::Class::Schema::Loader -- SQL schema to DBIx::Class
 315   \item DBIx::Class::DeploymentHandler -- Upgrades of Schema
 316   \item Debbugs::DB -- Debbugs SQL Module
 317   \end{itemize}
 318 \end{frame}
 319
 320
 321 \subsection{SQL Design}
 322 \begin{frame}{SQL Schema}
 323   \includegraphics[width=\textwidth,keepaspectratio]{figures/schema.png}
 324   \begin{itemize}
 325   \item Current Debbugs SQL Schema
 326   \end{itemize}
 327 \end{frame}
 328
 329 \subsubsection{dak SQL Design}
 330 \begin{frame}{dak SQL Schema}
 331   \includegraphics[width=\textwidth,keepaspectratio]{figures/dak_schema.png}
 332   \begin{itemize}
 333   \item Inspiration taken from dak SQL Schema where appropriate
 334   \end{itemize}
 335 \end{frame}
 336
 337 \subsection{SQL Loading}
 338 \begin{frame}[fragile]{SQL Loading}
 339   % SQL loading
 340   \begin{itemize}
 341   \item Load bugs
 342 \begin{minted}{sh}
 343 debbugs-loadsql bugs;
 344 debbugs-loadsql bugs archive;
 345 \end{minted}
 346   \item Load Versioning information
 347 \begin{minted}{sh}
 348 debbugs-loadsql versions;
 349 debbugs-loadsql debinfo;
 350 \end{minted}
 351   \end{itemize}
 352 \end{frame}
 353
 354 \subsection{SQL Working}
 355 \begin{frame}[fragile]{SQL Working}
 356   % example SQL query
 357 \begin{lstlisting}[language=SQL]
 358 SELECT count(*) FROM bug
 359 WHERE last_modified > '2014-07-01'
 360 AND done IS NOT NULL
 361 AND owner IS NOT NULL;
 362 \end{lstlisting}
 363 \begin{visibleenv}<2>
 364 \begin{lstlisting}
 365  count
 366 -------
 367    521
 368 \end{lstlisting}
 369 \end{visibleenv}
 370 \end{frame}
 371
 372 \subsection{Work still needed for SQL}
 373 \begin{frame}{Work Still needed for SQL}
 374 \begin{itemize}
 375 \item Logfile loading (for full text searching)
 376 \item Status Caching (for faster page loading)
 377 \item Testing
 378 \item Deployment
 379 \end{itemize}
 380 \end{frame}
 381
 382 \section{New(ish) Features}
 383
 384 \subsection{New Changes}
 385 \begin{frame}{New Changes}
 386   \begin{columns}
 387     \column{0.5\textwidth}
 388     \begin{itemize}
 389   \item mailto links (demo)
 390   \item forcemerge does the right thing
 391   \item control at submit time
 392   \end{itemize}
 393     \column{0.5\textwidth}
 394     \includegraphics[width=\textwidth,keepaspectratio]{images/telegraph_bug}
 395 \end{columns}
 396 \end{frame}
 397
 398 \section{Future features}
 399
 400 \subsection{Status Caching}
 401 \begin{frame}{Status Caching}
 402   \begin{itemize}
 403   \item Cache status of bugs (presence/absence at all distributions)
 404   \item Speed up display of \url{http://bugs.debian.org/src:linux-2.6}
 405   \item Allows for reverse status lookup
 406     \begin{itemize}
 407     \item Enables finding bugs which are present in testing or unstable
 408     \end{itemize}
 409   \end{itemize}
 410 \end{frame}
 411
 412 \subsection{Statistics}
 413 \begin{frame}{Statistics}
 414   \begin{columns}
 415     \column{0.5\textwidth}
 416     \begin{itemize}
 417     \item Track status changes over time
 418     \item Bugs found/fixed/absent in stable/testing/unstable over time
 419     \item Bug submission times
 420     \item Bug closure times
 421     \item Bug mail rates
 422     \item Per package, severity, maintainer tracking of the above
 423     \end{itemize}
 424     \column{0.5\textwidth}
 425     \begin{center}
 426       \includegraphics[width=\textwidth,height=0.35\textheight,keepaspectratio]{images/bug}
 427     \end{center}
 428   \end{columns}
 429 \end{frame}
 430
 431
 432 \subsection{Additional Planned Features}
 433 \begin{frame}{More planned features}
 434   \begin{columns}
 435     \column{0.5\textwidth}
 436     \begin{itemize}
 437     \item HTTP based reporting system with e-mail roundtrip
 438     \item Release Debbugs again
 439     \item Bug mailing lists in Debbugs
 440       \begin{itemize}
 441       \item Avoid duplicate mail
 442       \item Opt in/out of mail
 443       \end{itemize}
 444     \item Merging of merged bug reports
 445     \item Threading in report
 446     \end{itemize}
 447     \column{0.5\textwidth}
 448     \includegraphics[width=\textwidth,keepaspectratio]{images/fixed_bug}
 449   \end{columns}
 450 \end{frame}
 451
 452 \begin{frame}{Even More planned features}
 453   \begin{columns}
 454     \column{0.5\textwidth}
 455     \begin{itemize}
 456     \item Usercategory duplication and replay
 457     \item Remote Attachments
 458     \item New Spool Storage Format
 459     \item Usertags visibility
 460     \item Smarter CGI options
 461     \end{itemize}
 462     \column{0.5\textwidth}
 463     \includegraphics[width=\textwidth,keepaspectratio]{images/fixed_bug}
 464   \end{columns}
 465 \end{frame}
 466
 467
 468
 469
 470
 471 \section{Places you can help}
 472
 473 \begin{frame}{7 tasks seeking contributors}
 474   \begin{itemize}
 475   \item Documentation of Usercategories
 476   \item Implementation of rss feeds for packages and bugs
 477   \item CGI options on pkgreport.cgi for usertags et al. \#536378)
 478   \item Documentation of multipe-package reassign and when it or
 479     affects should be used
 480   \item Mailto link with subject, references, etc. all filled in
 481   \item Documentation for local-debbugs configuration file (behaves
 482     like bts select)
 483   \end{itemize}
 484 \end{frame}
 485
 486 \begin{frame}{Getting started on these tasks}
 487   \begin{itemize}
 488   \item Upstream branches: \url{http://bugs.debian.org/debbugs-source/}
 489     \begin{itemize}
 490     \item master (upstream debbugs): git clone \url{http://bugs.debian.org/debbugs-source/debbugs.git}
 491     \item debian (debian branch): git clone -b debian \url{http://bugs.debian.org/debbugs-source/debbugs.git}
 492     \item Checked out branches are also available at
 493       \url{http://bugs.debian.org/debbugs-source/master} and
 494       \url{http://bugs.debian.org/debbugs-source/debian}
 495     \end{itemize}
 496   \item My branches: \url{http://git.donarmstrong.com/debbugs.git}
 497   \item Mailing list debian-debbugs@lists.debian.org
 498   \item IRC \#debbugs or \#debian-bugs on irc.debian.org; I'm dondelelcaro
 499   \end{itemize}
 500 \end{frame}
 501
 502 \begin{frame}{The Debbugs Team}
 503   \begin{itemize}
 504   \item Current team
 505     \begin{itemize}
 506     \item Don Armstrong
 507     \item Blars Blarson
 508     \item Colin Watson
 509     \end{itemize}
 510   \item Emeritus Developers
 511     \begin{itemize}
 512     \item Steiner Gunderson
 513     \item Adam Heath
 514     \item Josip Rodin
 515     \item Anthony Towns
 516     \end{itemize}
 517   \item You!
 518   \end{itemize}
 519 \end{frame}
 520
 521 \begin{frame}{Questions?}
 522   \includegraphics[width=0.8\textwidth,height=0.8\textheight,keepaspectratio]{images/don_at_burning_man.jpg}
 523 \end{frame}
 524
 525 \end{document}
 526
 527