debbugs.Rnw

   1
   2 \usepackage{fontspec}
   3 \setmainfont{FreeSerif}
   4 \setsansfont{FreeSans}
   5 \setmonofont{FreeMono}
   6 \usepackage{url}
   7 \usepackage{fancyhdr}
   8 \usepackage{graphicx}
   9 \usepackage[bf]{caption}
  10 \usepackage{rotating}
  11 \usepackage{wrapfig}
  12 \usepackage{fancybox}
  13 \usepackage{booktabs}
  14 \usepackage{minted}
  15 \usepackage{tcolorbox}
  16 \usepackage{etoolbox}
  17 \BeforeBeginEnvironment{minted}{\begin{tcolorbox}}%
  18 \AfterEndEnvironment{minted}{\end{tcolorbox}}%
  19 % \usepackage{multirow}
  20 \usepackage{acronym}
  21 \usepackage{qrcode}
  22 \usepackage[backend=biber,natbib=true,hyperref=true,style=nature]{biblatex}
  23 \addbibresource{references.bib}
  24 % \usepackage[nomargin,inline,draft]{fixme}
  25 % \newcommand{\DLA}[1]{\textcolor{red}{\fxnote{DLA: #1}}}
  26 % \usepackage[hyperfigures,bookmarks,colorlinks,citecolor=black,filecolor=black,linkcolor=black,urlcolor=black]{hyperref}
  27 \usepackage{texshade}
  28 \usepackage{tikz}
  29 \usepackage{nameref}
  30 \usepackage{zref-xr,zref-user}
  31 \renewcommand*{\bibfont}{\tiny}
  32 \usepackage[absolute,overlay]{textpos}
  33
  34 \mode<presentation>
  35 {
  36   \usetheme{Montpellier}
  37   %\useoutertheme[hideallsubsections]{debiansidebar}
  38 %  \setbeamercovered{transparent}
  39 }
  40
  41 \setbeamertemplate{navigation symbols}{}%remove navigation symbols
  42
  43 \usepackage[nomargin,inline,draft]{fixme}
  44 \usepackage{listings}
  45
  46
  47 \logo{\begin{tikzpicture}% Pale figure
  48     {\node[opacity=0.3] {\includegraphics[width=2cm,keepaspectratio]{figures/openlogo-crop.pdf}};}%
  49     \end{tikzpicture}}
  50 \author{Don Armstrong}
  51 \title{Debbugs}
  52 \subtitle{22 Years of Bugs}
  53 \date{August 10th, 2017}
  54 \titlegraphic{\includegraphics[height=0.2\textheight,keepaspectratio]{figures/openlogo-crop.pdf}}
  55 \subject{BTS}
  56
  57 \setbeamercolor{postit}{fg=black,bg=yellow}
  58
  59 % State of the BTS: new features, changes and tips
  60 %
  61 % The bug tracking system is where all bugs that affect Debian are
  62 % filed. New features, including the affects, summary, and local debbugs
  63 % mirror will be presented and demonstrated. Some of the underlying
  64 % architectural changes to the codebase in debbugs to make it more
  65 % maintainable and deployable are also outlined. Finally, a set of tips
  66 % and tricks that can be used to make the BTS easier to use are
  67 % presented.
  68 %
  69 %  - New Features
  70 %    - affects
  71 %    - summary
  72 %    - local debbugs mirror for offline work
  73 %    - New SOAP
  74 %      - full text search
  75 %      - filtering bugs
  76 %    - multi-distribution/arch status
  77 %    - control changes at submit@ time
  78 %  - Code Architecture Changes
  79 %    - Modularization
  80 %    - Control abstraction
  81 %    - release to experimental
  82
  83
  84 <<load.libraries,echo=FALSE,results="hide",error=FALSE,message=FALSE,cache=FALSE>>=
  85 library(lattice)
  86 library(xtable)
  87 library(ggplot2)
  88 library("scales")
  89 library(reshape2)
  90 library("data.table")
  91 opts_chunk$set(dev="cairo_pdf",out.width="\\textwidth",out.height="0.8\\textheight",out.extra="keepaspectratio",fig.width=6/1.2,fig.height=4/1.2)
  92 opts_chunk$set(cache=TRUE, autodep=TRUE)
  93 options(device = function(file, width = 6/1.2, height = 4/1.2, ...) {
  94   cairo_pdf(tempfile(), width = width, height = height, ...)
  95 })
  96
  97 @
  98
  99 <<scientific_format,echo=FALSE>>=
 100 scientific_10 <- function(x) {
 101   parse(text=gsub("e", " %*% 10^", scientific_format()(x)))
 102 }
 103 @
 104
 105 \begin{document}
 106
 107 %\setbeamercolor{frametitle}[bg=-red!90!green!10,fg=black]
 108
 109 \frame[plain]{\titlepage
 110   Code and slides are here:
 111   \qrcode[padding]{https://dla2.us/p/debbugs2017}
 112   \url{https://dla2.us/p/debbugs2017}
 113
 114 }
 115
 116 % \begin{frame}{Debbugs}
 117 %   \tableofcontents[subsectionstyle=hide]
 118 % \end{frame}
 119
 120 \section{Introduction to Debbugs}
 121
 122 \subsection{Introduction}
 123 \begin{frame}{Goals of this talk}
 124   \begin{columns}
 125     \column{0.5\textwidth}
 126     \begin{itemize}
 127     \item Bug Statistics
 128     \item New features
 129     \item Planned features
 130     \item Places you can help
 131     \end{itemize}
 132     \column{0.5\textwidth}
 133     \includegraphics[width=\textwidth,keepaspectratio]{images/rc_buggy}
 134   \end{columns}
 135 \end{frame}
 136 \begin{frame}{Goals of the BTS}
 137   \begin{columns}
 138   \column{0.4\textwidth}
 139     \includegraphics[width=\textwidth,keepaspectratio]{images/not_a_bug}
 140   \column{0.6\textwidth}
 141   \begin{itemize}
 142   \item Reporting of \only<1>{Features}\only<2>{Bugs}
 143   \item Evolution of \only<1>{Features}\only<2>{Bugs}
 144   \item Fixing of \only<1>{Features}\only<2>{Bugs}
 145   \item Reducing impact of \only<1>{Features}\only<2>{Bugs}
 146   \end{itemize}
 147 \end{columns}
 148 \end{frame}
 149
 150 \section{Bugs in Debian}
 151 \subsection{Bug Reporting Rate}
 152 \begin{frame}{Bugs from the beginning of time}
 153   \begin{center}
 154 <<bug.growth,fig=TRUE,echo=FALSE,cache.extra=file.info("data/sorted_bug_growth_for_r_every_500.txt")[,"mtime"]>>=
 155 bug.growth <- fread("data/sorted_bug_growth_for_r_every_500.txt")
 156 colnames(bug.growth) <- c("time","bugs")
 157 bug.growth <- bug.growth[pmax(bugs) <= as.numeric(bugs),]
 158 bug.growth <- bug.growth[order(time),]
 159 while (bug.growth[,any(c(0,diff(bugs)) < 0)]) {
 160        bug.growth <-
 161            bug.growth[c(0,diff(bugs))>=0,]
 162 }
 163 bug.growth[,date:=
 164                 as.POSIXct(ISOdatetime(1970,1,1,0,0,0)+
 165                            as.numeric(bug.growth$time))]
 166 bug.growth[,`bugs per day`:=c(0,diff(bugs)/
 167                                 as.double(diff(date),"days"))
 168            ]
 169 print(ggplot(bug.growth,#[date > as.POSIXct(ISOdatetime(2005,1,1,0,0,0)),],
 170              aes(x=date,y=bugs))+
 171       xlab("Time")+ylab("Bugs filed in Debian")+
 172       scale_y_continuous(labels = scientific_10)+
 173       geom_point())
 174 @
 175 \end{center}
 176 \end{frame}
 177
 178 \begin{frame}[fragile]{Bug Growth Rate}
 179 <<bug_growth_rate,echo=FALSE>>=
 180 print(ggplot(bug.growth[date > as.POSIXct(ISOdatetime(2014,1,1,0,0,0)),],
 181              aes(x=date,y=`bugs per day`))+
 182       xlab("Time")+ylab("Bugs filed per day")
 183       + scale_y_log10(breaks=c(60,100,200,400,800))
 184       + geom_line()+stat_smooth(method="lm")
 185       + geom_label(data=data.table(date=as.POSIXct(ISOdatetime(2015,04,25,0,0,0)),"bugs per day"=400),label="Jessie",color="red")
 186       + geom_label(data=data.table(date=as.POSIXct(ISOdatetime(2017,06,17,0,0,0)),"bugs per day"=400),label="Stretch",color="purple")
 187       )
 188 @
 189 \end{frame}
 190
 191 \begin{frame}[fragile]{Is the bug filing rate decreasing?}
 192 \tiny
 193 <<bug_growth_rate_over_time>>=
 194 summary(lm(log(`bugs per day`)~date,
 195            bug.growth[date > "2014-01-01 PST",]))
 196 @
 197 \normalsize
 198 Not significantly decreasing.
 199 \end{frame}
 200
 201 \begin{frame}{My entries into Christian's game}
 202   \begin{columns}
 203     \column{0.5\textwidth}
 204   \begin{center}
 205 <<bugs.filed,fig=TRUE,echo=FALSE,warning=FALSE,fig.width=3,fig.height=3>>=
 206 print(ggplot(bug.growth[date > as.POSIXct(ISOdatetime(2015,1,1,0,0,0)),],
 207              aes(x=date,y=bugs))+
 208       xlab("Time")+ylab("Bugs filed in Debian")+
 209       scale_y_continuous(labels = scientific_10)+
 210       geom_point()+stat_smooth(method="lm"))
 211 bugs.filed.per.day <-
 212   lm(bugs~date,bug.growth)$coeff[2]*3600*24
 213 temp.lm <- lm(date~bugs,bug.growth[date > as.POSIXct(ISOdatetime(2015,1,1,0,0,0)),])
 214 bug.880000 <-
 215     as.POSIXct(ISOdatetime(1970,1,1,0,0,0)+as.numeric(predict(temp.lm,data.frame(bugs=880000,date=NA))))
 216 bug.900000 <-
 217     as.POSIXct(ISOdatetime(1970,1,1,0,0,0)+as.numeric(predict(temp.lm,data.frame(bugs=900000,date=NA))))
 218 bug.1000000 <-
 219     as.POSIXct(ISOdatetime(1970,1,1,0,0,0)+as.numeric(predict(temp.lm,data.frame(bugs=1000000,date=NA))))
 220 @
 221 \end{center}
 222 \column{0.5\textwidth}
 223 Roughly \Sexpr{format(bugs.filed.per.day,digits=1)} bugs are filed per
 224 day. The 880000th bug will be filed at \Sexpr{bug.880000}, the
 225 900000th bug will be filed at \Sexpr{bug.900000}, and bug one million will be filled
 226 \Sexpr{bug.1000000}.
 227 \end{columns}
 228 \end{frame}
 229
 230
 231 \subsection{Bug Fixing Rate}
 232 \begin{frame}
 233   \begin{center}
 234 <<bugs.closed,fig=TRUE,echo=FALSE,width=7,height=5,warning=FALSE,error=FALSE,cache.extra=file.info("data/bug_closed_time_series.txt")[,"mtime"]>>=
 235 bug.closed.series <- fread(file="data/bug_closed_time_series.txt")
 236 colnames(bug.closed.series) <- c("archived.bugs","year.week")
 237 bug.closed.series[,week:=
 238     gsub("(\\d+)-(\\d+)","\\2",year.week)]
 239 bug.closed.series[,year:=
 240     gsub("(\\d+)-(\\d+)","\\1",year.week)]
 241 bug.closed.series[,doy:= as.numeric(week)*7]
 242 bug.closed.series[,year.doy:=
 243                        paste(sep="-",year,doy)]
 244 bug.closed.series[,date:=
 245                        as.POSIXct(strptime(year.doy,
 246                                            format="%Y-%j"))]
 247
 248 # bug.closed.ts <- ts(bug.closed.series[,1],start=1,frequency=7)
 249
 250 print(ggplot(bug.closed.series[date > as.POSIXct(ISOdatetime(2008,1,1,0,0,0)),],
 251              aes(x=date,y=archived.bugs/7))+geom_line()+stat_smooth(method="lm")+
 252       ylab("Bugs archived Per Day")+xlab("Time"))
 253 bugs.closed.per.day <-
 254     bug.closed.series[,sum(archived.bugs)]/
 255     as.numeric(bug.closed.series[nrow(bug.closed.series),date]-
 256                bug.closed.series[1,date])
 257 @
 258   \end{center}
 259 Roughly \Sexpr{format(bugs.closed.per.day,digits=1)} bugs are closed per day.
 260 \end{frame}
 261
 262 \subsection{RC Bugs}
 263 \begin{frame}{RC Bugs in the Past Year}
 264   \begin{center}
 265 <<rc.bugs,fig=TRUE,echo=FALSE,warning=FALSE,cache.extra=file.info("data/rc_bugs.txt")[,"mtime"]>>=
 266 rc.bugs <-
 267     data.table(read.table(file="data/rc_bugs.txt",
 268                           header=TRUE,fill=TRUE))
 269 rc.bugs[,date:=
 270              as.POSIXct(strptime(date,
 271                                  format="%Y%m%d%H%M"))]
 272 rc.bugs[,unknown:=NULL]
 273 rc.bugs[,unknown.1:=NULL]
 274 for (i in 1:10) {
 275     rc.bugs <- rc.bugs[c(0,diff(total)) > -1000,]
 276 }
 277 rc.bugs.long <-
 278     data.table(melt(rc.bugs,id="date"))
 279 print(ggplot(rc.bugs.long[date >
 280                           as.POSIXct(ISOdatetime(2016,08,1,0,0,0)),]
 281             ,aes(x=date,y=value,color=variable))+
 282       geom_line()+
 283       ylab("# of Release Critical Bugs")+xlab("Time")+
 284       theme(legend.position="top")+
 285       scale_color_discrete("Measure"))
 286 @
 287 \end{center}
 288 \end{frame}
 289
 290 \section{Debbugs Structure and Infrastructure}
 291
 292 \subsection{BTS System Diagram}
 293 \begin{frame}{BTS system Diagram}
 294   \begin{center}
 295     \includegraphics[width=\textwidth,height=\textheight,keepaspectratio]{figures/bug_global_layout}
 296   \end{center}
 297 \end{frame}
 298
 299 \subsection{Debbugs Box Diagram}
 300 \begin{frame}{Debbugs Box Diagram}
 301   \begin{center}
 302     \input{debbugs_layout}
 303   \end{center}
 304 \end{frame}
 305
 306 \section{New Features}
 307
 308 \subsection{Database}
 309
 310 \subsubsection{Perl Database Infrastucture}
 311 \begin{frame}{Perl Database Infrastructure}
 312   \begin{itemize}
 313   \item DBIx::Class
 314   \item DBIx::Class::Schema::Loader -- SQL schema to DBIx::Class
 315   \item DBIx::Class::DeploymentHandler -- Upgrades of Schema
 316   \item Debbugs::DB -- Debbugs SQL Module
 317   \end{itemize}
 318 \end{frame}
 319
 320
 321 \subsubsection{SQL Loading}
 322 \begin{frame}[fragile]{SQL Loading}
 323   % SQL loading
 324   \begin{itemize}
 325   \item Load bugs
 326 \begin{minted}{sh}
 327 debbugs-loadsql bugs;
 328 debbugs-loadsql bugs archive;
 329 \end{minted}
 330   \item Load Versioning information
 331 \begin{minted}{sh}
 332 debbugs-loadsql versions;
 333 debbugs-loadsql debinfo;
 334 \end{minted}
 335   \end{itemize}
 336 \end{frame}
 337
 338 \subsubsection{Fun Queries}
 339
 340 \begin{frame}[fragile]{Messages to bugs}
 341 \small
 342 \begin{minted}{sql}
 343 SELECT count(*),c.addr FROM
 344 message_correspondent mc
 345 JOIN correspondent c ON mc.correspondent=c.id
 346 WHERE correspondent_type='from'
 347 GROUP BY mc.correspondent,c.addr
 348 ORDER BY count DESC LIMIT 5;
 349 \end{minted}
 350 \begin{verbatim}
 351  count |              addr
 352 -------+---------------------------------
 353   5123 | bubulle@debian.org
 354   4346 | joeyh@debian.org
 355   4214 | biebl@debian.org
 356   3875 | tbm@cyrius.com
 357   3632 | ftpmaster@ftp-master.debian.org
 358 \end{verbatim}
 359 \end{frame}
 360
 361 \begin{frame}[fragile]{Single-message correspondents}
 362 \small
 363 \begin{minted}{sql}
 364 SELECT count (*) FROM
 365 (SELECT count(*),c.addr
 366  FROM message_correspondent mc
 367  JOIN correspondent c ON
 368  mc.correspondent=c.id
 369  WHERE correspondent_type='from'
 370  GROUP BY mc.correspondent,
 371  c.addr
 372  HAVING count(*) = 1) AS foo;
 373 \end{minted}
 374 \begin{verbatim}
 375  count
 376 -------
 377  30215
 378 \end{verbatim}
 379 \end{frame}
 380
 381
 382
 383 \begin{frame}[fragile]{More fun queries}
 384 \begin{minted}{sql}
 385 SELECT count(*),t.tag
 386 FROM bug b
 387 JOIN bug_tag bt ON b.id=bt.bug
 388 JOIN tag t on bt.tag=t.id
 389 GROUP by bt.tag,t.tag
 390 ORDER by count DESC LIMIT 5;
 391 \end{minted}
 392 \begin{verbatim}
 393  count  |      tag
 394 --------+----------------
 395  141184 | patch
 396   42288 | upstream
 397   39466 | sid
 398   33640 | l10n
 399   25566 | fixed-upstream
 400 \end{verbatim}
 401 \end{frame}
 402
 403 \subsection{Database status caching}
 404 \begin{frame}[fragile]{Status caching ``working''}
 405 \begin{minted}{sql}
 406 SELECT count(*),status
 407 FROM bug_status_cache bsc
 408 JOIN suite s ON bsc.suite=s.id
 409 WHERE s.suite_name='unstable'
 410 GROUP by status;
 411 \end{minted}
 412 \begin{verbatim}
 413  count  | status
 414 --------+--------
 415  514488 | fixed
 416  234155 | absent
 417    3923 | found
 418 \end{verbatim}
 419 \end{frame}
 420
 421 \subsection{Work still needed for SQL}
 422 \begin{frame}{Work Still needed for SQL}
 423 \begin{itemize}
 424 \item Integration into CGI
 425 \item Testing
 426 \item Deployment
 427 \end{itemize}
 428 \end{frame}
 429
 430 \section{New(ish) Features}
 431
 432 \subsection{New Changes}
 433 \begin{frame}{New Changes}
 434   \begin{columns}
 435     \column{0.5\textwidth}
 436     \begin{itemize}
 437   \item mailto links (demo)
 438   \item forcemerge does the right thing
 439   \item control at submit time
 440   \end{itemize}
 441     \column{0.5\textwidth}
 442     \includegraphics[width=\textwidth,keepaspectratio]{images/telegraph_bug}
 443 \end{columns}
 444 \end{frame}
 445
 446 \section{Future features}
 447
 448 \subsection{Status Caching}
 449 \begin{frame}{Status Caching}
 450   \begin{itemize}
 451   \item Cache status of bugs (presence/absence at all distributions)
 452   \item Speed up display of \url{http://bugs.debian.org/src:linux-2.6}
 453   \item Allows for reverse status lookup
 454     \begin{itemize}
 455     \item Enables finding bugs which are present in testing or unstable
 456     \end{itemize}
 457   \end{itemize}
 458 \end{frame}
 459
 460 \subsection{Statistics}
 461 \begin{frame}{Statistics}
 462   \begin{columns}
 463     \column{0.5\textwidth}
 464     \begin{itemize}
 465     \item Track status changes over time
 466     \item Bugs found/fixed/absent in stable/testing/unstable over time
 467     \item Bug submission times
 468     \item Bug closure times
 469     \item Bug mail rates
 470     \item Per package, severity, maintainer tracking of the above
 471     \end{itemize}
 472     \column{0.5\textwidth}
 473     \begin{center}
 474       \includegraphics[width=\textwidth,height=0.35\textheight,keepaspectratio]{images/bug}
 475     \end{center}
 476   \end{columns}
 477 \end{frame}
 478
 479
 480 \subsection{Additional Planned Features}
 481 \begin{frame}{More planned features}
 482   \begin{columns}
 483     \column{0.5\textwidth}
 484     \begin{itemize}
 485     \item HTTP based reporting system with e-mail roundtrip
 486     \item Release Debbugs again
 487     \item Bug mailing lists in Debbugs
 488       \begin{itemize}
 489       \item Avoid duplicate mail
 490       \item Opt in/out of mail
 491       \end{itemize}
 492     \item Merging of merged bug reports
 493     \item Threading in report
 494     \end{itemize}
 495     \column{0.5\textwidth}
 496     \includegraphics[width=\textwidth,keepaspectratio]{images/fixed_bug}
 497   \end{columns}
 498 \end{frame}
 499
 500 \begin{frame}{Even More planned features}
 501   \begin{columns}
 502     \column{0.5\textwidth}
 503     \begin{itemize}
 504     \item Usercategory duplication and replay
 505     \item Remote Attachments
 506     \item New Spool Storage Format
 507     \item Usertags visibility
 508     \item Smarter CGI options
 509     \end{itemize}
 510     \column{0.5\textwidth}
 511     \includegraphics[width=\textwidth,keepaspectratio]{images/fixed_bug}
 512   \end{columns}
 513 \end{frame}
 514
 515
 516
 517
 518
 519 \section{Places you can help}
 520
 521 \begin{frame}{7 tasks seeking contributors}
 522   \begin{itemize}
 523   \item Documentation of Usercategories
 524   \item Implementation of rss feeds for packages and bugs
 525   \item CGI options on pkgreport.cgi for usertags et al. \#536378)
 526   \item Documentation of multipe-package reassign and when it or
 527     affects should be used
 528   \item Mailto link with subject, references, etc. all filled in
 529   \item Documentation for local-debbugs configuration file (behaves
 530     like bts select)
 531   \end{itemize}
 532 \end{frame}
 533
 534 \begin{frame}{Getting started on these tasks}
 535   \begin{itemize}
 536   \item Upstream branches: \url{http://bugs.debian.org/debbugs-source/}
 537     \begin{itemize}
 538     \item master (upstream debbugs): git clone \url{http://bugs.debian.org/debbugs-source/debbugs.git}
 539     \item debian (debian branch): git clone -b debian \url{http://bugs.debian.org/debbugs-source/debbugs.git}
 540     \item Checked out branches are also available at
 541       \url{http://bugs.debian.org/debbugs-source/master} and
 542       \url{http://bugs.debian.org/debbugs-source/debian}
 543     \end{itemize}
 544   \item My branches: \url{http://git.donarmstrong.com/debbugs.git}
 545   \item Mailing list debian-debbugs@lists.debian.org
 546   \item IRC \#debbugs or \#debian-bugs on irc.debian.org; I'm dondelelcaro
 547   \end{itemize}
 548 \end{frame}
 549
 550 \begin{frame}{The Debbugs Team}
 551   \begin{itemize}
 552   \item Current team
 553     \begin{itemize}
 554     \item Don Armstrong
 555     \item Blars Blarson
 556     \item Colin Watson
 557     \end{itemize}
 558   \item Emeritus Developers
 559     \begin{itemize}
 560     \item Steiner Gunderson
 561     \item Adam Heath
 562     \item Josip Rodin
 563     \item Anthony Towns
 564     \end{itemize}
 565   \item You!
 566   \end{itemize}
 567 \end{frame}
 568
 569 \begin{frame}{Questions?}
 570   \includegraphics[width=0.8\textwidth,height=0.8\textheight,keepaspectratio]{images/don_at_burning_man.jpg}
 571 \end{frame}
 572
 573 \end{document}
 574
 575