\mode { \usetheme[hideallsubsections]{Hannover} %\useoutertheme[hideallsubsections]{debiansidebar} \setbeamercovered{transparent} } \usepackage[no-math]{fontspec} \setmainfont[ExternalLocation, Mapping=tex-text, BoldFont=FreeSerifBold, ItalicFont=FreeSerifItalic, BoldItalicFont=FreeSerifBoldItalic]{FreeSerif} \setsansfont[ExternalLocation, Mapping=tex-text, BoldFont=FreeSerifBold, ItalicFont=FreeSerifItalic, BoldItalicFont=FreeSerifBoldItalic, Scale=MatchLowercase]{FreeSerif} \setmonofont{FreeMono} \usepackage{booktabs} \usepackage{multirow} \usepackage{setspace} \usepackage[backend=biber,natbib=true,hyperref=true,style=numeric-comp]{biblatex} \bibliography{references} % \usepackage[hyperfigures,bookmarks,colorlinks]{hyperref} \usepackage[nomargin,inline,draft]{fixme} %\usepackage[x11names,svgnames]{xcolor} \usepackage{texshade} \usepackage[absolute,overlay]{textpos} \usepackage{tikz} \usepackage{nameref} \usepackage{ulem} \usepackage{zref-xr,zref-user} \newenvironment{narrow}[2]{% \begin{list}{}{% \setlength{\topsep}{0pt}% \setlength{\leftmargin}{#1}% \setlength{\rightmargin}{#2}% \setlength{\listparindent}{\parindent}% \setlength{\itemindent}{\parindent}% \setlength{\parsep}{\parskip}}% \item[]}{\end{list}} \def\newblock{\hskip} \newenvironment{paperquote}{% \begin{quote}% \it }% {\end{quote}} \renewcommand{\textfraction}{0.15} \renewcommand{\topfraction}{0.85} \renewcommand{\bottomfraction}{0.65} \renewcommand{\floatpagefraction}{0.60} %\renewcommand{\baselinestretch}{1.8} \newenvironment{enumerate*}% {\begin{enumerate}% \setlength{\itemsep}{0pt}% \setlength{\parskip}{0pt}}% {\end{enumerate}} \newenvironment{itemize*}% {\begin{itemize}% \setlength{\itemsep}{0pt}% \setlength{\parskip}{0pt}}% {\end{itemize}} \logo{\includegraphics[width=2cm]{figures/openlogo.pdf}} \author{Don Armstrong} \title{Debbugs} \subtitle{Database Ho!} %\date{August 11th, 2008} \titlegraphic{\includegraphics[height=0.3\textheight,keepaspectratio]{figures/openlogo-crop.pdf}} \subject{BTS} % State of the BTS: new features, changes and tips % % The bug tracking system is where all bugs that affect Debian are % filed. New features, including the affects, summary, and local debbugs % mirror will be presented and demonstrated. Some of the underlying % architectural changes to the codebase in debbugs to make it more % maintainable and deployable are also outlined. Finally, a set of tips % and tricks that can be used to make the BTS easier to use are % presented. % % - New Features % - affects % - summary % - local debbugs mirror for offline work % - New SOAP % - full text search % - filtering bugs % - multi-distribution/arch status % - control changes at submit@ time % - Code Architecture Changes % - Modularization % - Control abstraction % - release to experimental <>= library(lattice) library(xtable) library(ggplot2) library(reshape2) opts_chunk$set(dev="cairo_pdf",out.width="\\textwidth",out.height="0.7\\textheight",out.extra="keepaspectratio") opts_chunk$set(cache=TRUE, autodep=TRUE) options(device = function(file, width = 8, height = 7, ...) { cairo_pdf(tempfile(), width = width, height = height, ...) }) @ \begin{document} %\setbeamercolor{frametitle}[bg=-red!90!green!10,fg=black] \frame[plain]{\titlepage Source available at \url{http://git.donarmstrong.com/debbugs-presentations.git/} } % \begin{frame}{Debbugs} % \tableofcontents[subsectionstyle=hide] % \end{frame} \section{Introduction to Debbugs} \subsection{Introduction} \begin{frame}{Goals of this talk} \begin{columns} \column{0.5\textwidth} \begin{itemize} \item Introduction to Debbugs \item New features \item Tips and Tricks \item Planned features \item Places you can help \end{itemize} \column{0.5\textwidth} \includegraphics[width=\textwidth,keepaspectratio]{images/rc_buggy} \end{columns} \end{frame} \begin{frame}{Goals of the BTS} \begin{columns} \column{0.4\textwidth} \includegraphics[width=\textwidth,keepaspectratio]{images/not_a_bug} \column{0.6\textwidth} \begin{itemize} \item Reporting of \only<1>{Features}\only<2>{Bugs} \item Evolution of \only<1>{Features}\only<2>{Bugs} \item Fixing of \only<1>{Features}\only<2>{Bugs} \item Reducing impact of \only<1>{Features}\only<2>{Bugs} \end{itemize} \end{columns} \end{frame} \section{Bugs in Debian} \subsection{Bug Reporting Rate} \begin{frame}{How many bugs do we have?} \begin{center} <>= bug.growth <- read.table("data/sorted_bug_growth_for_r_every_500.txt",stringsAsFactors=FALSE); colnames(bug.growth) <- c("time","bugs") bug.growth <- bug.growth[pmax(bug.growth$bugs) <= as.numeric(bug.growth$bugs),] bug.growth$date <- as.POSIXct(ISOdatetime(1970,1,1,0,0,0)+as.numeric(bug.growth$time)) print(ggplot(bug.growth[bug.growth$date > as.POSIXct(ISOdatetime(2005,1,1,0,0,0)),],aes(x=date,y=bugs))+ xlab("Time")+ylab("Bugs filed in Debian")+ geom_point()+stat_smooth(method="lm")+ ggtitle("Bug growth versus time")) @ \end{center} \end{frame} \begin{frame} \begin{center} <>= print(ggplot(bug.growth[bug.growth$date > as.POSIXct(ISOdatetime(2013,1,1,0,0,0)),],aes(x=date,y=bugs))+ xlab("Time")+ylab("Bugs filed in Debian")+ geom_point()+stat_smooth(method="lm")+ ggtitle("Christian Perrier's Plot")) bugs.filed.per.day <- lm(bugs~date,bug.growth)$coeff[2]*3600*24 @ \end{center} Roughly \Sexpr{format(bugs.filed.per.day,digits=1)} bugs are filed per day. \end{frame} \subsection{Bug Fixing Rate} \begin{frame} \begin{center} <>= bug.closed.series <- read.table(file="data/bug_closed_time_series.txt") colnames(bug.closed.series) <- c("archived.bugs","year.week") bug.closed.series$week <- gsub("(\\d+)-(\\d+)","\\2",bug.closed.series$year.week) bug.closed.series$year <- gsub("(\\d+)-(\\d+)","\\1",bug.closed.series$year.week) bug.closed.series$doy <- as.numeric(bug.closed.series$week)*7 bug.closed.series$year.doy <- paste(sep="-",bug.closed.series$year,bug.closed.series$doy) bug.closed.series$date <- as.POSIXct(strptime(bug.closed.series$year.doy, format="%Y-%j")) # bug.closed.ts <- ts(bug.closed.series[,1],start=1,frequency=7) print(ggplot(bug.closed.series[bug.closed.series$date > as.POSIXct(ISOdatetime(2008,1,1,0,0,0)),], aes(x=date,y=archived.bugs))+geom_line()+stat_smooth(method="lm")+ ylab("Archived bugs")+xlab("Time")) bugs.closed.per.day <- sum(bug.closed.series$archived.bugs)/ as.numeric(bug.closed.series$date[nrow(bug.closed.series)]-bug.closed.series$date[1]) @ \end{center} Roughly \Sexpr{format(bugs.closed.per.day,digits=1)} bugs are closed per day. \end{frame} \subsection{RC Bugs} \begin{frame} \begin{center} <>= rc.bugs <- read.table(file="data/rc_bugs.txt", header=TRUE,fill=TRUE) rc.bugs <- data.frame(rc.bugs) rc.bugs <- rc.bugs[,c(-5,-7)] rc.bugs$date <- as.POSIXct(strptime(rc.bugs$date, format="%Y%m%d%H%M")) rc.bugs.long <- melt(rc.bugs,id="date") print(ggplot(rc.bugs.long[rc.bugs.long$date > as.POSIXct(ISOdatetime(2013,08,1,0,0,0)),] ,aes(x=date,y=value,color=variable))+ geom_line()+ ggtitle("RC Bugs in the past year")+ theme(legend.position=c(0.9,0.5))+ guides(color=guide_legend(title="Measure"))) @ \end{center} \setbeamercolor{postit}{fg=black,bg=yellow} \begin{textblock}{4}(6,4) \begin{onlyenv}<2> \begin{beamercolorbox}[sep=1em,wd=5cm]{postit} \centering \huge Too many RC bugs! \end{beamercolorbox} \end{onlyenv} \end{textblock} \end{frame} \section{Debbugs Structure and Infrastructure} \subsection{BTS System Diagram} \begin{frame}{BTS system Diagram} \begin{center} \includegraphics[width=\textwidth,height=\textheight,keepaspectratio]{figures/bug_global_layout} \end{center} \end{frame} \subsection{Debbugs Box Diagram} \begin{frame}{Debbugs Box Diagram} \begin{center} \input{debbugs_layout} \end{center} \end{frame} \section{Database Ho!} \subsection{Overall Database Design} \begin{frame}{Overall Database Design} \begin{center} \input{debbugs_layout} \end{center} \end{frame} \subsection{Perl Database Infrastucture} \begin{frame}{Perl Database Infrastructure} \begin{itemize} \item DBIx::Class \item DBIx::Class::Schema::Loader -- SQL schema to DBIx::Class \item DBIx::Class::DeploymentHandler -- Upgrades of Schema \item Debbugs::DB -- Debbugs SQL Module \end{itemize} \end{frame} \subsection{SQL Design} \begin{frame}{SQL Schema} \includegraphics[width=\textwidth,keepaspectratio]{figures/schema.png} \end{frame} \subsubsection{DAK SQL Design} \begin{frame}{DAK SQL Schema} \includegraphics[width=\textwidth,keepaspectratio]{figures/dak_schema.png} \end{frame} \section{New(ish) Features} \subsection{New Changes} \begin{frame}{New Changes} \begin{itemize} \item CVE linkified (eg, \href{http://bugs.debian.org/570743}{\#570743}) \item status caching \item reverse status lookup \item forcemerge does the right thing \end{itemize} \end{frame} \subsection{Affects} \begin{frame}{Affects} \begin{itemize} \item Bugs in Package A causes problem in package B \item Reduce duplicates filed in package B by showing bug in package B's list \item \url{http://www.debian.org/Bugs/server-control\#affects} \item \texttt{bts affects 12345 foopkg;} \end{itemize} \end{frame} \begin{frame}{Affects: Implemented} % \includegraphics[width=\textwidth,height=\textheight,keepaspectratio]{figures/affects} \end{frame} \subsection{Summary} \begin{frame}{Summary} \begin{itemize} \item Short indication of what the bug was about \item Long discussion, summarized into a single paragraph \item Extract the first non-control, non-psuedoheader, non-quoted paragraph from a nominated message \item \url{http://www.debian.org/Bugs/server-control\#summary} \item \texttt{bts summary 12345 30} \end{itemize} \end{frame} \begin{frame}{Summary: Implemented} % \includegraphics[width=\textwidth,height=\textheight,keepaspectratio]{figures/summary} \end{frame} \subsection{Local Mirror} \begin{frame}{local-debbugs} \begin{itemize} \item Ability to run a full local copy of debbugs \item Select bugs interested in based on a configurable query \item By default selects unarchived bugs \begin{itemize} \item In maintained packages \item Correspondend with \item Submitted \item RC \end{itemize} \item Currently takes about 2G for me (most of which is indixes) \item Part of experimental debbugs package (in the archive) \end{itemize} \end{frame} \begin{frame}{local-debbugs demo} \begin{itemize} \item Update mirror: \texttt{local-debbugs \--\--mirror} \item Start daemon: \texttt{local-debbugs \--\--daemon} \item See bug \#441151: \texttt{local-debbugs \--\--show 441151} \item See bugs in lilypond: \texttt{local-debbugs \--\--search lilypond} \item See critical bugs: \texttt{local-debbugs \--\--search severity:critical} \end{itemize} \end{frame} \begin{frame}{local-debbugs issues} \begin{columns} \column{0.5\textwidth} \begin{itemize} \item Suboptimal missing bug handling \item Size of mirror large \item Could be faster to sync \item Needs better documentation \end{itemize} \column{0.5\textwidth} % \includegraphics[width=\textwidth,keepaspectratio]{figures/happy_bug} \end{columns} \end{frame} %\subsection{More SOAP} % \section{Tips and Tricks} % % - Tips and Tricks % % % - Full text search % \begin{frame}{Full Text Search} % \begin{itemize} % \item Using HyperEstraier % \item \url{http://bugs.debian.org/cgi-bin/search.cgi} % \item Searching for ``lp0 on fire'' % \item Searching for memory leaks in iceweasel % \end{itemize} % \end{frame} % % % - Using the new options limits % \begin{frame}{CGI Options} % \begin{itemize} % \item Bugs in debbugs which aren't tagged pending or wontfix owned by me % \item Bugs in debbugs tagged help % \item Bugs in debbugs whose title mentions cgi % \end{itemize} % \end{frame} \begin{frame}{Silly Symbols} \begin{center} \begin{tabular}{c c c c} Merged & = & Fixed Versions & ☺ \\ Blocked & ♙ & Blocks & ♔ \\ Forwarded & ↝ & Archived & ♲ \\ Affects & ☣ & wontfix & ☹ \\ patch & + & moreinfo & M \\ unreproducible & R & security & S \\ pending & P & fixed & F \\ help & H & fixed-upstream & U \\ upstream & u & & \\ \end{tabular} \end{center} \end{frame} % \begin{frame}{Silly Symbols} % \begin{center} % \begin{tabular}{c c c c} % Merged & = & Fixed Versions & \smiley \\ % Blocked & \includegraphics[height=0.5cm,keepaspectratio]{figures/blocked} & Blocks & \includegraphics[height=0.5cm,keepaspectratio]{figures/blocks} \\ % Forwarded & ↝ & Archived & \includegraphics[height=0.5cm,keepaspectratio]{figures/archived} \\ % Affects & \includegraphics[height=0.5cm,keepaspectratio]{figures/affects_icon} & wontfix & \frownie \\ % patch & + & moreinfo & M \\ % unreproducible & R & security & S \\ % pending & P & fixed & F \\ % help & H & fixed-upstream & U \\ % upstream & u & & \\ % \end{tabular} % \end{center} % \end{frame} \section{Planned features} \subsection{Status Caching} \begin{frame}{Status Caching} \begin{itemize} \item Cache status of bugs (presence/absence at all distributions) \item Speed up display of \url{http://bugs.debian.org/src:linux-2.6} \item Allows for reverse status lookup \begin{itemize} \item Enables finding bugs which are present in testing or unstable \item Eventually replace functionality of \url{bts.turmzimmer.net} \end{itemize} \end{itemize} \end{frame} \begin{frame}{Planned features} \begin{itemize} \item Control Commands in Submit \item<3-> Mail to nnn@bugs.debian.org goes to submitter \item<4-> Uservalues \item<6-> b.d.o/libc6 $\rightarrow$ b.d.o/src:eglibc \begin{itemize} \item<6-> in future, make binary-only views more difficult to get (?) \end{itemize} \end{itemize} \definecolor{peach}{rgb}{1,0.9,0.7} \setbeamercolor{controlcmd}{fg=black,bg=peach} \begin{textblock}{6}(7,7) \begin{onlyenv}<2> \begin{beamercolorbox}[sep=1em,wd=5.5cm]{controlcmd} \begin{tabular}{l} To: submit@bugs.debian.org\\ From: foo@example.com\\ \\ Package: blah\\ Control: user foo@debian.org\\ Control: usertag 0 bleargh\\ Control: clone 0 -1\\ Control: reassign -1 blah\\ \end{tabular} \end{beamercolorbox} \end{onlyenv} \begin{onlyenv}<5> \begin{beamercolorbox}[sep=1em,wd=5.5cm]{controlcmd} \begin{tabular}{l} To: control@bugs.debian.org\\ From: foo@example.com\\ \\ user foo@debian.org\\ uservalue 12345 priority 30\\ uservalue 12345 vcs-commit \textbackslash \\ \ http://q.fr/b/r1432 \\ uservalue 12345 difficulty 10\\ \end{tabular} \end{beamercolorbox} \end{onlyenv} \end{textblock} \end{frame} \begin{frame}{More planned features} \begin{columns} \column{0.5\textwidth} \begin{itemize} \item Merging of merged bug reports \item Threading in report \item Usercategory duplication and replay \item Remote Attachments \item New Spool Storage Format \item Backend index using postgresql \item Usertags visibility \item Smarter CGI options \end{itemize} \column{0.5\textwidth} % \includegraphics[width=\textwidth,keepaspectratio]{figures/oh_hai_bug} \end{columns} \end{frame} \begin{frame}{Statistics} % \begin{columns} % \column{0.5\textwidth} % \column{0.5\textwidth} \begin{itemize} \item Track status changes over time \item Bugs found/fixed/absent in stable/testing/unstable over time \item Bug submission times \item Bug closure times \item Bug mail rates \item Per package, severity, maintainer tracking of the above \end{itemize} \begin{center} % \includegraphics[width=\textwidth,height=0.35\textheight,keepaspectratio]{figures/surprised_bugs_life} \end{center} % \end{columns} \end{frame} \begin{frame}{Action Required Sorting} \begin{center} \begin{tabular}{c | c | c} & Care about & Don't care \\ \hline \multirow{3}{*}{\begin{sideways}\parbox{12mm}{Triage}\end{sideways}} & No response & Active responses \\ & Ancient found versions & found in multiple versions \\ & Incomplete report & Complete report \\ \hline \multirow{3}{*}{\begin{sideways}\parbox{12mm}{Maint.}\end{sideways}} & has a patch & pending/fixed \\ & moreinfo+response & just moreinfo \\ & high priority & wontfix \\ \hline \multirow{3}{*}{\begin{sideways}\parbox{12mm}{Submit.}\end{sideways}} & moreinfo & moreinfo+response \\ & mail from maintainer & no response \\ & bug is fixed & bug isn't fixed \\ \end{tabular} \end{center} \begin{itemize} \item Sorting by a combination of priority and action required \item Minimize time wasted finding a bug \item Maximize productivity \end{itemize} \end{frame} \subsection{Upstream and Downstream Integration} \begin{frame} \begin{itemize} \item Sharing state between upstream, downstream and sidestream \item Sharing comments between the same \item Distributed bug tracking \item Distributed version/commit tracking \end{itemize} \end{frame} \section{Places you can help} \begin{frame}{\sout{5} \sout{6} 7 tasks seeking contributors} \begin{itemize} \item Documentation of Usercategories \item Implementation of rss feeds for packages and bugs \item CGI options on pkgreport.cgi for usertags et al. \#536378) \item Documentation of multipe-package reassign and when it or affects should be used \item Mailto link with subject, references, etc. all filled in \item Documentation for local-debbugs configuration file (behaves like bts select) \end{itemize} \end{frame} \begin{frame}{Getting started on these tasks} \begin{itemize} \item Upstream branches: \url{http://bugs.debian.org/debbugs-source/} \begin{itemize} \item mainline (upstream debbugs): bzr clone \url{http://bugs.debian.org/debbugs-source/mainline} \item debian (debian branch): bzr clone \url{http://bugs.debian.org/debbugs-source/debian} \end{itemize} \item My branches: \url{http://bzr.donarmstrong.com/debbugs/branches} \item Mailing list debian-debbugs@lists.debian.org \item IRC \#debbugs or \#debian-bugs on irc.debian.org; I'm dondelelcaro \end{itemize} \end{frame} \begin{frame}{Setting up a BTS mirror for development} \begin{itemize} \item \url{http://svn.donarmstrong.com/don/trunk/projects/debbugs/local_mirror.sh} \begin{itemize} \item Set SPOOL and VERSIONS appropriately \item mkdir -p /path/to/spool/{db-h,archive} /path/to/versions; \item touch /path/to/spool/by-reverse.idx \end{itemize} \item \url{http://svn.donarmstrong.com/don/trunk/projects/debbugs/setup_config.sh} \begin{itemize} \item<2> Will exist suddenly (not there now) \end{itemize} \end{itemize} \end{frame} % - Tips and Tricks % - Full text search % - Using the new options limits % - Usertags % - Getting users to triage their bugs % * Overview of how the BTS works % o process, service % o web frontends % o soap, etc. % * New features % o user values % o action-required sorting (maintainer and submitter views) % + priority sorting (uservalue) % o merging of merged bug reports and threaded view % o control in messages to submit@ and nnn@ % o usertag/uservalue/usercategory duplication/replay % o rss feeds for packages and bugs % o remote attachments % o new spool storage format % o better statistics % * Old features that may be unfamiliar % o fun with bts select % o full text searching with HE % o local-debbugs % * A bald plea for more assistance % Bugs the BoF % % Upcoming improvements: % % * control@ commands in submit % * status caching % * reverse status lookup % * forcemerge % * b.d.o/libc6 -> b.d.o/src:eglibc % % % Proposed To-do: % % * web-based mail submission % ** needed due to blocking of port 25 % % * queue runs % ** cannot be synchronous % ** but may run queue on demand % % * per-arch buildd tags (or pseudopackages) % % * search by uploader % % % Other technical suggestions: % % * notification to blockees when blocking bugs are killed off % * documentation consistency? (e.g. reassign; could use clarification) % ** response: needs love; currently in WML % % Social Improvements: % % * Better statistics % ** Tells us: % *** What packagers need help % *** Which bug triagers should be recognized. Currently these people % receive no recognition; would like to encourage them to continue % this work. % % Discussion - whether mass-filed bugs should be included? % % * Common triager mistake: emailing the bug number instead of the submitter. % Don: Plan to address this via the reply list. % % Please configure mailing list subscriptions not to bounce/spam the owner. % % Q: Limit on size of attachments to BTS? ~200MB % % Idea: RFC exists that supports remote/offline attachments. % % How to perform debbugs development? % * see debbugs-local package % * can rsync the entire bugs database to local box % * apache configuration simple, mail setup somewhat more involved % * help: debbugs@l.d.o, #debbugs % * code is in bzr % ** see armstrong branches for work in progress % % How do you use the BTS? % % Problems: % * 200-300 open bugs: difficult to keep track of bugs already seen % * idea: improve documentation regarding user-tags/categorization features % * idea: display of possible set of user-tags via JavaScript - "make user-tags pretty" % * idea: user authention to allow management of tags % % * switch index from flat-files to Postgres; borrow from UDD % % Q: Complaints regarding email addresses on webpage. Should we obfuscate these? % ** Is there any hard data on which types of obfuscation work? % ** Can we blackhole robots? Use a honeypot? Issue is that bugs are % archives to the ML. Discussion around configuration on Ian Jackson's % site. % % Q: List of needs for new debbugs contributors? % * See how to help on the wiki. % % Q: Can you get a list of user-tags for an email address w/out the bug reports? % * Should be pretty easy. May be a patch for comand-line 'bts' tool in devscripts. % % What's next? % % Slides: http://svn/donarmstrong.com/don/trunk/projects/debian/debconf10/bugs_the_bof/ % Contact Don with questions. % % % For spam questions, reach out to Blars Blarson \begin{frame}{The Debbugs Team} \begin{itemize} \item Current team \begin{itemize} \item Don Armstrong \item Blars Blarson \item Colin Watson \end{itemize} \item Emeritus Developers \begin{itemize} \item Steiner Gunderson \item Adam Heath \item Josip Rodin \item Anthony Towns \end{itemize} \item You! \end{itemize} \end{frame} \begin{frame} \includegraphics[width=0.8\textwidth,height=0.8\textheight,keepaspectratio]{images/don_at_burning_man.jpg} \end{frame} \end{document}