Adding popcon plot to 'popularity' page

diff --git a/sphinx/_static/popconchart.js b/sphinx/_static/popconchart.js
new file mode 100644 (file)
index 0000000..4fa7d83
--- /dev/null
+++ b/sphinx/_static/popconchart.js
@@ -0,0 +1,25 @@
+d3.json('/_files/nd_popconstats.json', function(data) {
+  nv.addGraph(function() {
+    chart = nv.models.stackedAreaChart()
+                  .x(function(d) { return d[0] })
+                  .y(function(d) { return d[1] })
+                  .clipEdge(true);
+
+    chart.stacked.style('stacked');
+
+    chart.xAxis
+        .tickFormat(function(d) {
+            return d3.time.format('%d %b %Y')(new Date(d)) });
+
+    chart.yAxis
+        .tickFormat(d3.format(',.2f'));
+
+    d3.select('#popconchart')
+      .datum(data)
+        .transition().duration(500).call(chart);
+
+    nv.utils.windowResize(chart.update);
+
+    return chart;
+  });
+})

diff --git a/sphinx/popularity.rst b/sphinx/popularity.rst
index cefe24dbfd5a27931ad5b1cc80bcd553fac2a650..fb272772cd1baabb2e87e395821ca47d1ca6ccd7 100644 (file)
--- a/sphinx/popularity.rst
+++ b/sphinx/popularity.rst
@@ -36,8 +36,23 @@ We encourage you to participate in the `popularity
 contest <http://popcon.debian.org>`_ (popcon), which anonymously
 collects the list of packages you installed/use on your system.
 Collecting such statistics is of particular importance for research
-software projects as a prove of an existing user-base.  If upon
-installation of the system you rejected the invitation to participate
+software projects as a proof of existing user-base.
+
+In addition to popcon stats for your "core" distribution (e.g. `Debian
+<http://popcon.debian.org/>`__ or `Ubuntu
+<http://popcon.ubuntu.com/>`__), an interactive plot below summarizes number
+of submissions to NeuroDebian's popcon server.
+
+.. raw:: html
+
+  <div><svg style="height:500px" id="popconchart"></svg></div>
+
+  <script src="/_static/popconchart.js"></script>
+
+You can get more information about submissions from `NeuroDebian Popularity
+Contest <http://neuro.debian.net/popcon/>`__ page.
+
+If upon installation of the system you rejected the invitation to participate
 you can always change your decision by running::
 
  sudo dpkg-reconfigure popularity-contest
@@ -52,10 +67,5 @@ you can always change your decision by running::
     sed -i -e 's,PARTICIPATE *= *.no.,PARTICIPATE="yes",g' -e '/^ *MY_HOSTID/d' /etc/popularity-contest.conf
     DEBIAN_FRONTEND=noninteractive dpkg-reconfigure popularity-contest
 
-In addition to popcon pages for your "core" distribution (e.g. `Debian
-<http://popcon.debian.org/>`__ or `Ubuntu
-<http://popcon.ubuntu.com/>`__) you can see/get statistics for
-submissions to `NeuroDebian <http://neuro.debian.net/popcon/>`__ and
-know that you are already contributing back to the community.
 
 .. include:: link_names.txt

diff --git a/tools/nd_popcon2stats b/tools/nd_popcon2stats
new file mode 100755 (executable)
index 0000000..0fd57d1
--- /dev/null
+++ b/tools/nd_popcon2stats
@@ -0,0 +1,107 @@
+#!/usr/bin/python
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
+# vi: set ft=python sts=4 ts=4 sw=4 et:
+#
+import fileinput
+import sys
+import time
+from datetime import datetime
+import re
+import sets
+import json
+import operator
+
+
+releases = {
+    'etch': 'Debian GNU/Linux 4.0 (etch)',
+    'lenny': 'Debian GNU/Linux 5.0 (lenny)',
+    'squeeze': 'Debian GNU/Linux 6.0 (squeeze)',
+    'wheezy': 'Debian testing (wheezy)',
+    'sid': 'Debian unstable (sid)',
+    'hardy': 'Ubuntu 08.04 LTS "Hardy Heron" (hardy)',
+    'jaunty': 'Ubuntu 09.04 "Jaunty Jackalope" (jaunty)',
+    'karmic': 'Ubuntu 09.10 "Karmic Koala" (karmic)',
+    'lucid': 'Ubuntu 10.04 LTS "Lucid Lynx" (lucid)',
+    'maverick': 'Ubuntu 10.10 "Maverick Meerkat" (maverick)',
+    'natty': 'Ubuntu 11.04 "Natty Narwhal" (natty)',
+    'oneiric': 'Ubuntu 11.10 "Oneiric Ocelot" (oneiric)',
+    'precise': 'Ubuntu 12.04 LTS "Precise Pangolin" (precise)',
+    'quantal': 'Ubuntu 12.10 "Quantal Quetzal" (quantal)',
+    'raring': 'Ubuntu 13.04 "Raring Ringtail" (raring)',
+    'saucy': 'Ubuntu 13.10 "Saucy Salamander" (saucy)',
+}
+
+def error(msg):
+    sys.stderr.write('E: %s\n' % msg)
+
+def info(msg):
+    sys.stderr.write("I: %s\n" % msg)
+
+file_regex = re.compile('.*popcon-(\d{4}-\d{1,2}-\d{1,2})(|.gz)')
+
+def read_popcon_stats(filename, read_packages=True):
+    info("Reading %s" % filename)
+    entry = dict(submissions = None,
+                 package = {},
+                 release = {},
+                 architecture = {})
+
+    for line in fileinput.FileInput(filename, openhook=fileinput.hook_compressed):
+        key, values = [x.strip().lower() for x in line.split(':', 1)]
+        if key == 'package':          # most probable
+            if not read_packages:
+                break
+            try:
+                pkg, vote, old, recent, nofiles = values.split()
+            except ValueError:
+                raise ValueError("Failed to split %s" % values)
+            entry[key][pkg] = tuple(int(x) for x in (vote, old, recent, nofiles))
+        elif key in ('release', 'architecture'):
+            kvalue, value = values.split()
+            entry[key][kvalue] = int(value)
+        elif key == 'submissions':
+            entry[key] = int(values)
+        else:
+            raise ValueError("Do not know how to handle line" % line)
+    return entry
+
+if __name__ == '__main__':
+    data = {}
+
+    popcon_versions = {}
+    timestamps = sets.Set()
+
+    for f in sys.argv[1:]:
+        file_reg = file_regex.match(f)
+        if not file_reg:
+            error("Failed to recognize filename %s" % f)
+            continue
+
+        date = time.strptime(file_reg.groups()[0], '%Y-%m-%d')
+        entry = read_popcon_stats(f, read_packages=False)
+
+        date_int = int(time.mktime(date)*1000)
+        # Let's coarsen a bit -- to a week which makes sense anyways
+        # since popcon submissions are spread over a week for balanced
+        # load
+        coarsen_days = 7
+        coarsen = coarsen_days*24*3600*1000
+        # coarsen and place marker at the end of the duration
+        # but not later than today
+        date_int = min((date_int//coarsen + 1)*coarsen,
+                       time.time()*1000)
+        for version, count in entry['release'].iteritems():
+            if not version in popcon_versions:
+                popcon_versions[version] = {}
+            popcon_ = popcon_versions[version]
+            popcon_[date_int] = count + popcon_.get(date_int, 0)
+            timestamps.add(date_int)
+
+    # we need to make sure that for every date we have an entry for
+    # every version, otherwise d3 pukes because of ... d3.v2.js:expand
+    export = [{'key': k,
+               'values': [[date, popcon_versions[k].get(date, 0)/coarsen_days]
+                          for date in sorted(list(timestamps))]}
+              for k in sorted(popcon_versions.keys())]
+    print json.dumps(export)
+