#!/usr/bin/python from glob import glob try: from json import load as jload def jsonload(f): return jload(f) except ImportError: from json import read as jread def jsonload(f): return jread(f.read()) import sys, os import pylab as pl import numpy as np import time # uniform colors for OS results os_colors = ['#AA2029', '#D1942B', '#7FB142', '#69A7CE'] os_order = ['linux', 'mac', 'win', 'otheros'] time_order = ['notime', 'little', 'most', 'always'] time_colors = ['#FF0000', '#FF5500', '#FFAC00', '#FFFD08'] time_categories = { 'notime': "don't use it", 'little': "less than 50%", 'most': "more than 50%", 'always': "always" } # resources resource_categories = { 'vendor': 'Vendor/Project website', 'retailer': 'Retailer', 'os': 'Operating system', 'cpan': 'CPAN', 'cran': 'CRAN', 'epel': 'EPEL', 'fink': 'Fink', 'freebsdports': 'FreeBSD ports', 'incf': 'INCF', 'macports': 'Macports', 'matlabcentral': 'Matlab Central', 'neurodebian': 'NeuroDebian', 'nitrc': 'NITRC', 'pypi': 'PyPi', 'pythonbundles': 'Python bundles', 'sourceforge': 'Sourceforge', 'otherres': 'Other resource' } # software categories sw_categories = { 'general': 'General computing', 'dc': 'Distributed computing', 'img': 'Brain imaging', 'datamanage': 'Data management', 'neusys': 'Neural systems modeling', 'electro': 'Electrophysiology, MEG/EEG', 'bci': 'Brain-computer interface', 'acq': 'Hardware interface/Data acquisition', 'rt': 'Real-time solutions', 'psychphys': 'Psychophysics/Experiment control' } # some meaningful groups of OSes redhat_family = ["rhel", "centos", "fedora", "scilinux"] debian_family = ["debian", "ubuntu", "biolinux"] suse_family = ["suse", "slel"] other_linux_family = ["gentoo", "mandriva", "arch", "slackware", "otherlinux"] other_family = ["starbsd", "unix", "qnx", "beos", "solaris", "other", "dontknow"] os_cat_names = { 'win': 'Windows', 'mac': 'Mac OS', 'linux': 'GNU/Linux', 'otheros': 'Other OS' } os_family = { 'win': ["windows"], 'mac': ["macosx"], 'linux': redhat_family + debian_family + suse_family + other_linux_family, 'otheros': other_family } # end the reverse mapping os_family_rev = {} for ost in os_family: for os_ in os_family[ost]: os_family_rev[os_] = ost def load_list2dict(name): d = {} lfile = open(name) for line in lfile: if line.strip() == "": continue kv = line.split(':') if kv[0] in d: raise RuntimeError( "Got a line %s with a duplicate key %s whenever value for it " "is known already to be %r" % (line, kv[0], d[kv[0]])) d[kv[0]] = kv[1].strip().strip('"') return d class DB(dict): os_dict = load_list2dict('oslist.txt') datamod_dict = load_list2dict('datamodlist.txt') sw_dict = load_list2dict('swlist.txt') position_dict = load_list2dict('position-dd-list.txt') employer_dict = load_list2dict('employer-dd-list.txt') ratings_dict = load_list2dict('ratingslist.txt') vm_dict = load_list2dict('vmlist.txt') def __init__(self, srcdir): # eats the whole directory if srcdir is None: return datafilenames = glob('%s/*.json' % srcdir) for dfn in datafilenames: rawdata = jsonload(open(dfn)) self[rawdata['timestamp']] = rawdata def get_unique(self, key): # return a set of all (unique) values for a field id uniq = set() for d in self.values(): if key in d: el = d[key] if isinstance(el, list): uniq = uniq.union(el) else: uniq = uniq.union((el,)) return uniq def get_not_none(self, key): # return a list of all values of a specific field id # the second return value is count of submission that did not have data # for this field id val = [] missing = 0 for d in self.values(): if key in d: el = d[key] if isinstance(el, list): val.extend(el) else: if el == 'none': missing += 1 else: val.append(el) else: missing += 1 return val, missing def get_counts(self, key): # return a dict with field values as keys and respective submission # count as value vals = self.get_not_none(key)[0] uniq = np.unique(vals) counts = dict(zip(uniq, [vals.count(u) for u in uniq])) return counts def select_match(self, key, values): # return a db with all submissions were a field id has one of the # supplied values match = DB(None) for k, v in self.items(): if not key in v: continue el = v[key] if isinstance(el, list): if len(set(values).intersection(el)): match[k] = v elif el in values: match[k] = v return match def select_match_exactly(self, key, values): # return a db with all submissions were a field id has value # equal to the supplied match = DB(None) set_values = set(values) for k, v in self.items(): if not key in v: continue el = v[key] if isinstance(el, list): if set(el) == set_values: match[k] = v elif set([el]) == set_values: match[k] = v return match def get_nice_name(self, id): srcs = [DB.os_dict, os_cat_names, DB.sw_dict, sw_categories, resource_categories, time_categories, DB.datamod_dict, DB.position_dict, DB.employer_dict, DB.vm_dict, DB.ratings_dict] for src in srcs: if id in src: return src[id] # not found, nothing nicer return id def mkpic_os_per_env(db, destdir): envs = ['pers_os', 'man_os', 'virt_host_os', 'virt_guest_os'] env_names = ['Personal', 'Managed', 'Virt. Host', 'Virt. Guest'] env_stats = {} for env in envs: counts = db.get_counts(env) stats = dict(zip(os_family.keys(), [0] * len(os_family))) for os in counts: stats[os_family_rev[os]] += counts[os] total_count = np.sum(stats.values()) for osf in stats: if not total_count: stats[osf] = 0 else: stats[osf] = float(stats[osf]) / total_count env_stats[env] = stats # make stacked barplot pl.figure(figsize=(7.5, 4)) x = np.arange(len(envs)) bottoms = np.zeros(len(envs)) for i, os in enumerate(os_order): stat = [env_stats[e][os] for e in envs[::-1]] pl.barh(x, stat, left=bottoms, color=os_colors[i], label=db.get_nice_name(os), height=0.8) bottoms += stat pl.legend(loc='center left') pl.yticks(x + 0.4, env_names[::-1]) pl.ylim(-0.25, len(envs)) pl.xlim(0,1) pl.title("Operating system preference by environment") pl.xlabel("Fraction of submissions") pl.subplots_adjust(left=0.15, right=0.97) pl.savefig('%s/ospref_by_env.png' % destdir, format='png', dpi=80) def mkpic_time_per_env(db, destdir): envs = ['pers_time', 'man_time', 'virt_time'] env_names = ['Personal', 'Managed', 'Virtual'] env_stats = {} for env in envs: counts = dict(zip(time_order, [0] * len(time_order))) counts.update(db.get_counts(env)) total_count = np.sum(counts.values()) for c in counts: counts[c] = float(counts[c]) / total_count env_stats[env] = counts # make stacked barplot pl.figure(figsize=(7.5, 4)) x = np.arange(len(envs)) bottoms = np.zeros(len(envs)) for i, t in enumerate(time_order): stat = [env_stats[e][t] for e in envs[::-1]] pl.barh(x, stat, left=bottoms, color=time_colors[i], label=db.get_nice_name(t), height=.6) bottoms += stat pl.legend(loc='lower left') pl.yticks(x + 0.2, env_names[::-1]) pl.ylim(-0.4, len(envs)) pl.title("Research activity time by environment") pl.xlabel("Fraction of submissions") pl.subplots_adjust(right=0.97) pl.savefig('%s/time_by_env.png' % destdir, format='png', dpi=80) def mkpic_submissions_per_key(db, destdir, key, title, sortby='name', multiple=False): counts = db.get_counts(key) pl.figure(figsize=(6.4, (len(counts)-2) * 0.4 + 2)) if not len(counts): tmargin = 0.4 else: tmargin = .8/len(counts) if tmargin > 0.3: tmargin = 0.3 pl.subplots_adjust(left=0.03, right=0.97, top=1-tmargin, bottom=tmargin) pl.title(title) if not len(counts): pl.text(.5, .5, "[Insufficient data for this figure]", horizontalalignment='center') pl.axis('off') else: # sort by name if sortby == 'name': stats = sorted(counts.items(), cmp=lambda x, y: cmp(x[0], y[0])) elif sortby == 'count': stats = sorted(counts.items(), cmp=lambda x, y: cmp(x[1], y[1]))[::-1] else: raise ValueError("Specify either name or count for sortby") x = np.arange(len(stats)) pl.barh(x + (1./8), [s[1] for s in stats[::-1]], height=0.75, color = '#008200') pl.yticks(x + 0.5, ['' for s in stats]) text_offset = pl.gca().get_xlim()[1] / 30. for i, s in enumerate(stats[::-1]): pl.text(text_offset, i+.5, db.get_nice_name(s[0]) + " [%d]" % (s[1],), horizontalalignment='left', verticalalignment='center', bbox=dict(facecolor='white', alpha=0.8, edgecolor='white')) pl.ylim(0, len(stats)) yl = "Number of submissions" if multiple: yl += "\n(multiple choices per submission possible)" pl.xlabel(yl) pl.savefig('%s/submissions_per_%s.png' % (destdir, key), format='png', dpi=80) def mkpic_software(db, destdir): for typ in sw_categories.keys(): mkpic_submissions_per_key( db, destdir, 'sw_%s' % typ, title="Software popularity: %s" % db.get_nice_name(typ), sortby='count') def mkpic_rating_by_os(db, env, items, destdir, title): from mvpa.misc.plot.base import plot_bars pl.figure(figsize=(6.4, 4.8)) for i, os in enumerate(os_order): ratings = [db.select_match(env, os_family[os]).get_not_none('%s' % (it,))[0] for it in items] plot_bars(ratings, offset=((i+1)*0.2)-0.1, color=os_colors[i], title=title, ylabel="Mean rating", label=db.get_nice_name(os)) pl.ylim((0,3)) pl.xlim((0,len(items))) pl.yticks((0, 3), ['Disagree', 'Agree'], rotation=90) pl.xticks(np.arange(len(items))+0.5, [i[-2:] for i in items], horizontalalignment='center') pl.legend(loc='lower right') pl.savefig('%s/ratings_%s.png' % (destdir, env), format='png', dpi=80) def mkpic_rating_by_os_hor_joined(db, env, items, destdir='.', title=None, intro_sentence="I agree with the statements", suffix='', max_rating=4): per_os_width = 1 #pl.figure(figsize=(6.4, 4.8)) if max_rating is None: assert(len(items) == 1) # We need to query for it field = items[0] max_rating = np.max([db[k][field] for k in db if field in db[k]]) + 1 print "max ", max_rating nos = len(os_order) rst = open('figures/ratings_%s%s.rst' % (env, suffix), 'w') rst.write(""" %s %s %s .. raw:: html
%(it_nice)s |