]> git.donarmstrong.com Git - neurodebian.git/commitdiff
Merge branch 'enh/neurodebian-installer' -- aiming for the release
authorYaroslav Halchenko <debian@onerussian.com>
Fri, 8 Aug 2014 18:21:50 +0000 (14:21 -0400)
committerYaroslav Halchenko <debian@onerussian.com>
Fri, 8 Aug 2014 18:21:50 +0000 (14:21 -0400)
* enh/neurodebian-installer: (25 commits)
  ENH: report to debconf interface what would be the release/flavor to be used if 'auto' gets selected
  ENH: slight tune up of the test
  RF: absorb neurodebian-repository into neurodebian package (and rename debconf variables as well)
  RF: nd-aptenable -> nd-configurerepo  per our discussion
  BF: actually we need to aim for multiverse (not restricted) on ubuntus to decide for "full" flavor
  ENH: sketch for a test to check installation of the package
  BF: Disable logic for demanding netselect (absent on ubuntus)
  some notes + adding netselect into pre-depends of the neurodebian-repository
  Move keyring into Depends for repository since we do not apt-get update anyways
  BF: right -- we can't apt-get update while installing the package (via apt at least, assuming it to be default way)
  initial postinst and postrm scripts for neurodebian-repository pkg
  ENH: initial debconf .config and few tune ups for that in the nd-aptenable
  Adding neurodebian and neurodebian-repository packages
  ENH: use -I for netselect where available (ICMP is more likely to be allowed to reach than UDP)
  ENH: minor -- rename apt file into -failed.disabled (not .failed-disabled)
  BF/ENH: if apt-get update fails because of us -- move our generated file aside
  ENH: do not provide -D to netselect on older (squeeze) deployments
  ENH: Allow to pass config via environment, install needed packages (if --install), env controlled mode to just do defuns
  BF: fetching/reporting of the config file
  ENH: removed all obsolete TODOs, fix execution around netselect, added doc on depends
  ...

Conflicts:
debian/changelog

12 files changed:
artwork/ndhackergotchi.png [deleted file]
debian/TODO
debian/changelog
debian/control
debian/neurodebian.config [new file with mode: 0755]
debian/neurodebian.install [new file with mode: 0644]
debian/neurodebian.postinst [new file with mode: 0755]
debian/neurodebian.postrm [new file with mode: 0755]
debian/neurodebian.templates [new file with mode: 0644]
tools/nd-configurerepo [new file with mode: 0755]
tools/tests/run_in_debian_dockers [new file with mode: 0755]
tools/tests/test_neurodebian-repository-deployment [new file with mode: 0755]

diff --git a/artwork/ndhackergotchi.png b/artwork/ndhackergotchi.png
deleted file mode 100644 (file)
index 9f74ec5..0000000
Binary files a/artwork/ndhackergotchi.png and /dev/null differ
index 66496b0846bfd3b893b59f5f9e10d198781e3868..0e69205d89c095be4ce0f401403317fdf6adb469 100644 (file)
@@ -4,3 +4,50 @@
   would need to assure that mvpa2-tutorial is aware of ipython1x first
   then... not now
 * make all nd_* scripts usable out of the box on non-x86 boxes
+* TODO testing of neurodebian "installer" packages
+** notes from manual attempts in a docker:
+
+*** just dpkg -i way
+rm neurodebian-*deb; apt-get update -y; apt-get install -y netselect python-minimal wget; wget http://www.onerussian.com/tmp/neurodebian-{repository,keyring}_0.33~a1_all.deb; DEBIAN_FRONTEND=noninteractive dpkg -i neurodebian-{keyring,repository}_0.33~a1_all.deb
+
+*** via apt
+
+#/bin/bash
+set -ex
+
+# minimal installation with few of those packages
+apt_file=/etc/apt/sources.list.d/local.list
+
+# clean up from a previous test
+dpkg --purge neurodebian neurodebian-keyring || :
+echo PURGE | debconf-communicate neurodebian || :
+rm -rf /tmp/apt $apt_file;
+
+mkdir -p /tmp/apt; cd /tmp/apt;
+rm -f neurodebian-*deb; apt-get update -y; apt-get install -y wget;
+wget http://www.onerussian.com/tmp/neurodebian_0.33~a1_all.deb;
+wget http://www.onerussian.com/tmp/neurodebian-keyring_0.33~a1_all.deb;
+
+# generate APT repo
+apt-get install -y dpkg-dev ;  dpkg-scanpackages  . /dev/null > Packages
+# Add it to the list of available
+echo "deb file:///tmp/apt ./" >| $apt_file; apt-get update
+# Install the beast
+apt-get install -y --allow-unauthenticated neurodebian
+
+# Verify that it has correct entries
+release=$(grep -v '^[\s#]*$' /etc/apt/sources.list | head -1 | awk '{print $3;}')
+grep -E "^deb\s+http.* $release main\s*\$" /etc/apt/sources.list.d/neurodebian.sources.list
+grep -E '^deb\s+http.* data main\s*$' /etc/apt/sources.list.d/neurodebian.sources.list
+# May be even retrospect stored debconf configuration
+apt-get install debconf-utils
+debconf-get-selections | grep neurodebian/release.*select.*auto
+# And apt-get must be working
+apt-get update
+# Make sure it is active
+apt-cache policy | grep o=NeuroDebian,.*,n=data
+apt-cache policy | grep o=NeuroDebian,.*,n=$release
+
+echo "I: success"
+
+*** ~/bin/docker-debians-run /bin/bash -c "apt-get -y update >/dev/null; apt-get install -y wget >/dev/null; bash <(wget -q --no-check-certificate -O- https://raw.githubusercontent.com/yarikoptic/neurodebian/enh/neurodebian-installer/tools/nd-configurerepo) --overwrite --install --mirror=us-nh" 
index 452040160c2d7755ed703c1d85a5ffdceb7061b0..6d0d39e3575e8d240335067305e043faaf2f77a9 100644 (file)
@@ -1,10 +1,13 @@
-neurodebian (0.33~a1) UNRELEASED; urgency=medium
+neurodebian (0.33) unstable; urgency=medium
 
   * Support popularity-contest >= 1.61 encrypting submissions:
     - provide GPG public key for encryption of submissions
   * Updated list of currently supported Ubuntu releases
+  * New features:
+    - neurodebian and neurodebian packages to provide 1-click
+      NeuroDebian deployment on Debian/Ubuntu boxesw
 
- -- Yaroslav Halchenko <debian@onerussian.com>  Wed, 06 Aug 2014 16:21:04 -0400
+ -- Yaroslav Halchenko <debian@onerussian.com>  Fri, 08 Aug 2014 14:21:46 -0400
 
 neurodebian (0.32) neurodebian; urgency=low
 
index 0517d5d46ec757914805e3bf7a55f440e76036f2..37835341a62da6f9acbf09d6d724abd091a7676c 100644 (file)
@@ -10,6 +10,21 @@ Vcs-Browser: http://git.debian.org/?p=pkg-exppsy/neurodebian.git
 Vcs-Git: git://git.debian.org/git/pkg-exppsy/neurodebian.git
 XS-DM-Upload-Allowed: yes
 
+Package: neurodebian
+Architecture: all
+Pre-depends: python-minimal, wget
+Depends: neurodebian-keyring, ${misc:Depends}
+Recommends: netselect
+Suggests: neurodebian-desktop, neurodebian-popularity-contest
+Description: turnkey platform for the neuroscience
+ The NeuroDebian project integrates and maintain a variety of
+ neuroscience-oriented (such as AFNI, FSL, PsychoPy, etc.) and many
+ generic computational (such as condor, pandas, etc.) software projects
+ within Debian.
+ .
+ This package enables NeuroDebian repository on top of the stock
+ Debian or Ubuntu system.
+
 
 Package: neurodebian-dev
 Architecture: all
diff --git a/debian/neurodebian.config b/debian/neurodebian.config
new file mode 100755 (executable)
index 0000000..ea6f628
--- /dev/null
@@ -0,0 +1,96 @@
+#!/bin/bash
+set -e
+
+. /usr/share/debconf/confmodule
+
+db_capb backup # allow to go back, for that we implement that STATE loop
+
+debug()
+{
+    [ "${DEBCONF_DEBUG:-}" != "developer" ] || echo -e "D: $*" >&2
+}
+
+db_settitle neurodebian/title
+
+if [ "$1" = "configure" ]; then
+    # Must have been called in preinst state, so no nd-configurerepo
+    # tool is yet installed, thus delay asking questions to
+    # whenever will be called in postinst state
+    which nd-configurerepo 1>/dev/null 2>&1 || exit 0
+fi
+
+# Create temporary directory which will be reused
+export ND_AE_TEMPDIR=$(mktemp -d)
+trap "rm -rf \"$ND_AE_TEMPDIR\"" TERM INT EXIT
+
+neurodebian_releases="auto"
+
+# This implements a simple state machine so the back button can be handled.
+# taken from debconf demo example
+STATE=1
+while [ "$STATE" != 0 -a "$STATE" != 8 ]; do
+    case $STATE in
+        1)
+                       neurodebian_releases="$(ND_IFS=', ' nd-configurerepo --print-releases)"
+                       debian_release="$(ND_IFS=', ' nd-configurerepo --print-release)"
+                       if [ -z "$debian_release" ]; then # just a failover
+                               debian_release="sid"
+                       fi
+                       debug "releases: $neurodebian_releases"
+                       db_subst neurodebian/release releases "$neurodebian_releases"
+                       db_subst neurodebian/release release  "$debian_release"
+                       db_input medium neurodebian/release || true
+                       ;;
+        2)
+                       neurodebian_mirrors="$(ND_IFS=', ' nd-configurerepo --print-mirrors)"
+                       debug "mirrors: $neurodebian_mirrors"
+                       db_subst neurodebian/mirror mirrors "$neurodebian_mirrors"
+                       db_input medium neurodebian/mirror || true
+                       mirror_selection_ret="$RET"
+                       debug "mirror select return: <$mirror_selection_ret>"
+                       ;;
+        3)  neurodebian_flavor="$(ND_IFS=', ' nd-configurerepo --print-flavor)"
+                       db_subst neurodebian/flavor flavor "$neurodebian_flavor"
+                       db_input medium neurodebian/flavor || true ;;
+        4)  db_input medium neurodebian/components || true ;;
+        5)  db_input low neurodebian/overwrite || true ;;
+        6)  db_input low neurodebian/suffix || true ;;
+        7)  db_input medium neurodebian/run-update-note || true ;;
+    esac
+
+    if db_go; then
+# Ubuntus managed to remove netselect "to accompany apt-netselect"
+# https://bugs.launchpad.net/ubuntu/+source/netselect/+bug/337377
+# So for now remove this mandatory demanding of netselect for "best", and
+# rely on logic in nd-configurerepo to select default mirror (currently origin)
+# as the 'best'
+#
+#              case $STATE in
+#                      2)
+#                              db_metaget neurodebian/mirror value
+#                              selected_mirror="$RET"
+#                              # TODOs:
+#                              # - handle custom
+#                              # - separate out alias from url
+#                              debug "selected mirror: $selected_mirror"
+#                              if [ "$selected_mirror" = "best" ] && ! which netselect &>/dev/null; then
+#                                      if [ "$mirror_selection_ret" = "30 question skipped" ]; then
+#                                              # if the question was not even shown, we would
+#                                              # not get a chance to specify mirror, thus
+#                                              # better just fail altogether and demand intervention
+#                                              db_input high neurodebian/netselect-cannot-be-used
+#                                              exit 1
+#                                      else
+#                                              db_input high neurodebian/netselect-not-found
+#                                              continue;
+#                                      fi
+#                              fi
+#              esac
+        STATE=$(($STATE + 1))
+    else
+        STATE=$(($STATE - 1))
+    fi
+done
+
+db_stop
+
diff --git a/debian/neurodebian.install b/debian/neurodebian.install
new file mode 100644 (file)
index 0000000..93aa380
--- /dev/null
@@ -0,0 +1 @@
+tools/nd-configurerepo usr/bin
diff --git a/debian/neurodebian.postinst b/debian/neurodebian.postinst
new file mode 100755 (executable)
index 0000000..97cbc86
--- /dev/null
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+
+. /usr/share/debconf/confmodule
+
+
+case "$1" in
+    configure)
+               # obtain all the options and call nd-configurerepo
+               opts="--do-not-update"
+               db_get neurodebian/release; [ "$RET" = "auto" ] || opts+=" -r '$RET'"
+               db_get neurodebian/mirror;  nd_mirror="${RET##* }" # get just a url if it came together with alias
+               db_get neurodebian/flavor;  nd_flavor="$RET"
+               db_get neurodebian/components; nd_components="${RET// /}"
+
+               db_get neurodebian/overwrite;  [ "$RET" = "true" ] && opts+=" --overwrite" || :
+               db_get neurodebian/suffix;     [ "$RET" = "" ]     || opts+=" --suffix='$RET'" || :
+
+               eval nd-configurerepo -m "$nd_mirror" -c "$nd_components" $opts
+    ;;
+    abort-upgrade|abort-remove|abort-deconfigure)
+        # nothing to do
+    ;;
+
+    *)
+        echo "postinst called with unknown argument \`$1'" >&2
+        exit 1
+    ;;
+esac
+
+#DEBHELPER#
diff --git a/debian/neurodebian.postrm b/debian/neurodebian.postrm
new file mode 100755 (executable)
index 0000000..902c3f4
--- /dev/null
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+
+case "$1" in
+    purge)
+               rm -f /etc/apt/sources.list.d/neurodebian.sources*.list
+    ;;
+
+    remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear)
+    ;;
+
+    *)
+        echo "postrm called with unknown argument \`$1'" >&2
+        exit 1
+    ;;
+esac
+
+#DEBHELPER#
+
+exit 0
diff --git a/debian/neurodebian.templates b/debian/neurodebian.templates
new file mode 100644 (file)
index 0000000..1d54520
--- /dev/null
@@ -0,0 +1,106 @@
+Template: neurodebian/title
+Type: title
+Description: NeuroDebian APT repository installer
+
+
+Template: neurodebian/release
+Type: select
+Choices: auto, ${releases}
+Default: auto
+Description: Release name of the base system
+ Specify for which Debian or Ubuntu release (e.g. wheezy or trusty).
+ .
+ If 'auto', Debian or Ubuntu release name will be '${release}' as
+ deduced from the output of apt-cache policy.  If the release of your
+ system is not '${release}' -- please choose specific one which
+ matches best.
+
+
+Template: neurodebian/mirror
+Type: select
+Choices: origin, best, custom, ${mirrors}
+Default: best
+Description: NeuroDebian mirror to use:
+ NeuroDebian project has a number of community-maintainer mirrors
+ around the globe.
+ .
+ If you do not know which mirror URL to choose, select among
+ .
+  - origin: original NeuroDebian repository
+  - best: will try to use netselect to select "closest" mirror.
+    Depending on the configuration of the firewall, and actual mirror
+    setup, might fail to select actually closest one.  If netselect
+    is not available, default mirror (possibly 'origin') will be used.
+
+
+Template: neurodebian/flavor
+Type: select
+Choices: auto, libre, full
+Default: auto
+Description: NeuroDebian flavor to use:
+ NeuroDebian project adheres to Debian Free Software Guidelines and
+ has three areas of the archive in all suites/releases:
+ .
+  libre
+    only the DFSG-compliant materials
+  full
+    all three areas (main, contrib, non-free)
+  auto
+    according to output of apt-cache policy '${flavor}' will be used.
+    If that is not the flavor you need, select manually another one.
+
+Template: neurodebian/components
+Type: multiselect
+Choices: software, data, devel
+Default: software, data
+Description: Select NeuroDebian repository components to enable
+ NeuroDebian repository provides
+ .
+  software
+   Packages containing software packages, often backports of stable
+   software releases for previous Debian/Ubuntu releases
+  devel
+   Additional "bleeding edge" software packages, which might not be a
+   good fit for being enabled by default. (Similar to Debian
+   experimental)
+  data
+   Packages containing data (e.g. atlases, sample datasets), often
+   required by software packages.  Should generally be enabled.
+
+
+Template: neurodebian/overwrite
+Type: boolean
+Default: true
+Description: Should existing NeuroDebian APT file be overridden?
+ If not allowed, configuration would fail if there is
+ an existing apt configuration file for the NeuroDebian.
+
+Template: neurodebian/suffix
+Type: string
+Default:
+Description: Additional suffix to use in the NeuroDebian APT file name
+ For instance if you would like to enable additional repository
+ (e.g. NeuroDebian devel) or release, without interfering with the
+ main/default configuration file.
+
+Template: neurodebian/run-update-note
+Type: note
+Description: Upon completion of installation invoke update in your APT frontend.
+ For the installed NeuroDebian APT configuration to take an effect,
+ list of packages needs to be updated.  This needs to be done
+ "manually" (e.g. apt-get update) upon completion of installation of
+ the neurodebian package.
+
+Template: neurodebian/netselect-not-found
+Type: error
+Description: netselect tool was not found
+ Please install netselect tool first (apt-get install netselect) or
+ choose a specific mirror.
+
+Template: neurodebian/netselect-cannot-be-used
+Type: error
+Description: netselect tool was not found (no other mirror)
+ Please install netselect tool first (apt-get install netselect)
+ before installing/configuring this package, or run with a lower
+ DEBIAN_PRIORITY to choose a specific mirror.
+
diff --git a/tools/nd-configurerepo b/tools/nd-configurerepo
new file mode 100755 (executable)
index 0000000..5720fe8
--- /dev/null
@@ -0,0 +1,583 @@
+#!/bin/bash
+#emacs: -*- mode: shell-script; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
+#ex: set sts=4 ts=4 sw=4 et:
+
+# Depends:    apt (assumed to be present), python, wget
+# Recommends: netselect
+
+# play safe
+set -e
+set -u
+
+############
+# Defaults #
+############
+
+nd_aptenable_version=0.1
+
+nd_key_id=0x2649A5A9
+nd_config_url=https://raw.githubusercontent.com/neurodebian/neurodebian/master/neurodebian.cfg
+nd_config_file=/etc/neurodebian/neurodebian.cfg
+nd_mirror_origin=http://neuro.debian.net/debian
+nd_mirror_default=$nd_mirror_origin # or may be AWS?
+
+# To be set by cmdline args or via env variables with prefix ND_AE_
+ae_release=${ND_AE_RELEASE:-}
+ae_components=${ND_AE_COMPONENTS:-software,data}
+ae_flavor=${ND_AE_FLAVOR:-}
+ae_mirror=${ND_AE_MIRROR:-best}
+ae_suffix=${ND_AE_SUFFIX:-}
+ae_verbose=${ND_AE_VERBOSE:-1}
+ae_overwrite=${ND_AE_OVERWRITE:-}
+ae_sources=${ND_AE_SOURCES:-}
+ae_install=${ND_AE_INSTALL:-}
+ae_update=${ND_AE_UPDATE:-1}
+ae_dry_run=${ND_AE_DRY_RUN:-}
+ae_defun_only=${ND_AE_DEFUN_ONLY:-} # mode to source this file as a "library"
+
+ae_sudo=
+exe_dir=$(dirname $0)
+do_print_release=
+do_print_flavor=
+
+# TODOs:
+# - apt priority! (so we could avoid automagic upgrades etc)
+# - multiarch setups
+
+if [ -z "${ND_AE_TEMPDIR:-}" ]; then
+    ae_tempdir=$(mktemp -d)
+    trap "rm -rf \"$ae_tempdir\"" TERM INT EXIT
+else
+    # reuse the same directory/fetched configuration if was specified
+    ae_tempdir="${ND_AE_TEMPDIR:-}"
+fi
+
+
+nd_config_file_fresh="$ae_tempdir/neurodebian.cfg"
+
+print_verbose()
+{
+    level=$1; shift
+       if [ "$ae_verbose" -ge $level ]; then
+        # use stderr for printing within functions stdout of which might be used
+        echo -n "I: " >&2
+        i=1; while [ $i -lt $level ]; do echo -ne " ">&2; i=$(($i+1)); done
+        echo -e "$*" >&2
+    fi
+}
+
+error()
+{
+    code=$1; shift
+       echo -e "E: $*" >&2
+    exit $code
+}
+
+print_version()
+{
+    cat << EOT
+nd-configurerepo $nd_aptenable_version
+
+Copyright (C) 2014 Yaroslav Halchenko <debian@onerussian.com>
+
+Licensed under GNU Public License version 3 or later.
+This is free software; see the source for copying conditions.  There is NO
+warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+Written by Yaroslav Halchenko for the NeuroDebian project.
+
+EOT
+}
+
+eval_dry()
+{
+    if [ -z "$ae_dry_run" ]; then
+        if eval "$ae_sudo $@" 1>|"$ae_tempdir/eval.log" 2>&1; then
+            rm "$ae_tempdir/eval.log"
+        else
+            error $? "Command $@ failed with exit code $?.  Output was: `cat $ae_tempdir/eval.log`"
+        fi
+    else
+        echo "DRY: $@" >&2
+    fi
+}
+
+print_help()
+{
+    cat << EOT
+
+Usage:  nd-configurerepo [options]
+
+Enables NeuroDebian repository for the current Debian or Ubuntu release.
+
+Options:
+
+  -r, --release=RELEASE
+    Name of the Debian/Ubuntu release to be used.  If not specified,
+    it is deduced from the  apt-cache policy  output, by taking repository
+    of Debian or Ubuntu origin with highest priority.
+
+  --print-releases
+    Print a list of releases present in NeuroDebian repository.
+
+  --print-release
+    Print the release deduced from the output of apt-cache policy.
+
+  -f, --flavor=full|libre
+    Which flavor of the repository should be enabled:
+     libre -- Only  main  component, containing only DFSG-compliant content.
+     full -- Includes main, contrib, and non-free.
+    If not specified -- deduced from the output of apt-cache policy.
+
+  --print-flavor
+    Print the flavor deduced from the output of apt-cache policy.
+
+  -c, --components=c1,c2,c3
+    Comma separated list of components to enable among:
+     software -- primary software repository
+     data -- data packages
+     devel -- "overlay" of development versions (like Debians' "experimental").
+              Not sufficient on its own and available only from the main site
+    If not specified -- "software,data"
+
+  -m, --mirror=NAME|URL
+    Which mirror to use.  Could be a mirror code-name (as specified in
+    /etc/neurodebian/neurodebian.cfg), or a URL.
+
+  --print-mirrors
+    Return a list (with abbreviation) of known NeuroDebian mirrors.
+
+  --overwrite,
+    If apt file already present, it would not be overriden (by default).
+    Use this option to overwrite.
+
+  --suffix=SUFFIX
+    Which suffix to add to the apt file, in case you are trying to enable
+    multiple repositories
+
+  --sources, --no-sources
+    Either to enable deb-src lines.  If none specified -- would be enabled if
+    sources for a core package (apt) are available.
+
+  -n, --dry-run
+    Do not perform any changes -- generated configurations and commands will
+    simply be printed to stdout
+
+  --install
+    If found absent, all necessary tools (wget, netselect) if available will
+    be apt-get installed
+
+  -v, --verbose
+    Enable additional progress messages.  Could be used multiple times
+
+  -q, --quiet
+    Make operation quiet -- only error messages would be output
+
+  -h, --help
+    Print short description, usage summary and option list.
+
+  --version
+    Print version information and exit.
+
+Exit status:
+
+  non-0 exit status in case of error.  Error exit code would depend
+  on which command has failed
+
+Examples:
+
+  - Enable software and data components from the optimal (according to
+    netselect) mirror.  Some information about progress will be printed
+
+    nd-configurerepo
+
+  - Quietly enable -devel repository for the current release, and place apt
+    configuration into /etc/apt/sources.list.d/neurodebian.sources-devel.list
+
+    nd-configurerepo -q --suffix=-devel -c devel
+
+  - Force sid distribution, all the components, from the Japan mirror:
+
+    nd-configurerepo -q --suffix=-de-sid-full -c software,data,devel -m jp
+
+EOT
+}
+
+get_neurodebian_cfg()
+{
+    if [ -s "$nd_config_file_fresh" ]; then
+        print_verbose 3 "Config file $nd_config_file_fresh exists -- not fetching"
+        echo "$nd_config_file_fresh"
+        return 0
+    fi
+    # First we try to fetch the most recent version from the github
+    print_verbose 3 "Fetching config file from the github repository"
+    assure_command_from_package wget wget 1
+    wget --no-check-certificate -c -q -O$nd_config_file_fresh \
+        $nd_config_url \
+        && { echo "$nd_config_file_fresh"; } \
+        || { [ -e "$nd_config_file" ] \
+             && echo "$nd_config_file" \
+             || error 10 "Neither could fetch $nd_config_url, nor found $nd_config_file"; }
+}
+
+query_cfg_section()
+{
+    config_file="$1"
+    section="$2"
+    print_verbose 3 "Querying config $config_file section $section"
+    assure_command_from_package python python-minimal 1
+    python -c "from ConfigParser import SafeConfigParser as SP; cfg = SP(); cfg.read('$config_file'); print('\n'.join([' '.join(x) for x in cfg.items('$section')]))"
+}
+
+get_mirrors()
+{
+    nd_config=`get_neurodebian_cfg`
+#    $exe_dir/nd_querycfg -F" " --config-file="$nd_config" "mirrors" \
+    n=""
+    query_cfg_section "$nd_config" "mirrors" \
+    | while read mirror_name mirror_url; do
+        # verify that url is just a url
+        if echo "$mirror_url" | grep -v -e '^[a-z0-9:+]*://[-+_%.a-z0-9/]*$'; then
+            print_verbose 1 "Mirror $mirror_name has 'illegit' URL: $mirror_url.  Skipping"
+        fi
+        [ -z "$n" ] || echo -ne "${ND_IFS:-\n}"; n+=1
+        echo -n "$mirror_name $mirror_url"
+    done
+}
+
+get_releases()
+{
+    nd_config=`get_neurodebian_cfg`
+    n=""
+    query_cfg_section "$nd_config" "release files" \
+    | while read release_name release_url; do
+        # verify that url is just a url
+        if [ "$release_name" = "data" ]; then
+            # skip data
+            continue
+        fi
+        [ -z "$n" ] || echo -ne "${ND_IFS:-\n}"; n+=1
+        echo -n "$release_name"
+    done
+}
+
+get_package_version()
+{
+    pkg_version=$(apt-cache policy "$1" | awk '/^ *Installed:/{print $2;}')
+    [ "$pkg_version" != '(none)' ] || pkg_version=''
+    echo "$pkg_version"
+}
+
+netselect_mirror() {
+    # select "closest" mirror according to netselect.
+    print_verbose 2 "Selecting the 'best' mirror using netselect"
+    assure_command_from_package netselect
+    if ! which netselect >&/dev/null; then
+        print_verbose 1 "netselect (apt-get install netselect) needed to select the 'best' mirror was not found"
+        print_verbose 1 "Selecting the default repository: $nd_mirror_default"
+        echo $nd_mirror_default
+    else
+        # squeeze version doesn't have -D yet to force output of the URL not IP, but for our mirrors ATM it shouldn't matter
+        netselect_opts="-s 1"
+        netselect_version="$(get_package_version netselect)"
+        if dpkg --compare-versions "$netselect_version" ge 0.3.ds1-17; then
+            netselect_opts+=" -D"
+        fi
+        if dpkg --compare-versions "$netselect_version" ge 0.3.ds1-15; then
+            netselect_opts+=" -I"
+        fi
+        best_mirror=$(get_mirrors | awk '{print $2;}' | eval $ae_sudo xargs netselect $netselect_opts | awk '{print $2;}')
+        if [ -z "$best_mirror" ]; then
+            print_verbose 1 "Failed to select mirror using netselect. Selecting default one ($nd_mirror_default)"
+            echo "$nd_mirror_default"
+        else
+            print_verbose 2 "Best mirror: $best_mirror"
+            echo $best_mirror
+        fi
+    fi
+}
+
+get_mirror_url()
+{
+    # given mirror alias -- find its url
+    url=$(get_mirrors | awk "/^$1 /{print \$2;}")
+    if [ -z "$url" ]; then
+        error 9 "Cannot resolve mirror $1 to the URL"
+    fi
+    echo $url
+}
+
+get_apt_policy()
+{
+    # Get apt-cache policy output in a single list for matching suites
+    # (could be a separated with \| or , for multiple choices, e.g.
+    #
+    # get_apt_policy Debian,Ubuntu
+    # or
+    # get_apt_policy NeuroDebian
+    suites="$1"
+    $ae_sudo apt-cache policy | grep -B1 -e "o=\(${suites//,/\\|}\)" | tr '\n' ' ' | sed -e 's, -- ,\n,g' | grep -v -e '-\(updates\|security\)' | sort -nr
+}
+
+is_component_included()
+{
+    echo "$ae_components" | tr ',' '\n' | grep -q "^$1\$"
+}
+
+is_sources_enabled()
+{
+    apt-cache showsrc apt >&/dev/null && echo 1 || echo 0
+}
+
+assure_command_from_package()
+{
+    cmd=$1
+    pkg=${2:-$cmd}
+    fail=${3:-}
+
+    which "$cmd" >&/dev/null && return 0
+
+    # if absent -- check availability of the package
+    apt_cache=$(LANG=C apt-cache policy "$pkg" 2>&1)
+    if [[ "$apt_cache" =~ Unable\ to\ locate\ package ]] || [[ "$apt_cache" =~ Candidate:\ (none) ]]; then
+        print_verbose 1 "Package $pkg providing command $cmd is N/A. Skipping"
+        return 10;
+    fi
+    if echo "$apt_cache" | grep -q '^\s*\*\*\*'; then
+        print_verbose 1 "WARNING -- command $cmd is N/A but package $pkg is claimed to be installed"
+        [ -z "$fail" ] && return 11 || error $fail "Command $cmd is required to proceed"
+    fi
+    if [ "$ae_install" = "1" ]; then
+        print_verbose 1 "Installing $pkg package to get $cmd command"
+        eval_dry DEBIAN_FRONTEND=noninteractive apt-get install -y "$pkg"
+        return
+    else
+        print_verbose 1 "Command $cmd (from package $pkg) is N/A."
+        print_verbose 1 "Use with --install to get all necessary packages installed automatically"
+        [ -z "$fail" ] && return 12 || error $fail "Command $cmd is required to proceed"
+    fi
+}
+
+# if it was requested -- return without doing anything
+[ -z "$ae_defun_only" ] || return
+
+#
+# Commandline options handling
+#
+
+# Parse commandline options (taken from the getopt examples from the Debian util-linux package)
+# Note that we use `"$@"' to let each command-line parameter expand to a
+# separate word. The quotes around `$@' are essential!
+# We need CLOPTS as the `eval set --' would nuke the return value of getopt.
+CLOPTS=`getopt -o h,r:,m:,f:,c:,q,v,n --long help,version,quiet,verbose,mirror:,release:,flavor:,components:,suffix:,overwrite,sources,no-sources,install,dry-run,do-not-update,print-releases,print-release,print-mirrors,print-best-mirror,print-flavor -n 'nd-configurerepo' -- "$@"`
+
+if [ $? != 0 ] ; then
+  error 2 "Problem with parsing cmdline.  Terminating..."
+fi
+
+# Note the quotes around `$CLOPTS': they are essential!
+eval set -- "$CLOPTS"
+
+if [ `whoami` != "root" ]; then
+    ae_sudo=sudo
+fi
+
+while true ; do
+  case "$1" in
+         -r|--release) shift; ae_release="$1"; shift;;
+         -f|--flavor) shift;  ae_flavor="$1"; shift;;
+         -c|--components) shift; ae_components="$1"; shift;;
+      -m|--mirror) shift;  ae_mirror="$1"; shift;;
+         --print-mirrors)  get_mirrors; exit 0;;
+         --print-best-mirror)  netselect_mirror; exit 0;;
+         --print-releases)  get_releases; exit 0;;
+         --print-release)  do_print_release=1; shift;;
+         --print-flavor)  do_print_flavor=1; shift;;
+      -n|--dry-run)        ae_dry_run=1; shift;;
+         --suffix) shift;  ae_suffix="$1"; shift;;
+         --overwrite)      ae_overwrite="$1"; shift;;
+         --do-not-update)  ae_update=""; shift;;
+         --sources)        ae_sources=1; shift;;
+         --no-sources)     ae_sources=0; shift;;
+         --install)        ae_install=1; shift;;
+         -q|--quiet)          ae_verbose=0; shift;;
+         -v|--verbose)        ae_verbose=$(($ae_verbose+1)); shift;;
+         -h|--help) print_help; exit 0;;
+         --version) print_version; exit 0;;
+         --) shift ; break ;;
+         *) error 1 "Internal error! ($1)";;
+  esac
+done
+
+
+if [ $# -gt 0 ] ; then
+    print_help >&2
+    exit 2
+fi
+
+# Inform!
+[ -z "$ae_sudo" ] || print_verbose 1 "This script requires root access.  Since current user is not root, sudo will be used"
+
+#
+# Basic system/environment knowledge
+#
+
+ae_output_file=/etc/apt/sources.list.d/neurodebian.sources${ae_suffix}.list
+
+apt_policy=$(get_apt_policy "Debian,Ubuntu" )
+
+if [ -z "$ae_release" ]; then
+    ae_release=$(echo "$apt_policy" | head -1 | sed -e 's/.*,n=\([^,]*\),.*/\1/g')
+    if [ ! -z "$do_print_release" ]; then
+        echo $ae_release
+        exit 0
+    fi
+fi
+
+if [ -z "$ae_flavor" ]; then
+    ae_flavor=$(echo "$apt_policy" | grep -e ",n=$ae_release," | grep -qe 'c=\(non-free\|multiverse\)' && echo "full" || echo "libre")
+    if [ ! -z "$do_print_flavor" ]; then
+        echo $ae_flavor
+        exit 0
+    fi
+fi
+
+#
+# Determine which mirror to use
+#
+
+# knowing mirror is not necessary for -devel available only from the main site
+if is_component_included software || is_component_included data; then
+    # for now just use default
+    if [ -z "$ae_mirror" ]; then # none specified
+        ae_mirror_url=$nd_mirror_origin
+    else
+        if ! [[ "$ae_mirror" =~ .*://.* ]]; then
+            case "$ae_mirror" in
+                best)    ae_mirror_url=$(netselect_mirror);;
+                default) ae_mirror_url=$nd_mirror_default;;
+                origin)  ae_mirror_url=$nd_mirror_origin;;
+                *)       ae_mirror_url=$(get_mirror_url "$ae_mirror");;
+            esac
+        else
+            ae_mirror_url="$ae_mirror" # it was some kind of a URL already
+        fi
+    fi
+fi
+
+#
+# Prepare APT file
+#
+
+case $ae_flavor in
+ full)  apt_flavor="contrib non-free";;
+ libre) apt_flavor="";;
+ *) error 5 "Unknown value of flavor $apt_flavor.  Must be full or libre"
+esac
+
+if [ -z "$ae_sources" ]; then
+    ae_sources=$(is_sources_enabled)
+fi
+
+if [ $ae_sources -eq 0 ]; then
+    sources_comment="#"
+else
+    sources_comment=""
+fi
+
+apt_list=
+
+if is_component_included software; then
+    apt_list+="
+# NeuroDebian software repository
+deb     $ae_mirror_url $ae_release main $apt_flavor
+${sources_comment}deb-src $ae_mirror_url $ae_release main $apt_flavor
+"
+fi
+
+if is_component_included data; then
+    apt_list+="
+# NeuroDebian data repository
+deb     $ae_mirror_url data main $apt_flavor
+${sources_comment}deb-src $ae_mirror_url data main $apt_flavor
+"
+fi
+
+if is_component_included devel; then
+    apt_list+="
+# NeuroDebian -devel repository
+deb     http://neuro.debian.net/debian-devel $ae_release main $apt_flavor
+${sources_comment}deb-src http://neuro.debian.net/debian-devel $ae_release main $apt_flavor
+"
+fi
+
+if [ -e "$ae_output_file" ] && [ -z "$ae_overwrite" ]; then
+    if diff "$ae_output_file" <(echo "$apt_list") | grep -q .; then
+        # error 3
+        print_verbose 1 "File $ae_output_file already exists, containing:\n`cat \"$ae_output_file\"`\n\nI: New configuration is different:\n$apt_list"
+        if get_apt_policy NeuroDebian >/dev/null; then
+            print_verbose 1 "NeuroDebian repositories are already available, thus skipping the rest."
+            print_verbose 1 "Rerun with --overwrite if you would like to reconfigure."
+            exit 0
+        else
+            print_verbose 1 "NeuroDebian configuration is found but not yet available -- continuing with new configuration."
+        fi
+    else
+        print_verbose 1 "New configuration is identical to existing and NeuroDebian repository is already enabled."
+        print_verbose 1 "Skiping the rest. Rerun with --overwrite if you would like to reconfigure."
+        exit 0
+    fi
+else
+    print_verbose 1 "Generating $ae_output_file"
+    if [ -z "$ae_dry_run" ]; then
+        echo "$apt_list" | $ae_sudo bash -c "cat - >| '$ae_output_file'"
+    else
+        echo "DRY:"
+        echo "$apt_list"
+    fi
+fi
+
+
+#
+# Assure present archive GPG key for APT system
+#
+
+# Figure out if key needs to be imported (if ran within package,
+# should already be there due to neurodebian-keyring package)
+if LANG=C eval $ae_sudo apt-key export $nd_key_id 2>&1 1>/dev/null | grep -qe "nothing exported"; then
+    print_verbose 1 "Fetching the key from the server"
+    eval_dry apt-key adv --recv-keys --keyserver pgp.mit.edu $nd_key_id
+fi
+
+
+#
+# Finalizing (apt-get update etc)
+#
+
+if [ ! -z "$ae_update" ]; then
+    print_verbose 1 "Updating APT listings, might take a few minutes"
+    if [ -z "$ae_dry_run" ]; then
+        apt_logfile="$ae_tempdir/apt.log"
+        $ae_sudo apt-get update 1>"$apt_logfile" 2>&1 \
+            && rm -f "$apt_logfile" \
+            || {
+                 apt_log=$(cat "$apt_logfile")
+                 echo "$apt_log"
+                 if echo "$apt_log" | grep -q "Malformed line [0-9]* in source list $ae_output_file"; then
+                     $ae_sudo mv "${ae_output_file}" "${ae_output_file}-failed.disabled"
+                     error 6 "Update failed to possible errorneous APT listing file produced by this script.  Generated $ae_output_file renamed to ${ae_output_file}-failed.disabled to not interfer"
+                 fi
+                 error 5 "Update failed with exit code $? (above output logged into $apt_logfile)."
+                 }
+    else
+        eval_dry apt-get update
+    fi
+else
+    print_verbose 1 "apt-get update  was not run. Please run to take an effect of changes"
+fi
+
+if [ "$ae_verbose" -ge 2 ]; then
+    print_verbose 2 "Currently enabled NeuroDebian suites/mirrors:"
+    get_apt_policy NeuroDebian
+fi
diff --git a/tools/tests/run_in_debian_dockers b/tools/tests/run_in_debian_dockers
new file mode 100755 (executable)
index 0000000..b12098c
--- /dev/null
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -u
+test=$1
+
+docker.io images 2>&1 | grep -e  "\(debian\|ubuntu\)[ ]*[0-9]"  | sort | awk '{print $1,$2;}' \
+| while read d v; do
+    echo "== $d $v"
+    chronic docker.io run --rm=true -v $PWD:$PWD $d:$v /bin/bash $PWD/$test
+done
diff --git a/tools/tests/test_neurodebian-repository-deployment b/tools/tests/test_neurodebian-repository-deployment
new file mode 100755 (executable)
index 0000000..6900417
--- /dev/null
@@ -0,0 +1,44 @@
+#/bin/bash
+set -ex
+
+# minimal installation with few of those packages
+apt_file=/etc/apt/sources.list.d/local.list
+apt_nd_file=/etc/apt/sources.list.d/neurodebian.sources.list
+
+# clean up from a previous test
+dpkg --purge neurodebian neurodebian-keyring || :
+echo PURGE | debconf-communicate neurodebian || :
+rm -rf /tmp/apt $apt_file;
+
+mkdir -p /tmp/apt; cd /tmp/apt;
+rm -f neurodebian-*deb; apt-get update -y; apt-get install -y wget;
+wget http://www.onerussian.com/tmp/neurodebian_0.33~a1_all.deb;
+wget http://www.onerussian.com/tmp/neurodebian-keyring_0.33~a1_all.deb;
+
+# generate APT repo
+apt-get install -y dpkg-dev ;  dpkg-scanpackages  . /dev/null > Packages
+# Add it to the list of available
+echo "deb file:///tmp/apt ./" >| $apt_file; apt-get update
+# Install the beast
+apt-get install -y --allow-unauthenticated neurodebian
+# Verify that it has correct entries
+# Very cruel way to figure out the default release. Would fail if
+# options, such as multiarch, are specified after deb
+release=$(grep -e '^\s*deb\s' /etc/apt/sources.list | head -1 | awk '{print $3;}')
+grep -E "^deb\s+http.* $release main\s*\$" $apt_nd_file
+grep -E '^deb\s+http.* data main\s*$'      $apt_nd_file
+# May be even retrospect stored debconf configuration
+apt-get install debconf-utils
+debconf-get-selections | grep neurodebian/release.*select.*auto
+# And apt-get must be working
+apt-get update
+# Make sure it is active
+apt-cache policy | grep o=NeuroDebian,.*,n=data
+apt-cache policy | grep o=NeuroDebian,.*,n=$release
+
+# Now lets remove
+dpkg --purge neurodebian
+[ ! -e $apt_nd_file ]
+apt-cache policy | grep o=NeuroDebian && exit 1 || :
+
+echo "I: success"