From: Yaroslav Halchenko Date: Thu, 7 Aug 2014 17:01:43 +0000 (-0400) Subject: RF: nd-aptenable -> nd-configurerepo per our discussion X-Git-Url: https://git.donarmstrong.com/?a=commitdiff_plain;ds=sidebyside;h=0468a1159b6a8349389f2300d8b3648f80283bed;p=neurodebian.git RF: nd-aptenable -> nd-configurerepo per our discussion --- diff --git a/debian/TODO b/debian/TODO index c1c3f9c..33dfa55 100644 --- a/debian/TODO +++ b/debian/TODO @@ -50,4 +50,4 @@ apt-cache policy | grep o=NeuroDebian,.*,n=$release echo "I: success" -*** ~/bin/docker-debians-run /bin/bash -c "apt-get -y update >/dev/null; apt-get install -y wget >/dev/null; bash <(wget -q --no-check-certificate -O- https://raw.githubusercontent.com/yarikoptic/neurodebian/enh/neurodebian-installer/tools/nd-aptenable) --overwrite --install --mirror=us-nh" +*** ~/bin/docker-debians-run /bin/bash -c "apt-get -y update >/dev/null; apt-get install -y wget >/dev/null; bash <(wget -q --no-check-certificate -O- https://raw.githubusercontent.com/yarikoptic/neurodebian/enh/neurodebian-installer/tools/nd-configurerepo) --overwrite --install --mirror=us-nh" diff --git a/debian/neurodebian-repository.config b/debian/neurodebian-repository.config index 0f3776c..4d9ee97 100755 --- a/debian/neurodebian-repository.config +++ b/debian/neurodebian-repository.config @@ -13,10 +13,10 @@ debug() db_settitle neurodebian-repository/title if [ "$1" = "configure" ]; then - # Must have been called in preinst state, so no nd-aptenable + # Must have been called in preinst state, so no nd-configurerepo # tool is yet installed, thus delay asking questions to # whenever will be called in postinst state - which nd-aptenable 1>/dev/null 2>&1 || exit 0 + which nd-configurerepo 1>/dev/null 2>&1 || exit 0 fi # Create temporary directory which will be reused @@ -31,13 +31,13 @@ STATE=1 while [ "$STATE" != 0 -a "$STATE" != 8 ]; do case $STATE in 1) - export neurodebian_releases="$(ND_IFS=', ' nd-aptenable --print-releases)" + export neurodebian_releases="$(ND_IFS=', ' nd-configurerepo --print-releases)" debug "releases: $neurodebian_releases" db_subst neurodebian-repository/release releases "$neurodebian_releases" db_input medium neurodebian-repository/release || true ;; 2) - export neurodebian_mirrors="$(ND_IFS=', ' nd-aptenable --print-mirrors)" + export neurodebian_mirrors="$(ND_IFS=', ' nd-configurerepo --print-mirrors)" debug "mirrors: $neurodebian_mirrors" db_subst neurodebian-repository/mirror mirrors "$neurodebian_mirrors" db_input medium neurodebian-repository/mirror || true @@ -55,7 +55,7 @@ while [ "$STATE" != 0 -a "$STATE" != 8 ]; do # Ubuntus managed to remove netselect "to accompany apt-netselect" # https://bugs.launchpad.net/ubuntu/+source/netselect/+bug/337377 # So for now remove this mandatory demanding of netselect for "best", and -# rely on logic in nd-aptenable to select default mirror (currently origin) +# rely on logic in nd-configurerepo to select default mirror (currently origin) # as the 'best' # # case $STATE in diff --git a/debian/neurodebian-repository.install b/debian/neurodebian-repository.install index 8760a43..93aa380 100644 --- a/debian/neurodebian-repository.install +++ b/debian/neurodebian-repository.install @@ -1 +1 @@ -tools/nd-aptenable usr/bin +tools/nd-configurerepo usr/bin diff --git a/debian/neurodebian-repository.postinst b/debian/neurodebian-repository.postinst index 49dda25..dcae7cc 100755 --- a/debian/neurodebian-repository.postinst +++ b/debian/neurodebian-repository.postinst @@ -7,7 +7,7 @@ set -e case "$1" in configure) - # obtain all the options and call nd-aptenable + # obtain all the options and call nd-configurerepo opts="--do-not-update" db_get neurodebian-repository/release; [ "$RET" = "auto" ] || opts+=" -r '$RET'" db_get neurodebian-repository/mirror; nd_mirror="${RET##* }" # get just a url if it came together with alias @@ -17,7 +17,7 @@ case "$1" in db_get neurodebian-repository/overwrite; [ "$RET" = "true" ] && opts+=" --overwrite" || : db_get neurodebian-repository/suffix; [ "$RET" = "" ] || opts+=" --suffix='$RET'" || : - eval nd-aptenable -m "$nd_mirror" -c "$nd_components" $opts + eval nd-configurerepo -m "$nd_mirror" -c "$nd_components" $opts ;; abort-upgrade|abort-remove|abort-deconfigure) # nothing to do diff --git a/tools/nd-aptenable b/tools/nd-aptenable deleted file mode 100755 index d2c2c9d..0000000 --- a/tools/nd-aptenable +++ /dev/null @@ -1,565 +0,0 @@ -#!/bin/bash -#emacs: -*- mode: shell-script; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- -#ex: set sts=4 ts=4 sw=4 et: - -# Depends: apt (assumed to be present), python, wget -# Recommends: netselect - -# play safe -set -e -set -u - -############ -# Defaults # -############ - -nd_aptenable_version=0.1 - -nd_key_id=0x2649A5A9 -nd_config_url=https://raw.githubusercontent.com/neurodebian/neurodebian/master/neurodebian.cfg -nd_config_file=/etc/neurodebian/neurodebian.cfg -nd_mirror_origin=http://neuro.debian.net/debian -nd_mirror_default=$nd_mirror_origin # or may be AWS? - -# To be set by cmdline args or via env variables with prefix ND_AE_ -ae_release=${ND_AE_RELEASE:-} -ae_components=${ND_AE_COMPONENTS:-software,data} -ae_flavor=${ND_AE_FLAVOR:-} -ae_mirror=${ND_AE_MIRROR:-best} -ae_suffix=${ND_AE_SUFFIX:-} -ae_verbose=${ND_AE_VERBOSE:-1} -ae_overwrite=${ND_AE_OVERWRITE:-} -ae_sources=${ND_AE_SOURCES:-} -ae_install=${ND_AE_INSTALL:-} -ae_update=${ND_AE_UPDATE:-1} -ae_dry_run=${ND_AE_DRY_RUN:-} -ae_defun_only=${ND_AE_DEFUN_ONLY:-} # mode to source this file as a "library" - -ae_sudo= -exe_dir=$(dirname $0) - -# TODOs: -# - apt priority! (so we could avoid automagic upgrades etc) -# - multiarch setups - -if [ -z "${ND_AE_TEMPDIR:-}" ]; then - ae_tempdir=$(mktemp -d) - trap "rm -rf \"$ae_tempdir\"" TERM INT EXIT -else - # reuse the same directory/fetched configuration if was specified - ae_tempdir="${ND_AE_TEMPDIR:-}" -fi - - -nd_config_file_fresh="$ae_tempdir/neurodebian.cfg" - -print_verbose() -{ - level=$1; shift - if [ "$ae_verbose" -ge $level ]; then - # use stderr for printing within functions stdout of which might be used - echo -n "I: " >&2 - i=1; while [ $i -lt $level ]; do echo -ne " ">&2; i=$(($i+1)); done - echo -e "$*" >&2 - fi -} - -error() -{ - code=$1; shift - echo -e "E: $*" >&2 - exit $code -} - -print_version() -{ - cat << EOT -nd-aptenable $nd_aptenable_version - -Copyright (C) 2014 Yaroslav Halchenko - -Licensed under GNU Public License version 3 or later. -This is free software; see the source for copying conditions. There is NO -warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - -Written by Yaroslav Halchenko for the NeuroDebian project. - -EOT -} - -eval_dry() -{ - if [ -z "$ae_dry_run" ]; then - if eval "$ae_sudo $@" 1>|"$ae_tempdir/eval.log" 2>&1; then - rm "$ae_tempdir/eval.log" - else - error $? "Command $@ failed with exit code $?. Output was: `cat $ae_tempdir/eval.log`" - fi - else - echo "DRY: $@" >&2 - fi -} - -print_help() -{ - cat << EOT - -Usage: nd-aptenable [options] - -Enables NeuroDebian repository for the current Debian or Ubuntu release. - -Options: - - -r, --release=RELEASE - Name of the Debian/Ubuntu release to be used. If not specified, - it is deduced from the apt-cache policy output, by taking repository - of Debian or Ubuntu origin with highest priority. - - --print-releases - Return a list of releases present in NeuroDebian repository. - - -f, --flavor=full|libre - Which flavor of the repository should be enabled: - libre -- Only main component, containing only DFSG-compliant content. - full -- Includes main, contrib, and non-free. - If not specified -- deduced from the output of apt-cache policy - - -c, --components=c1,c2,c3 - Comma separated list of components to enable among: - software -- primary software repository - data -- data packages - devel -- "overlay" of development versions (like Debians' "experimental"). - Not sufficient on its own and available only from the main site - If not specified -- "software,data" - - -m, --mirror=NAME|URL - Which mirror to use. Could be a mirror code-name (as specified in - /etc/neurodebian/neurodebian.cfg), or a URL. - - --print-mirrors - Return a list (with abbreviation) of known NeuroDebian mirrors. - - --overwrite, - If apt file already present, it would not be overriden (by default). - Use this option to overwrite. - - --suffix=SUFFIX - Which suffix to add to the apt file, in case you are trying to enable - multiple repositories - - --sources, --no-sources - Either to enable deb-src lines. If none specified -- would be enabled if - sources for a core package (apt) are available. - - -n, --dry-run - Do not perform any changes -- generated configurations and commands will - simply be printed to stdout - - --install - If found absent, all necessary tools (wget, netselect) if available will - be apt-get installed - - -v, --verbose - Enable additional progress messages. Could be used multiple times - - -q, --quiet - Make operation quiet -- only error messages would be output - - -h, --help - Print short description, usage summary and option list. - - --version - Print version information and exit. - -Exit status: - - non-0 exit status in case of error. Error exit code would depend - on which command has failed - -Examples: - - - Enable software and data components from the optimal (according to - netselect) mirror. Some information about progress will be printed - - nd-aptenable - - - Quietly enable -devel repository for the current release, and place apt - configuration into /etc/apt/sources.list.d/neurodebian.sources-devel.list - - nd-aptenable -q --suffix=-devel -c devel - - - Force sid distribution, all the components, from the Japan mirror: - - nd-aptenable -q --suffix=-de-sid-full -c software,data,devel -m jp - -EOT -} - -get_neurodebian_cfg() -{ - if [ -s "$nd_config_file_fresh" ]; then - print_verbose 3 "Config file $nd_config_file_fresh exists -- not fetching" - echo "$nd_config_file_fresh" - return 0 - fi - # First we try to fetch the most recent version from the github - print_verbose 3 "Fetching config file from the github repository" - assure_command_from_package wget wget 1 - wget --no-check-certificate -c -q -O$nd_config_file_fresh \ - $nd_config_url \ - && { echo "$nd_config_file_fresh"; } \ - || { [ -e "$nd_config_file" ] \ - && echo "$nd_config_file" \ - || error 10 "Neither could fetch $nd_config_url, nor found $nd_config_file"; } -} - -query_cfg_section() -{ - config_file="$1" - section="$2" - print_verbose 3 "Querying config $config_file section $section" - assure_command_from_package python python-minimal 1 - python -c "from ConfigParser import SafeConfigParser as SP; cfg = SP(); cfg.read('$config_file'); print('\n'.join([' '.join(x) for x in cfg.items('$section')]))" -} - -get_mirrors() -{ - nd_config=`get_neurodebian_cfg` -# $exe_dir/nd_querycfg -F" " --config-file="$nd_config" "mirrors" \ - n="" - query_cfg_section "$nd_config" "mirrors" \ - | while read mirror_name mirror_url; do - # verify that url is just a url - if echo "$mirror_url" | grep -v -e '^[a-z0-9:+]*://[-+_%.a-z0-9/]*$'; then - print_verbose 1 "Mirror $mirror_name has 'illegit' URL: $mirror_url. Skipping" - fi - [ -z "$n" ] || echo -ne "${ND_IFS:-\n}"; n+=1 - echo -n "$mirror_name $mirror_url" - done -} - -get_releases() -{ - nd_config=`get_neurodebian_cfg` - n="" - query_cfg_section "$nd_config" "release files" \ - | while read release_name release_url; do - # verify that url is just a url - if [ "$release_name" = "data" ]; then - # skip data - continue - fi - [ -z "$n" ] || echo -ne "${ND_IFS:-\n}"; n+=1 - echo -n "$release_name" - done -} - -get_package_version() -{ - pkg_version=$(apt-cache policy "$1" | awk '/^ *Installed:/{print $2;}') - [ "$pkg_version" != '(none)' ] || pkg_version='' - echo "$pkg_version" -} - -netselect_mirror() { - # select "closest" mirror according to netselect. - print_verbose 2 "Selecting the 'best' mirror using netselect" - assure_command_from_package netselect - if ! which netselect >&/dev/null; then - print_verbose 1 "netselect (apt-get install netselect) needed to select the 'best' mirror was not found" - print_verbose 1 "Selecting the default repository: $nd_mirror_default" - echo $nd_mirror_default - else - # squeeze version doesn't have -D yet to force output of the URL not IP, but for our mirrors ATM it shouldn't matter - netselect_opts="-s 1" - netselect_version="$(get_package_version netselect)" - if dpkg --compare-versions "$netselect_version" ge 0.3.ds1-17; then - netselect_opts+=" -D" - fi - if dpkg --compare-versions "$netselect_version" ge 0.3.ds1-15; then - netselect_opts+=" -I" - fi - best_mirror=$(get_mirrors | awk '{print $2;}' | eval $ae_sudo xargs netselect $netselect_opts | awk '{print $2;}') - if [ -z "$best_mirror" ]; then - print_verbose 1 "Failed to select mirror using netselect. Selecting default one ($nd_mirror_default)" - echo "$nd_mirror_default" - else - print_verbose 2 "Best mirror: $best_mirror" - echo $best_mirror - fi - fi -} - -get_mirror_url() -{ - # given mirror alias -- find its url - url=$(get_mirrors | awk "/^$1 /{print \$2;}") - if [ -z "$url" ]; then - error 9 "Cannot resolve mirror $1 to the URL" - fi - echo $url -} - -get_apt_policy() -{ - # Get apt-cache policy output in a single list for matching suites - # (could be a separated with \| or , for multiple choices, e.g. - # - # get_apt_policy Debian,Ubuntu - # or - # get_apt_policy NeuroDebian - suites="$1" - $ae_sudo apt-cache policy | grep -B1 -e "o=\(${suites//,/\\|}\)" | tr '\n' ' ' | sed -e 's, -- ,\n,g' | grep -v -e '-\(updates\|security\)' | sort -nr -} - -is_component_included() -{ - echo "$ae_components" | tr ',' '\n' | grep -q "^$1\$" -} - -is_sources_enabled() -{ - apt-cache showsrc apt >&/dev/null && echo 1 || echo 0 -} - -assure_command_from_package() -{ - cmd=$1 - pkg=${2:-$cmd} - fail=${3:-} - - which "$cmd" >&/dev/null && return 0 - - # if absent -- check availability of the package - apt_cache=$(LANG=C apt-cache policy "$pkg" 2>&1) - if [[ "$apt_cache" =~ Unable\ to\ locate\ package ]] || [[ "$apt_cache" =~ Candidate:\ (none) ]]; then - print_verbose 1 "Package $pkg providing command $cmd is N/A. Skipping" - return 10; - fi - if echo "$apt_cache" | grep -q '^\s*\*\*\*'; then - print_verbose 1 "WARNING -- command $cmd is N/A but package $pkg is claimed to be installed" - [ -z "$fail" ] && return 11 || error $fail "Command $cmd is required to proceed" - fi - if [ "$ae_install" = "1" ]; then - print_verbose 1 "Installing $pkg package to get $cmd command" - eval_dry DEBIAN_FRONTEND=noninteractive apt-get install -y "$pkg" - return - else - print_verbose 1 "Command $cmd (from package $pkg) is N/A." - print_verbose 1 "Use with --install to get all necessary packages installed automatically" - [ -z "$fail" ] && return 12 || error $fail "Command $cmd is required to proceed" - fi -} - -# if it was requested -- return without doing anything -[ -z "$ae_defun_only" ] || return - -# -# Commandline options handling -# - -# Parse commandline options (taken from the getopt examples from the Debian util-linux package) -# Note that we use `"$@"' to let each command-line parameter expand to a -# separate word. The quotes around `$@' are essential! -# We need CLOPTS as the `eval set --' would nuke the return value of getopt. -CLOPTS=`getopt -o h,r:,m:,f:,c:,q,v,n --long help,version,quiet,verbose,mirror:,release:,flavor:,components:,suffix:,overwrite,sources,no-sources,install,dry-run,do-not-update,print-releases,print-mirrors,print-best-mirror -n 'nd-aptenable' -- "$@"` - -if [ $? != 0 ] ; then - error 2 "Problem with parsing cmdline. Terminating..." -fi - -# Note the quotes around `$CLOPTS': they are essential! -eval set -- "$CLOPTS" - -if [ `whoami` != "root" ]; then - ae_sudo=sudo -fi - -while true ; do - case "$1" in - -r|--release) shift; ae_release="$1"; shift;; - -f|--flavor) shift; ae_flavor="$1"; shift;; - -c|--components) shift; ae_components="$1"; shift;; - -m|--mirror) shift; ae_mirror="$1"; shift;; - --print-mirrors) get_mirrors; exit 0;; - --print-best-mirror) netselect_mirror; exit 0;; - --print-releases) get_releases; exit 0;; - -n|--dry-run) ae_dry_run=1; shift;; - --suffix) shift; ae_suffix="$1"; shift;; - --overwrite) ae_overwrite="$1"; shift;; - --do-not-update) ae_update=""; shift;; - --sources) ae_sources=1; shift;; - --no-sources) ae_sources=0; shift;; - --install) ae_install=1; shift;; - -q|--quiet) ae_verbose=0; shift;; - -v|--verbose) ae_verbose=$(($ae_verbose+1)); shift;; - -h|--help) print_help; exit 0;; - --version) print_version; exit 0;; - --) shift ; break ;; - *) error 1 "Internal error! ($1)";; - esac -done - - -if [ $# -gt 0 ] ; then - print_help >&2 - exit 2 -fi - -# Inform! -[ -z "$ae_sudo" ] || print_verbose 1 "This script requires root access. Since current user is not root, sudo will be used" - -# -# Basic system/environment knowledge -# - -ae_output_file=/etc/apt/sources.list.d/neurodebian.sources${ae_suffix}.list - -apt_policy=$(get_apt_policy "Debian,Ubuntu" ) - -if [ -z "$ae_release" ]; then - ae_release=$(echo "$apt_policy" | head -1 | sed -e 's/.*,n=\([^,]*\),.*/\1/g') -fi - -if [ -z "$ae_flavor" ]; then - ae_flavor=$(echo "$apt_policy" | grep -e ",n=$ae_release," | grep -qe 'c=\(non-free\|multiverse\)' && echo "full" || echo "libre") -fi - -# -# Determine which mirror to use -# - -# knowing mirror is not necessary for -devel available only from the main site -if is_component_included software || is_component_included data; then - # for now just use default - if [ -z "$ae_mirror" ]; then # none specified - ae_mirror_url=$nd_mirror_origin - else - if ! [[ "$ae_mirror" =~ .*://.* ]]; then - case "$ae_mirror" in - best) ae_mirror_url=$(netselect_mirror);; - default) ae_mirror_url=$nd_mirror_default;; - origin) ae_mirror_url=$nd_mirror_origin;; - *) ae_mirror_url=$(get_mirror_url "$ae_mirror");; - esac - else - ae_mirror_url="$ae_mirror" # it was some kind of a URL already - fi - fi -fi - -# -# Prepare APT file -# - -case $ae_flavor in - full) apt_flavor="contrib non-free";; - libre) apt_flavor="";; - *) error 5 "Unknown value of flavor $apt_flavor. Must be full or libre" -esac - -if [ -z "$ae_sources" ]; then - ae_sources=$(is_sources_enabled) -fi - -if [ $ae_sources -eq 0 ]; then - sources_comment="#" -else - sources_comment="" -fi - -apt_list= - -if is_component_included software; then - apt_list+=" -# NeuroDebian software repository -deb $ae_mirror_url $ae_release main $apt_flavor -${sources_comment}deb-src $ae_mirror_url $ae_release main $apt_flavor -" -fi - -if is_component_included data; then - apt_list+=" -# NeuroDebian data repository -deb $ae_mirror_url data main $apt_flavor -${sources_comment}deb-src $ae_mirror_url data main $apt_flavor -" -fi - -if is_component_included devel; then - apt_list+=" -# NeuroDebian -devel repository -deb http://neuro.debian.net/debian-devel $ae_release main $apt_flavor -${sources_comment}deb-src http://neuro.debian.net/debian-devel $ae_release main $apt_flavor -" -fi - -if [ -e "$ae_output_file" ] && [ -z "$ae_overwrite" ]; then - if diff "$ae_output_file" <(echo "$apt_list") | grep -q .; then - # error 3 - print_verbose 1 "File $ae_output_file already exists, containing:\n`cat \"$ae_output_file\"`\n\nI: New configuration is different:\n$apt_list" - if get_apt_policy NeuroDebian >/dev/null; then - print_verbose 1 "NeuroDebian repositories are already available, thus skipping the rest." - print_verbose 1 "Rerun with --overwrite if you would like to reconfigure." - exit 0 - else - print_verbose 1 "NeuroDebian configuration is found but not yet available -- continuing with new configuration." - fi - else - print_verbose 1 "New configuration is identical to existing and NeuroDebian repository is already enabled." - print_verbose 1 "Skiping the rest. Rerun with --overwrite if you would like to reconfigure." - exit 0 - fi -else - print_verbose 1 "Generating $ae_output_file" - if [ -z "$ae_dry_run" ]; then - echo "$apt_list" | $ae_sudo bash -c "cat - >| '$ae_output_file'" - else - echo "DRY:" - echo "$apt_list" - fi -fi - - -# -# Assure present archive GPG key for APT system -# - -# Figure out if key needs to be imported (if ran within package, -# should already be there due to neurodebian-keyring package) -if LANG=C eval $ae_sudo apt-key export $nd_key_id 2>&1 1>/dev/null | grep -qe "nothing exported"; then - print_verbose 1 "Fetching the key from the server" - eval_dry apt-key adv --recv-keys --keyserver pgp.mit.edu $nd_key_id -fi - - -# -# Finalizing (apt-get update etc) -# - -if [ ! -z "$ae_update" ]; then - print_verbose 1 "Updating APT listings, might take a few minutes" - if [ -z "$ae_dry_run" ]; then - apt_logfile="$ae_tempdir/apt.log" - $ae_sudo apt-get update 1>"$apt_logfile" 2>&1 \ - && rm -f "$apt_logfile" \ - || { - apt_log=$(cat "$apt_logfile") - echo "$apt_log" - if echo "$apt_log" | grep -q "Malformed line [0-9]* in source list $ae_output_file"; then - $ae_sudo mv "${ae_output_file}" "${ae_output_file}-failed.disabled" - error 6 "Update failed to possible errorneous APT listing file produced by this script. Generated $ae_output_file renamed to ${ae_output_file}-failed.disabled to not interfer" - fi - error 5 "Update failed with exit code $? (above output logged into $apt_logfile)." - } - else - eval_dry apt-get update - fi -else - print_verbose 1 "apt-get update was not run. Please run to take an effect of changes" -fi - -if [ "$ae_verbose" -ge 2 ]; then - print_verbose 2 "Currently enabled NeuroDebian suites/mirrors:" - get_apt_policy NeuroDebian -fi diff --git a/tools/nd-configurerepo b/tools/nd-configurerepo new file mode 100755 index 0000000..72993d4 --- /dev/null +++ b/tools/nd-configurerepo @@ -0,0 +1,565 @@ +#!/bin/bash +#emacs: -*- mode: shell-script; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- +#ex: set sts=4 ts=4 sw=4 et: + +# Depends: apt (assumed to be present), python, wget +# Recommends: netselect + +# play safe +set -e +set -u + +############ +# Defaults # +############ + +nd_aptenable_version=0.1 + +nd_key_id=0x2649A5A9 +nd_config_url=https://raw.githubusercontent.com/neurodebian/neurodebian/master/neurodebian.cfg +nd_config_file=/etc/neurodebian/neurodebian.cfg +nd_mirror_origin=http://neuro.debian.net/debian +nd_mirror_default=$nd_mirror_origin # or may be AWS? + +# To be set by cmdline args or via env variables with prefix ND_AE_ +ae_release=${ND_AE_RELEASE:-} +ae_components=${ND_AE_COMPONENTS:-software,data} +ae_flavor=${ND_AE_FLAVOR:-} +ae_mirror=${ND_AE_MIRROR:-best} +ae_suffix=${ND_AE_SUFFIX:-} +ae_verbose=${ND_AE_VERBOSE:-1} +ae_overwrite=${ND_AE_OVERWRITE:-} +ae_sources=${ND_AE_SOURCES:-} +ae_install=${ND_AE_INSTALL:-} +ae_update=${ND_AE_UPDATE:-1} +ae_dry_run=${ND_AE_DRY_RUN:-} +ae_defun_only=${ND_AE_DEFUN_ONLY:-} # mode to source this file as a "library" + +ae_sudo= +exe_dir=$(dirname $0) + +# TODOs: +# - apt priority! (so we could avoid automagic upgrades etc) +# - multiarch setups + +if [ -z "${ND_AE_TEMPDIR:-}" ]; then + ae_tempdir=$(mktemp -d) + trap "rm -rf \"$ae_tempdir\"" TERM INT EXIT +else + # reuse the same directory/fetched configuration if was specified + ae_tempdir="${ND_AE_TEMPDIR:-}" +fi + + +nd_config_file_fresh="$ae_tempdir/neurodebian.cfg" + +print_verbose() +{ + level=$1; shift + if [ "$ae_verbose" -ge $level ]; then + # use stderr for printing within functions stdout of which might be used + echo -n "I: " >&2 + i=1; while [ $i -lt $level ]; do echo -ne " ">&2; i=$(($i+1)); done + echo -e "$*" >&2 + fi +} + +error() +{ + code=$1; shift + echo -e "E: $*" >&2 + exit $code +} + +print_version() +{ + cat << EOT +nd-configurerepo $nd_aptenable_version + +Copyright (C) 2014 Yaroslav Halchenko + +Licensed under GNU Public License version 3 or later. +This is free software; see the source for copying conditions. There is NO +warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + +Written by Yaroslav Halchenko for the NeuroDebian project. + +EOT +} + +eval_dry() +{ + if [ -z "$ae_dry_run" ]; then + if eval "$ae_sudo $@" 1>|"$ae_tempdir/eval.log" 2>&1; then + rm "$ae_tempdir/eval.log" + else + error $? "Command $@ failed with exit code $?. Output was: `cat $ae_tempdir/eval.log`" + fi + else + echo "DRY: $@" >&2 + fi +} + +print_help() +{ + cat << EOT + +Usage: nd-configurerepo [options] + +Enables NeuroDebian repository for the current Debian or Ubuntu release. + +Options: + + -r, --release=RELEASE + Name of the Debian/Ubuntu release to be used. If not specified, + it is deduced from the apt-cache policy output, by taking repository + of Debian or Ubuntu origin with highest priority. + + --print-releases + Return a list of releases present in NeuroDebian repository. + + -f, --flavor=full|libre + Which flavor of the repository should be enabled: + libre -- Only main component, containing only DFSG-compliant content. + full -- Includes main, contrib, and non-free. + If not specified -- deduced from the output of apt-cache policy + + -c, --components=c1,c2,c3 + Comma separated list of components to enable among: + software -- primary software repository + data -- data packages + devel -- "overlay" of development versions (like Debians' "experimental"). + Not sufficient on its own and available only from the main site + If not specified -- "software,data" + + -m, --mirror=NAME|URL + Which mirror to use. Could be a mirror code-name (as specified in + /etc/neurodebian/neurodebian.cfg), or a URL. + + --print-mirrors + Return a list (with abbreviation) of known NeuroDebian mirrors. + + --overwrite, + If apt file already present, it would not be overriden (by default). + Use this option to overwrite. + + --suffix=SUFFIX + Which suffix to add to the apt file, in case you are trying to enable + multiple repositories + + --sources, --no-sources + Either to enable deb-src lines. If none specified -- would be enabled if + sources for a core package (apt) are available. + + -n, --dry-run + Do not perform any changes -- generated configurations and commands will + simply be printed to stdout + + --install + If found absent, all necessary tools (wget, netselect) if available will + be apt-get installed + + -v, --verbose + Enable additional progress messages. Could be used multiple times + + -q, --quiet + Make operation quiet -- only error messages would be output + + -h, --help + Print short description, usage summary and option list. + + --version + Print version information and exit. + +Exit status: + + non-0 exit status in case of error. Error exit code would depend + on which command has failed + +Examples: + + - Enable software and data components from the optimal (according to + netselect) mirror. Some information about progress will be printed + + nd-configurerepo + + - Quietly enable -devel repository for the current release, and place apt + configuration into /etc/apt/sources.list.d/neurodebian.sources-devel.list + + nd-configurerepo -q --suffix=-devel -c devel + + - Force sid distribution, all the components, from the Japan mirror: + + nd-configurerepo -q --suffix=-de-sid-full -c software,data,devel -m jp + +EOT +} + +get_neurodebian_cfg() +{ + if [ -s "$nd_config_file_fresh" ]; then + print_verbose 3 "Config file $nd_config_file_fresh exists -- not fetching" + echo "$nd_config_file_fresh" + return 0 + fi + # First we try to fetch the most recent version from the github + print_verbose 3 "Fetching config file from the github repository" + assure_command_from_package wget wget 1 + wget --no-check-certificate -c -q -O$nd_config_file_fresh \ + $nd_config_url \ + && { echo "$nd_config_file_fresh"; } \ + || { [ -e "$nd_config_file" ] \ + && echo "$nd_config_file" \ + || error 10 "Neither could fetch $nd_config_url, nor found $nd_config_file"; } +} + +query_cfg_section() +{ + config_file="$1" + section="$2" + print_verbose 3 "Querying config $config_file section $section" + assure_command_from_package python python-minimal 1 + python -c "from ConfigParser import SafeConfigParser as SP; cfg = SP(); cfg.read('$config_file'); print('\n'.join([' '.join(x) for x in cfg.items('$section')]))" +} + +get_mirrors() +{ + nd_config=`get_neurodebian_cfg` +# $exe_dir/nd_querycfg -F" " --config-file="$nd_config" "mirrors" \ + n="" + query_cfg_section "$nd_config" "mirrors" \ + | while read mirror_name mirror_url; do + # verify that url is just a url + if echo "$mirror_url" | grep -v -e '^[a-z0-9:+]*://[-+_%.a-z0-9/]*$'; then + print_verbose 1 "Mirror $mirror_name has 'illegit' URL: $mirror_url. Skipping" + fi + [ -z "$n" ] || echo -ne "${ND_IFS:-\n}"; n+=1 + echo -n "$mirror_name $mirror_url" + done +} + +get_releases() +{ + nd_config=`get_neurodebian_cfg` + n="" + query_cfg_section "$nd_config" "release files" \ + | while read release_name release_url; do + # verify that url is just a url + if [ "$release_name" = "data" ]; then + # skip data + continue + fi + [ -z "$n" ] || echo -ne "${ND_IFS:-\n}"; n+=1 + echo -n "$release_name" + done +} + +get_package_version() +{ + pkg_version=$(apt-cache policy "$1" | awk '/^ *Installed:/{print $2;}') + [ "$pkg_version" != '(none)' ] || pkg_version='' + echo "$pkg_version" +} + +netselect_mirror() { + # select "closest" mirror according to netselect. + print_verbose 2 "Selecting the 'best' mirror using netselect" + assure_command_from_package netselect + if ! which netselect >&/dev/null; then + print_verbose 1 "netselect (apt-get install netselect) needed to select the 'best' mirror was not found" + print_verbose 1 "Selecting the default repository: $nd_mirror_default" + echo $nd_mirror_default + else + # squeeze version doesn't have -D yet to force output of the URL not IP, but for our mirrors ATM it shouldn't matter + netselect_opts="-s 1" + netselect_version="$(get_package_version netselect)" + if dpkg --compare-versions "$netselect_version" ge 0.3.ds1-17; then + netselect_opts+=" -D" + fi + if dpkg --compare-versions "$netselect_version" ge 0.3.ds1-15; then + netselect_opts+=" -I" + fi + best_mirror=$(get_mirrors | awk '{print $2;}' | eval $ae_sudo xargs netselect $netselect_opts | awk '{print $2;}') + if [ -z "$best_mirror" ]; then + print_verbose 1 "Failed to select mirror using netselect. Selecting default one ($nd_mirror_default)" + echo "$nd_mirror_default" + else + print_verbose 2 "Best mirror: $best_mirror" + echo $best_mirror + fi + fi +} + +get_mirror_url() +{ + # given mirror alias -- find its url + url=$(get_mirrors | awk "/^$1 /{print \$2;}") + if [ -z "$url" ]; then + error 9 "Cannot resolve mirror $1 to the URL" + fi + echo $url +} + +get_apt_policy() +{ + # Get apt-cache policy output in a single list for matching suites + # (could be a separated with \| or , for multiple choices, e.g. + # + # get_apt_policy Debian,Ubuntu + # or + # get_apt_policy NeuroDebian + suites="$1" + $ae_sudo apt-cache policy | grep -B1 -e "o=\(${suites//,/\\|}\)" | tr '\n' ' ' | sed -e 's, -- ,\n,g' | grep -v -e '-\(updates\|security\)' | sort -nr +} + +is_component_included() +{ + echo "$ae_components" | tr ',' '\n' | grep -q "^$1\$" +} + +is_sources_enabled() +{ + apt-cache showsrc apt >&/dev/null && echo 1 || echo 0 +} + +assure_command_from_package() +{ + cmd=$1 + pkg=${2:-$cmd} + fail=${3:-} + + which "$cmd" >&/dev/null && return 0 + + # if absent -- check availability of the package + apt_cache=$(LANG=C apt-cache policy "$pkg" 2>&1) + if [[ "$apt_cache" =~ Unable\ to\ locate\ package ]] || [[ "$apt_cache" =~ Candidate:\ (none) ]]; then + print_verbose 1 "Package $pkg providing command $cmd is N/A. Skipping" + return 10; + fi + if echo "$apt_cache" | grep -q '^\s*\*\*\*'; then + print_verbose 1 "WARNING -- command $cmd is N/A but package $pkg is claimed to be installed" + [ -z "$fail" ] && return 11 || error $fail "Command $cmd is required to proceed" + fi + if [ "$ae_install" = "1" ]; then + print_verbose 1 "Installing $pkg package to get $cmd command" + eval_dry DEBIAN_FRONTEND=noninteractive apt-get install -y "$pkg" + return + else + print_verbose 1 "Command $cmd (from package $pkg) is N/A." + print_verbose 1 "Use with --install to get all necessary packages installed automatically" + [ -z "$fail" ] && return 12 || error $fail "Command $cmd is required to proceed" + fi +} + +# if it was requested -- return without doing anything +[ -z "$ae_defun_only" ] || return + +# +# Commandline options handling +# + +# Parse commandline options (taken from the getopt examples from the Debian util-linux package) +# Note that we use `"$@"' to let each command-line parameter expand to a +# separate word. The quotes around `$@' are essential! +# We need CLOPTS as the `eval set --' would nuke the return value of getopt. +CLOPTS=`getopt -o h,r:,m:,f:,c:,q,v,n --long help,version,quiet,verbose,mirror:,release:,flavor:,components:,suffix:,overwrite,sources,no-sources,install,dry-run,do-not-update,print-releases,print-mirrors,print-best-mirror -n 'nd-configurerepo' -- "$@"` + +if [ $? != 0 ] ; then + error 2 "Problem with parsing cmdline. Terminating..." +fi + +# Note the quotes around `$CLOPTS': they are essential! +eval set -- "$CLOPTS" + +if [ `whoami` != "root" ]; then + ae_sudo=sudo +fi + +while true ; do + case "$1" in + -r|--release) shift; ae_release="$1"; shift;; + -f|--flavor) shift; ae_flavor="$1"; shift;; + -c|--components) shift; ae_components="$1"; shift;; + -m|--mirror) shift; ae_mirror="$1"; shift;; + --print-mirrors) get_mirrors; exit 0;; + --print-best-mirror) netselect_mirror; exit 0;; + --print-releases) get_releases; exit 0;; + -n|--dry-run) ae_dry_run=1; shift;; + --suffix) shift; ae_suffix="$1"; shift;; + --overwrite) ae_overwrite="$1"; shift;; + --do-not-update) ae_update=""; shift;; + --sources) ae_sources=1; shift;; + --no-sources) ae_sources=0; shift;; + --install) ae_install=1; shift;; + -q|--quiet) ae_verbose=0; shift;; + -v|--verbose) ae_verbose=$(($ae_verbose+1)); shift;; + -h|--help) print_help; exit 0;; + --version) print_version; exit 0;; + --) shift ; break ;; + *) error 1 "Internal error! ($1)";; + esac +done + + +if [ $# -gt 0 ] ; then + print_help >&2 + exit 2 +fi + +# Inform! +[ -z "$ae_sudo" ] || print_verbose 1 "This script requires root access. Since current user is not root, sudo will be used" + +# +# Basic system/environment knowledge +# + +ae_output_file=/etc/apt/sources.list.d/neurodebian.sources${ae_suffix}.list + +apt_policy=$(get_apt_policy "Debian,Ubuntu" ) + +if [ -z "$ae_release" ]; then + ae_release=$(echo "$apt_policy" | head -1 | sed -e 's/.*,n=\([^,]*\),.*/\1/g') +fi + +if [ -z "$ae_flavor" ]; then + ae_flavor=$(echo "$apt_policy" | grep -e ",n=$ae_release," | grep -qe 'c=\(non-free\|multiverse\)' && echo "full" || echo "libre") +fi + +# +# Determine which mirror to use +# + +# knowing mirror is not necessary for -devel available only from the main site +if is_component_included software || is_component_included data; then + # for now just use default + if [ -z "$ae_mirror" ]; then # none specified + ae_mirror_url=$nd_mirror_origin + else + if ! [[ "$ae_mirror" =~ .*://.* ]]; then + case "$ae_mirror" in + best) ae_mirror_url=$(netselect_mirror);; + default) ae_mirror_url=$nd_mirror_default;; + origin) ae_mirror_url=$nd_mirror_origin;; + *) ae_mirror_url=$(get_mirror_url "$ae_mirror");; + esac + else + ae_mirror_url="$ae_mirror" # it was some kind of a URL already + fi + fi +fi + +# +# Prepare APT file +# + +case $ae_flavor in + full) apt_flavor="contrib non-free";; + libre) apt_flavor="";; + *) error 5 "Unknown value of flavor $apt_flavor. Must be full or libre" +esac + +if [ -z "$ae_sources" ]; then + ae_sources=$(is_sources_enabled) +fi + +if [ $ae_sources -eq 0 ]; then + sources_comment="#" +else + sources_comment="" +fi + +apt_list= + +if is_component_included software; then + apt_list+=" +# NeuroDebian software repository +deb $ae_mirror_url $ae_release main $apt_flavor +${sources_comment}deb-src $ae_mirror_url $ae_release main $apt_flavor +" +fi + +if is_component_included data; then + apt_list+=" +# NeuroDebian data repository +deb $ae_mirror_url data main $apt_flavor +${sources_comment}deb-src $ae_mirror_url data main $apt_flavor +" +fi + +if is_component_included devel; then + apt_list+=" +# NeuroDebian -devel repository +deb http://neuro.debian.net/debian-devel $ae_release main $apt_flavor +${sources_comment}deb-src http://neuro.debian.net/debian-devel $ae_release main $apt_flavor +" +fi + +if [ -e "$ae_output_file" ] && [ -z "$ae_overwrite" ]; then + if diff "$ae_output_file" <(echo "$apt_list") | grep -q .; then + # error 3 + print_verbose 1 "File $ae_output_file already exists, containing:\n`cat \"$ae_output_file\"`\n\nI: New configuration is different:\n$apt_list" + if get_apt_policy NeuroDebian >/dev/null; then + print_verbose 1 "NeuroDebian repositories are already available, thus skipping the rest." + print_verbose 1 "Rerun with --overwrite if you would like to reconfigure." + exit 0 + else + print_verbose 1 "NeuroDebian configuration is found but not yet available -- continuing with new configuration." + fi + else + print_verbose 1 "New configuration is identical to existing and NeuroDebian repository is already enabled." + print_verbose 1 "Skiping the rest. Rerun with --overwrite if you would like to reconfigure." + exit 0 + fi +else + print_verbose 1 "Generating $ae_output_file" + if [ -z "$ae_dry_run" ]; then + echo "$apt_list" | $ae_sudo bash -c "cat - >| '$ae_output_file'" + else + echo "DRY:" + echo "$apt_list" + fi +fi + + +# +# Assure present archive GPG key for APT system +# + +# Figure out if key needs to be imported (if ran within package, +# should already be there due to neurodebian-keyring package) +if LANG=C eval $ae_sudo apt-key export $nd_key_id 2>&1 1>/dev/null | grep -qe "nothing exported"; then + print_verbose 1 "Fetching the key from the server" + eval_dry apt-key adv --recv-keys --keyserver pgp.mit.edu $nd_key_id +fi + + +# +# Finalizing (apt-get update etc) +# + +if [ ! -z "$ae_update" ]; then + print_verbose 1 "Updating APT listings, might take a few minutes" + if [ -z "$ae_dry_run" ]; then + apt_logfile="$ae_tempdir/apt.log" + $ae_sudo apt-get update 1>"$apt_logfile" 2>&1 \ + && rm -f "$apt_logfile" \ + || { + apt_log=$(cat "$apt_logfile") + echo "$apt_log" + if echo "$apt_log" | grep -q "Malformed line [0-9]* in source list $ae_output_file"; then + $ae_sudo mv "${ae_output_file}" "${ae_output_file}-failed.disabled" + error 6 "Update failed to possible errorneous APT listing file produced by this script. Generated $ae_output_file renamed to ${ae_output_file}-failed.disabled to not interfer" + fi + error 5 "Update failed with exit code $? (above output logged into $apt_logfile)." + } + else + eval_dry apt-get update + fi +else + print_verbose 1 "apt-get update was not run. Please run to take an effect of changes" +fi + +if [ "$ae_verbose" -ge 2 ]; then + print_verbose 2 "Currently enabled NeuroDebian suites/mirrors:" + get_apt_policy NeuroDebian +fi