X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=tools%2Fnd-aptenable;h=0bf694836dcb72a3d65e4e214948d5afb99b70d0;hb=155e0b4c8045a03ce3ecfc7d797667018114b323;hp=492a26479f66e89532e3cae9415cc51d69d0bed5;hpb=36d2663c16bc62ee3b7c5f69d3fb8ba1d388a8df;p=neurodebian.git diff --git a/tools/nd-aptenable b/tools/nd-aptenable index 492a264..0bf6948 100755 --- a/tools/nd-aptenable +++ b/tools/nd-aptenable @@ -2,6 +2,9 @@ #emacs: -*- mode: shell-script; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- #ex: set sts=4 ts=4 sw=4 et: +# Depends: apt (assumed to be present), python, wget +# Recommends: netselect + # play safe set -e set -u @@ -13,38 +16,43 @@ set -u nd_aptenable_version=0.1 nd_key_id=0x2649A5A9 - -# To be set by cmdline args -ae_release= -ae_components=software,data -ae_flavor= -ae_mirror_main=http://neuro.debian.net/debian -ae_mirror= -ae_suffix= -ae_verbose=1 -ae_overwrite= -ae_sources= -ae_dry_run= +nd_config_url=https://raw.githubusercontent.com/neurodebian/neurodebian/master/neurodebian.cfg +nd_config_file=/etc/neurodebian/neurodebian.cfg +nd_mirror_origin=http://neuro.debian.net/debian +nd_mirror_default=$nd_mirror_origin # or may be AWS? + +# To be set by cmdline args or via env variables with prefix ND_AE_ +ae_release=${ND_AE_RELEASE:-} +ae_components=${ND_AE_COMPONENTS:-software,data} +ae_flavor=${ND_AE_FLAVOR:-} +ae_mirror=${ND_AE_MIRROR:-best} +ae_suffix=${ND_AE_SUFFIX:-} +ae_verbose=${ND_AE_VERBOSE:-1} +ae_overwrite=${ND_AE_OVERWRITE:-} +ae_sources=${ND_AE_SOURCES:-} +ae_install=${ND_AE_INSTALL:-} +ae_dry_run=${ND_AE_DRY_RUN:-} +ae_defun_only=${ND_AE_DEFUN_ONLY:-} # mode to source this file as a "library" ae_sudo= +exe_dir=$(dirname $0) # TODOs: -# - distribute/enable key # - apt priority! (so we could avoid automagic upgrades etc) # - multiarch setups +ae_tempdir=$(mktemp -d) +trap "rm -rf \"$ae_tempdir\"" TERM INT EXIT -# TODO -- consider fetching the most recent version from online -# concerns -- without any validation might be subject to injection through man-in-the-middle etc -nd_configfile=/etc/neurodebian/neurodebian.cfg # TODO - comes from neurodebian-devel pkg... -# TODO: Or should it be parsed out from the NeuroDebian website itself? -# may be we could provide that file from the neurodebian website... print_verbose() { level=$1; shift if [ "$ae_verbose" -ge $level ]; then - echo -e "I: $*" + # use stderr for printing within functions stdout of which might be used + echo -n "I: " >&2 + i=1; while [ $i -lt $level ]; do echo -ne " ">&2; i=$(($i+1)); done + echo -e "$*" >&2 fi } @@ -57,7 +65,7 @@ error() print_version() { -cat << EOT + cat << EOT nd-aptenable $nd_aptenable_version Copyright (C) 2014 Yaroslav Halchenko @@ -74,15 +82,19 @@ EOT eval_dry() { if [ -z "$ae_dry_run" ]; then - eval "$ae_sudo $@" + if eval "$ae_sudo $@" 1>|"$ae_tempdir/eval.log" 2>&1; then + rm "$ae_tempdir/eval.log" + else + error $? "Command $@ failed with exit code $?. Output was: `cat $ae_tempdir/eval.log`" + fi else - echo "DRY: $@" + echo "DRY: $@" >&2 fi } print_help() { -cat << EOT + cat << EOT Usage: nd-aptenable [options] @@ -105,13 +117,13 @@ Options: Comma separated list of components to enable among: software -- primary software repository data -- data packages - devel -- "overlay" of development versions (like Debian "experimental"). + devel -- "overlay" of development versions (like Debians' "experimental"). Not sufficient on its own and available only from the main site If not specified -- "software,data" -m, --mirror=NAME|URL Which mirror to use. Could be a mirror code-name (as specified in - /etc/neurodebian/neurodebian.cfg), or a URL (TODO). + /etc/neurodebian/neurodebian.cfg), or a URL. --overwrite, If apt file already present, it would not be overriden (by default). @@ -122,12 +134,17 @@ Options: multiple repositories --sources, --no-sources - Either to enable deb-src lines. If not specified -- deduced based on ??? TODO + Either to enable deb-src lines. If none specified -- would be enabled if + sources for a core package (apt) are available. -n, --dry-run Do not perform any changes -- generated configurations and commands will simply be printed to stdout + --install + If found absent, all necessary tools (wget, netselect) if available will + be apt-get installed + -v, --verbose Enable additional progress messages. Could be used multiple times @@ -164,17 +181,86 @@ Examples: EOT } -get_mirrors() { - # Determine available mirrors by fetching our .cfg file - /etc/neurodebian/neurodebian.cfg +get_neurodebian_cfg() +{ + # First we try to fetch the most recent version from the github + print_verbose 3 "Fetching config file from the github repository" + cfgfile_temp="$ae_tempdir/neurodebian.cfg" + assure_command_from_package wget wget 1 + wget --no-check-certificate -c -q -O$cfgfile_temp \ + $nd_config_url \ + && { echo $cfgfile_temp; } \ + || { [ -e "$nd_config_file" ] \ + && echo "$nd_config_file" \ + || error 10 "Neither could fetch $nd_config_url, nor found $nd_config_file"; } +} + +query_cfg_section() +{ + config_file="$1" + section="$2" + print_verbose 3 "Querying config $config_file section $section" + assure_command_from_package python python-minimal 1 + python -c "from ConfigParser import SafeConfigParser as SP; cfg = SP(); cfg.read('$config_file'); print('\n'.join([' '.join(x) for x in cfg.items('$section')]))" +} + +get_mirrors() +{ + nd_config=`get_neurodebian_cfg` +# $exe_dir/nd_querycfg -F" " --config-file="$nd_config" "mirrors" \ + query_cfg_section "$nd_config" "mirrors" \ + | while read mirror_name mirror_url; do + # verify that url is just a url + if echo "$mirror_url" | grep -v -e '^[a-z0-9:+]*://[-+_%.a-z0-9/]*$'; then + print_verbose 1 "Mirror $mirror_name has 'illegit' URL: $mirror_url. Skipping" + fi + echo "$mirror_name $mirror_url" + done +} + +get_package_version() +{ + pkg_version=$(apt-cache policy "$1" | awk '/^ *Installed:/{print $2;}') + [ "$pkg_version" != '(none)' ] || pkg_version='' + echo "$pkg_version" } -print_mirrors() { - echo "TODO" - exit 1 +netselect_mirror() { + # select "closest" mirror according to netselect. + print_verbose 2 "Selecting the 'best' mirror using netselect" + assure_command_from_package netselect + if ! which netselect >&/dev/null; then + print_verbose 1 "netselect (apt-get install netselect) needed to select the 'best' mirror was not found" + print_verbose 1 "Selecting the default repository: $nd_mirror_default" + echo $nd_mirror_default + else + # squeeze version doesn't have -D yet to force output of the URL not IP, but for our mirrors ATM it shouldn't matter + netselect_opts="-s 1" + netselect_version="$(get_package_version netselect)" + if dpkg --compare-versions "$netselect_version" ge 0.3.ds1-17; then + netselect_opts+=" -D" + fi + if dpkg --compare-versions "$netselect_version" ge 0.3.ds1-15; then + netselect_opts+=" -I" + fi + best_mirror=$(get_mirrors | awk '{print $2;}' | eval $ae_sudo xargs netselect $netselect_opts | awk '{print $2;}') + print_verbose 2 "Best mirror: $best_mirror" + echo $best_mirror + fi } -get_apt_policy() { +get_mirror_url() +{ + # given mirror alias -- find its url + url=$(get_mirrors | awk "/^$1 /{print \$2;}") + if [ -z "$url" ]; then + error 9 "Cannot resolve mirror $1 to the URL" + fi + echo $url +} + +get_apt_policy() +{ # Get apt-cache policy output in a single list for matching suites # (could be a separated with \| or , for multiple choices, e.g. # @@ -182,13 +268,50 @@ get_apt_policy() { # or # get_apt_policy NeuroDebian suites="$1" - apt-cache policy | grep -B1 -e "o=\(${suites//,/\\|}\)" | tr '\n' ' ' | sed -e 's, -- ,\n,g' | grep -v -e '-\(updates\|security\)' | sort -nr + $ae_sudo apt-cache policy | grep -B1 -e "o=\(${suites//,/\\|}\)" | tr '\n' ' ' | sed -e 's, -- ,\n,g' | grep -v -e '-\(updates\|security\)' | sort -nr } -include_component() { +is_component_included() +{ echo "$ae_components" | tr ',' '\n' | grep -q "^$1\$" } +is_sources_enabled() +{ + apt-cache showsrc apt >&/dev/null && echo 1 || echo 0 +} + +assure_command_from_package() +{ + cmd=$1 + pkg=${2:-$cmd} + fail=${3:-} + + which "$cmd" >&/dev/null && return 0 + + # if absent -- check availability of the package + apt_cache=$(LANG=C apt-cache policy "$pkg" 2>&1) + if [[ "$apt_cache" =~ "Unable to locate package" ]] || [[ "$apt_cache" =~ "Candidate: (none)" ]]; then + print_verbose 1 "Package $pkg providing command $cmd is N/A. Skipping" + return 10; + fi + if echo "$apt_cache" | grep -q '^\s*\*\*\*'; then + print_verbose 1 "WARNING -- command $cmd is N/A but package $pkg is claimed to be installed" + [ -z "$fail" ] && return 11 || error $fail "Command $cmd is required to proceed" + fi + if [ "$ae_install" = "1" ]; then + print_verbose 1 "Installing $pkg package to get $cmd command" + eval_dry DEBIAN_FRONTEND=noninteractive apt-get install -y "$pkg" + return + else + print_verbose 1 "Command $cmd (from package $pkg) is N/A." + print_verbose 1 "Use with --install to get all necessary packages installed automatically" + [ -z "$fail" ] && return 12 || error $fail "Command $cmd is required to proceed" + fi +} + +# if it was requested -- return without doing anything +[ -z "$ae_defun_only" ] || return # # Commandline options handling @@ -198,7 +321,7 @@ include_component() { # Note that we use `"$@"' to let each command-line parameter expand to a # separate word. The quotes around `$@' are essential! # We need CLOPTS as the `eval set --' would nuke the return value of getopt. -CLOPTS=`getopt -o h,r:,m:,f:,c:,q,v,n --long help,version,quiet,verbose,mirror:,release:,flavor:,components:,suffix:,overwrite,sources,no-sources,dry-run,print-mirrors -n 'nd-aptenable' -- "$@"` +CLOPTS=`getopt -o h,r:,m:,f:,c:,q,v,n --long help,version,quiet,verbose,mirror:,release:,flavor:,components:,suffix:,overwrite,sources,no-sources,install,dry-run,print-mirrors,print-best-mirror -n 'nd-aptenable' -- "$@"` if [ $? != 0 ] ; then error 2 "Problem with parsing cmdline. Terminating..." @@ -207,20 +330,24 @@ fi # Note the quotes around `$CLOPTS': they are essential! eval set -- "$CLOPTS" +if [ `whoami` != "root" ]; then + ae_sudo=sudo +fi + while true ; do case "$1" in -r|--release) shift; ae_release="$1"; shift;; -f|--flavor) shift; ae_flavor="$1"; shift;; -c|--components) shift; ae_components="$1"; shift;; - # TODO -m|--mirror) shift; ae_mirror="$1"; shift;; - # TODO - --print-mirrors) print_mirrors; exit 0;; + --print-mirrors) get_mirrors; exit 0;; + --print-best-mirror) netselect_mirror; exit 0;; -n|--dry-run) ae_dry_run=1; shift;; --suffix) shift; ae_suffix="$1"; shift;; --overwrite) ae_overwrite="$1"; shift;; --sources) ae_sources=1; shift;; --no-sources) ae_sources=0; shift;; + --install) ae_install=1; shift;; -q|--quiet) ae_verbose=0; shift;; -v|--verbose) ae_verbose=$(($ae_verbose+1)); shift;; -h|--help) print_help; exit 0;; @@ -236,6 +363,8 @@ if [ $# -gt 0 ] ; then exit 2 fi +# Inform! +[ -z "$ae_sudo" ] || print_verbose 1 "This script requires root access. Since current user is not root, sudo will be used" # # Basic system/environment knowledge @@ -243,11 +372,6 @@ fi ae_output_file=/etc/apt/sources.list.d/neurodebian.sources${ae_suffix}.list -if [ `whoami` != "root" ]; then - print_verbose 1 "This script requires root access. Since current user is not root, sudo will be used" - ae_sudo=sudo -fi - apt_policy=$(get_apt_policy "Debian,Ubuntu" ) if [ -z "$ae_release" ]; then @@ -262,20 +386,25 @@ fi # Determine which mirror to use # -# TODO -- determine mirror URL -# Not necessary for -devel available only from the main site -if include_component software || include_component data; then +# knowing mirror is not necessary for -devel available only from the main site +if is_component_included software || is_component_included data; then # for now just use default if [ -z "$ae_mirror" ]; then # none specified - ae_mirror_url=$ae_mirror_main + ae_mirror_url=$nd_mirror_origin else - if [ ! "$ae_mirror" ~= ".*://.*" ]; then - # TODO -- determine from the abbreviation + if ! [[ "$ae_mirror" =~ ".*://.*" ]]; then + case "$ae_mirror" in + best) ae_mirror_url=$(netselect_mirror);; + default) ae_mirror_url=$nd_mirror_default;; + origin) ae_mirror_url=$nd_mirror_origin;; + *) ae_mirror_url=$(get_mirror_url "$ae_mirror");; + esac + else + ae_mirror_url="$ae_mirror" # it was some kind of a URL already fi fi fi - # # Prepare APT file # @@ -286,7 +415,11 @@ case $ae_flavor in *) error 5 "Unknown value of flavor $apt_flavor. Must be full or libre" esac -if [ -z "$ae_sources" ] || [ $ae_sources -eq 0 ]; then +if [ -z "$ae_sources" ]; then + ae_sources=$(is_sources_enabled) +fi + +if [ $ae_sources -eq 0 ]; then sources_comment="#" else sources_comment="" @@ -294,7 +427,7 @@ fi apt_list= -if include_component software; then +if is_component_included software; then apt_list+=" # NeuroDebian software repository deb $ae_mirror_url $ae_release main $apt_flavor @@ -302,7 +435,7 @@ ${sources_comment}deb-src $ae_mirror_url $ae_release main $apt_flavor " fi -if include_component data; then +if is_component_included data; then apt_list+=" # NeuroDebian data repository deb $ae_mirror_url data main $apt_flavor @@ -310,7 +443,7 @@ ${sources_comment}deb-src $ae_mirror_url data main $apt_flavor " fi -if include_component devel; then +if is_component_included devel; then apt_list+=" # NeuroDebian -devel repository deb http://neuro.debian.net/debian-devel $ae_release main $apt_flavor @@ -319,12 +452,25 @@ ${sources_comment}deb-src http://neuro.debian.net/debian-devel $ae_release main fi if [ -e "$ae_output_file" ] && [ -z "$ae_overwrite" ]; then - # error 3 - print_verbose 1 "File $ae_output_file already exists, containing:\n\n`cat \"$ae_output_file\"`\n\nI: Use --overwrite option to regenerate with:\n\n$apt_list" + if diff "$ae_output_file" <(echo "$apt_list") | grep -q .; then + # error 3 + print_verbose 1 "File $ae_output_file already exists, containing:\n`cat \"$ae_output_file\"`\n\nI: New configuration is different:\n$apt_list" + if get_apt_policy NeuroDebian >/dev/null; then + print_verbose 1 "NeuroDebian repositories are already available, thus skipping the rest." + print_verbose 1 "Rerun with --overwrite if you would like to reconfigure." + exit 0 + else + print_verbose 1 "NeuroDebian configuration is found but not yet available -- continuing with new configuration." + fi + else + print_verbose 1 "New configuration is identical to existing and NeuroDebian repository is already enabled." + print_verbose 1 "Skiping the rest. Rerun with --overwrite if you would like to reconfigure." + exit 0 + fi else print_verbose 1 "Generating $ae_output_file" if [ -z "$ae_dry_run" ]; then - echo "$apt_list" >| "$ae_output_file" + echo "$apt_list" | $ae_sudo bash -c "cat - >| '$ae_output_file'" else echo "DRY:" echo "$apt_list" @@ -343,18 +489,24 @@ if LANG=C eval $ae_sudo apt-key export $nd_key_id 2>&1 1>/dev/null | grep -qe "n eval_dry apt-key adv --recv-keys --keyserver pgp.mit.edu $nd_key_id fi + # # Finalizing (apt-get update etc) # print_verbose 1 "Updating APT listings, might take a few minutes" if [ -z "$ae_dry_run" ]; then - apt_logfile=$(mktemp) + apt_logfile="$ae_tempdir/apt.log" $ae_sudo apt-get update 1>"$apt_logfile" 2>&1 \ && rm -f "$apt_logfile" \ || { - cat "$apt_logfile" - error 5 "E: Update failed with exit code $? (above output logged into $apt_logfile)." + apt_log=$(cat "$apt_logfile") + echo "$apt_log" + if echo "$apt_log" | grep -q "Malformed line [0-9]* in source list $ae_output_file"; then + $ae_sudo mv "${ae_output_file}" "${ae_output_file}-failed.disabled" + error 6 "Update failed to possible errorneous APT listing file produced by this script. Generated $ae_output_file renamed to ${ae_output_file}-failed.disabled to not interfer" + fi + error 5 "Update failed with exit code $? (above output logged into $apt_logfile)." } else eval_dry apt-get update