2 #emacs: -*- mode: shell-script; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
3 #ex: set sts=4 ts=4 sw=4 et:
5 # Depends: apt (assumed to be present), python, wget
6 # Recommends: netselect
16 nd_aptenable_version=0.1
19 nd_config_url=https://raw.githubusercontent.com/neurodebian/neurodebian/master/neurodebian.cfg
20 nd_config_file=/etc/neurodebian/neurodebian.cfg
21 nd_mirror_origin=http://neuro.debian.net/debian
22 nd_mirror_default=$nd_mirror_origin # or may be AWS?
24 # To be set by cmdline args or via env variables with prefix ND_AE_
25 ae_release=${ND_AE_RELEASE:-}
26 ae_components=${ND_AE_COMPONENTS:-software,data}
27 ae_flavor=${ND_AE_FLAVOR:-}
28 ae_mirror=${ND_AE_MIRROR:-best}
29 ae_suffix=${ND_AE_SUFFIX:-}
30 ae_verbose=${ND_AE_VERBOSE:-1}
31 ae_overwrite=${ND_AE_OVERWRITE:-}
32 ae_sources=${ND_AE_SOURCES:-}
33 ae_install=${ND_AE_INSTALL:-}
34 ae_update=${ND_AE_UPDATE:-1}
35 ae_dry_run=${ND_AE_DRY_RUN:-}
36 ae_defun_only=${ND_AE_DEFUN_ONLY:-} # mode to source this file as a "library"
44 # - apt priority! (so we could avoid automagic upgrades etc)
47 if [ -z "${ND_AE_TEMPDIR:-}" ]; then
48 ae_tempdir=$(mktemp -d)
49 trap "rm -rf \"$ae_tempdir\"" TERM INT EXIT
51 # reuse the same directory/fetched configuration if was specified
52 ae_tempdir="${ND_AE_TEMPDIR:-}"
56 nd_config_file_fresh="$ae_tempdir/neurodebian.cfg"
61 if [ "$ae_verbose" -ge $level ]; then
62 # use stderr for printing within functions stdout of which might be used
64 i=1; while [ $i -lt $level ]; do echo -ne " ">&2; i=$(($i+1)); done
79 nd-configurerepo $nd_aptenable_version
81 Copyright (C) 2014 Yaroslav Halchenko <debian@onerussian.com>
83 Licensed under GNU Public License version 3 or later.
84 This is free software; see the source for copying conditions. There is NO
85 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
87 Written by Yaroslav Halchenko for the NeuroDebian project.
94 if [ -z "$ae_dry_run" ]; then
95 if eval "$ae_sudo $@" 1>|"$ae_tempdir/eval.log" 2>&1; then
96 rm "$ae_tempdir/eval.log"
98 error $? "Command $@ failed with exit code $?. Output was: `cat $ae_tempdir/eval.log`"
109 Usage: nd-configurerepo [options]
111 Enables NeuroDebian repository for the current Debian or Ubuntu release.
115 -r, --release=RELEASE
116 Name of the Debian/Ubuntu release to be used. If not specified,
117 it is deduced from the apt-cache policy output, by taking repository
118 of Debian or Ubuntu origin with highest priority.
121 Print a list of releases present in NeuroDebian repository.
124 Print the release deduced from the output of apt-cache policy.
126 -f, --flavor=full|libre
127 Which flavor of the repository should be enabled:
128 libre -- Only main component, containing only DFSG-compliant content.
129 full -- Includes main, contrib, and non-free.
130 If not specified -- deduced from the output of apt-cache policy.
133 Print the flavor deduced from the output of apt-cache policy.
135 -c, --components=c1,c2,c3
136 Comma separated list of components to enable among:
137 software -- primary software repository
138 data -- data packages
139 devel -- "overlay" of development versions (like Debians' "experimental").
140 Not sufficient on its own and available only from the main site
141 If not specified -- "software,data"
143 -m, --mirror=NAME|URL
144 Which mirror to use. Could be a mirror code-name (as specified in
145 /etc/neurodebian/neurodebian.cfg), or a URL.
148 Return a list (with abbreviation) of known NeuroDebian mirrors.
151 If apt file already present, it would not be overriden (by default).
152 Use this option to overwrite.
155 Which suffix to add to the apt file, in case you are trying to enable
156 multiple repositories
158 --sources, --no-sources
159 Either to enable deb-src lines. If none specified -- would be enabled if
160 sources for a core package (apt) are available.
163 Do not perform any changes -- generated configurations and commands will
164 simply be printed to stdout
167 If found absent, all necessary tools (wget, netselect) if available will
171 Enable additional progress messages. Could be used multiple times
174 Make operation quiet -- only error messages would be output
177 Print short description, usage summary and option list.
180 Print version information and exit.
184 non-0 exit status in case of error. Error exit code would depend
185 on which command has failed
189 - Enable software and data components from the optimal (according to
190 netselect) mirror. Some information about progress will be printed
194 - Quietly enable -devel repository for the current release, and place apt
195 configuration into /etc/apt/sources.list.d/neurodebian.sources-devel.list
197 nd-configurerepo -q --suffix=-devel -c devel
199 - Force sid distribution, all the components, from the Japan mirror:
201 nd-configurerepo -q --suffix=-de-sid-full -c software,data,devel -m jp
206 get_neurodebian_cfg()
208 if [ -s "$nd_config_file_fresh" ]; then
209 print_verbose 3 "Config file $nd_config_file_fresh exists -- not fetching"
210 echo "$nd_config_file_fresh"
213 # First we try to fetch the most recent version from the github
214 print_verbose 3 "Fetching config file from the github repository"
215 assure_command_from_package wget wget 1
216 wget --no-check-certificate -c -q -O$nd_config_file_fresh \
218 && { echo "$nd_config_file_fresh"; } \
219 || { [ -e "$nd_config_file" ] \
220 && echo "$nd_config_file" \
221 || error 10 "Neither could fetch $nd_config_url, nor found $nd_config_file"; }
228 print_verbose 3 "Querying config $config_file section $section"
229 assure_command_from_package python python-minimal 1
230 python -c "from ConfigParser import SafeConfigParser as SP; cfg = SP(); cfg.read('$config_file'); print('\n'.join([' '.join(x) for x in cfg.items('$section')]))"
235 nd_config=`get_neurodebian_cfg`
236 # $exe_dir/nd_querycfg -F" " --config-file="$nd_config" "mirrors" \
238 query_cfg_section "$nd_config" "mirrors" \
239 | while read mirror_name mirror_url; do
240 # verify that url is just a url
241 if echo "$mirror_url" | grep -v -e '^[a-z0-9:+]*://[-+_%.a-z0-9/]*$'; then
242 print_verbose 1 "Mirror $mirror_name has 'illegit' URL: $mirror_url. Skipping"
244 [ -z "$n" ] || echo -ne "${ND_IFS:-\n}"; n+=1
245 echo -n "$mirror_name $mirror_url"
251 nd_config=`get_neurodebian_cfg`
253 query_cfg_section "$nd_config" "release files" \
254 | while read release_name release_url; do
255 # verify that url is just a url
256 if [ "$release_name" = "data" ]; then
260 [ -z "$n" ] || echo -ne "${ND_IFS:-\n}"; n+=1
261 echo -n "$release_name"
265 get_package_version()
267 pkg_version=$(apt-cache policy "$1" | awk '/^ *Installed:/{print $2;}')
268 [ "$pkg_version" != '(none)' ] || pkg_version=''
273 # select "closest" mirror according to netselect.
274 print_verbose 2 "Selecting the 'best' mirror using netselect"
275 assure_command_from_package netselect
276 if ! which netselect >&/dev/null; then
277 print_verbose 1 "netselect (apt-get install netselect) needed to select the 'best' mirror was not found"
278 print_verbose 1 "Selecting the default repository: $nd_mirror_default"
279 echo $nd_mirror_default
281 # squeeze version doesn't have -D yet to force output of the URL not IP, but for our mirrors ATM it shouldn't matter
282 netselect_opts="-s 1"
283 netselect_version="$(get_package_version netselect)"
284 if dpkg --compare-versions "$netselect_version" ge 0.3.ds1-17; then
285 netselect_opts+=" -D"
287 if dpkg --compare-versions "$netselect_version" ge 0.3.ds1-15; then
288 netselect_opts+=" -I"
290 best_mirror=$(get_mirrors | awk '{print $2;}' | eval $ae_sudo xargs netselect $netselect_opts | awk '{print $2;}')
291 if [ -z "$best_mirror" ]; then
292 print_verbose 1 "Failed to select mirror using netselect. Selecting default one ($nd_mirror_default)"
293 echo "$nd_mirror_default"
295 print_verbose 2 "Best mirror: $best_mirror"
303 # given mirror alias -- find its url
304 url=$(get_mirrors | awk "/^$1 /{print \$2;}")
305 if [ -z "$url" ]; then
306 error 9 "Cannot resolve mirror $1 to the URL"
313 # Get apt-cache policy output in a single list for matching suites
314 # (could be a separated with \| or , for multiple choices, e.g.
316 # get_apt_policy Debian,Ubuntu
318 # get_apt_policy NeuroDebian
320 $ae_sudo apt-cache policy | grep -B1 -e "o=\(${suites//,/\\|}\)" | tr '\n' ' ' | sed -e 's, -- ,\n,g' | grep -v -e '-\(updates\|security\)' | sort -nr
323 is_component_included()
325 echo "$ae_components" | tr ',' '\n' | grep -q "^$1\$"
330 apt-cache showsrc apt >&/dev/null && echo 1 || echo 0
333 assure_command_from_package()
339 which "$cmd" >&/dev/null && return 0
341 # if absent -- check availability of the package
342 apt_cache=$(LANG=C apt-cache policy "$pkg" 2>&1)
343 if [[ "$apt_cache" =~ Unable\ to\ locate\ package ]] || [[ "$apt_cache" =~ Candidate:\ (none) ]]; then
344 print_verbose 1 "Package $pkg providing command $cmd is N/A. Skipping"
347 if echo "$apt_cache" | grep -q '^\s*\*\*\*'; then
348 print_verbose 1 "WARNING -- command $cmd is N/A but package $pkg is claimed to be installed"
349 [ -z "$fail" ] && return 11 || error $fail "Command $cmd is required to proceed"
351 if [ "$ae_install" = "1" ]; then
352 print_verbose 1 "Installing $pkg package to get $cmd command"
353 eval_dry DEBIAN_FRONTEND=noninteractive apt-get install -y "$pkg"
356 print_verbose 1 "Command $cmd (from package $pkg) is N/A."
357 print_verbose 1 "Use with --install to get all necessary packages installed automatically"
358 [ -z "$fail" ] && return 12 || error $fail "Command $cmd is required to proceed"
362 # if it was requested -- return without doing anything
363 [ -z "$ae_defun_only" ] || return
366 # Commandline options handling
369 # Parse commandline options (taken from the getopt examples from the Debian util-linux package)
370 # Note that we use `"$@"' to let each command-line parameter expand to a
371 # separate word. The quotes around `$@' are essential!
372 # We need CLOPTS as the `eval set --' would nuke the return value of getopt.
373 CLOPTS=`getopt -o h,r:,m:,f:,c:,q,v,n --long help,version,quiet,verbose,mirror:,release:,flavor:,components:,suffix:,overwrite,sources,no-sources,install,dry-run,do-not-update,print-releases,print-release,print-mirrors,print-best-mirror,print-flavor -n 'nd-configurerepo' -- "$@"`
375 if [ $? != 0 ] ; then
376 error 2 "Problem with parsing cmdline. Terminating..."
379 # Note the quotes around `$CLOPTS': they are essential!
380 eval set -- "$CLOPTS"
382 if [ `whoami` != "root" ]; then
388 -r|--release) shift; ae_release="$1"; shift;;
389 -f|--flavor) shift; ae_flavor="$1"; shift;;
390 -c|--components) shift; ae_components="$1"; shift;;
391 -m|--mirror) shift; ae_mirror="$1"; shift;;
392 --print-mirrors) get_mirrors; exit 0;;
393 --print-best-mirror) netselect_mirror; exit 0;;
394 --print-releases) get_releases; exit 0;;
395 --print-release) do_print_release=1; shift;;
396 --print-flavor) do_print_flavor=1; shift;;
397 -n|--dry-run) ae_dry_run=1; shift;;
398 --suffix) shift; ae_suffix="$1"; shift;;
399 --overwrite) ae_overwrite="$1"; shift;;
400 --do-not-update) ae_update=""; shift;;
401 --sources) ae_sources=1; shift;;
402 --no-sources) ae_sources=0; shift;;
403 --install) ae_install=1; shift;;
404 -q|--quiet) ae_verbose=0; shift;;
405 -v|--verbose) ae_verbose=$(($ae_verbose+1)); shift;;
406 -h|--help) print_help; exit 0;;
407 --version) print_version; exit 0;;
409 *) error 1 "Internal error! ($1)";;
414 if [ $# -gt 0 ] ; then
420 [ -z "$ae_sudo" ] || print_verbose 1 "This script requires root access. Since current user is not root, sudo will be used"
423 # Basic system/environment knowledge
426 ae_output_file=/etc/apt/sources.list.d/neurodebian.sources${ae_suffix}.list
428 apt_policy=$(get_apt_policy "Debian,Ubuntu" )
430 if [ -z "$ae_release" ]; then
431 ae_release=$(echo "$apt_policy" | head -1 | sed -e 's/.*,n=\([^,]*\),.*/\1/g')
432 if [ ! -z "$do_print_release" ]; then
438 if [ -z "$ae_flavor" ]; then
439 ae_flavor=$(echo "$apt_policy" | grep -e ",n=$ae_release," | grep -qe 'c=\(non-free\|multiverse\)' && echo "full" || echo "libre")
440 if [ ! -z "$do_print_flavor" ]; then
447 # Determine which mirror to use
450 # knowing mirror is not necessary for -devel available only from the main site
451 if is_component_included software || is_component_included data; then
452 # for now just use default
453 if [ -z "$ae_mirror" ]; then # none specified
454 ae_mirror_url=$nd_mirror_origin
456 if ! [[ "$ae_mirror" =~ .*://.* ]]; then
458 best) ae_mirror_url=$(netselect_mirror);;
459 default) ae_mirror_url=$nd_mirror_default;;
460 origin) ae_mirror_url=$nd_mirror_origin;;
461 *) ae_mirror_url=$(get_mirror_url "$ae_mirror");;
464 ae_mirror_url="$ae_mirror" # it was some kind of a URL already
474 full) apt_flavor="contrib non-free";;
475 libre) apt_flavor="";;
476 *) error 5 "Unknown value of flavor $apt_flavor. Must be full or libre"
479 if [ -z "$ae_sources" ]; then
480 ae_sources=$(is_sources_enabled)
483 if [ $ae_sources -eq 0 ]; then
491 if is_component_included software; then
493 # NeuroDebian software repository
494 deb $ae_mirror_url $ae_release main $apt_flavor
495 ${sources_comment}deb-src $ae_mirror_url $ae_release main $apt_flavor
499 if is_component_included data; then
501 # NeuroDebian data repository
502 deb $ae_mirror_url data main $apt_flavor
503 ${sources_comment}deb-src $ae_mirror_url data main $apt_flavor
507 if is_component_included devel; then
509 # NeuroDebian -devel repository
510 deb http://neuro.debian.net/debian-devel $ae_release main $apt_flavor
511 ${sources_comment}deb-src http://neuro.debian.net/debian-devel $ae_release main $apt_flavor
515 if [ -e "$ae_output_file" ] && [ -z "$ae_overwrite" ]; then
516 if diff "$ae_output_file" <(echo "$apt_list") | grep -q .; then
518 print_verbose 1 "File $ae_output_file already exists, containing:\n`cat \"$ae_output_file\"`\n\nI: New configuration is different:\n$apt_list"
519 if get_apt_policy NeuroDebian >/dev/null; then
520 print_verbose 1 "NeuroDebian repositories are already available, thus skipping the rest."
521 print_verbose 1 "Rerun with --overwrite if you would like to reconfigure."
524 print_verbose 1 "NeuroDebian configuration is found but not yet available -- continuing with new configuration."
527 print_verbose 1 "New configuration is identical to existing and NeuroDebian repository is already enabled."
528 print_verbose 1 "Skiping the rest. Rerun with --overwrite if you would like to reconfigure."
532 print_verbose 1 "Generating $ae_output_file"
533 if [ -z "$ae_dry_run" ]; then
534 echo "$apt_list" | $ae_sudo bash -c "cat - >| '$ae_output_file'"
543 # Assure present archive GPG key for APT system
546 # Figure out if key needs to be imported (if ran within package,
547 # should already be there due to neurodebian-keyring package)
548 if LANG=C eval $ae_sudo apt-key export $nd_key_id 2>&1 1>/dev/null | grep -qe "nothing exported"; then
549 print_verbose 1 "Fetching the key from the server"
550 eval_dry apt-key adv --recv-keys --keyserver pgp.mit.edu $nd_key_id
555 # Finalizing (apt-get update etc)
558 if [ ! -z "$ae_update" ]; then
559 print_verbose 1 "Updating APT listings, might take a few minutes"
560 if [ -z "$ae_dry_run" ]; then
561 apt_logfile="$ae_tempdir/apt.log"
562 $ae_sudo apt-get update 1>"$apt_logfile" 2>&1 \
563 && rm -f "$apt_logfile" \
565 apt_log=$(cat "$apt_logfile")
567 if echo "$apt_log" | grep -q "Malformed line [0-9]* in source list $ae_output_file"; then
568 $ae_sudo mv "${ae_output_file}" "${ae_output_file}-failed.disabled"
569 error 6 "Update failed to possible errorneous APT listing file produced by this script. Generated $ae_output_file renamed to ${ae_output_file}-failed.disabled to not interfer"
571 error 5 "Update failed with exit code $? (above output logged into $apt_logfile)."
574 eval_dry apt-get update
577 print_verbose 1 "apt-get update was not run. Please run to take an effect of changes"
580 if [ "$ae_verbose" -ge 2 ]; then
581 print_verbose 2 "Currently enabled NeuroDebian suites/mirrors:"
582 get_apt_policy NeuroDebian