From d6e20ea4c083d9978ad4c3e5eccaffa2e887698a Mon Sep 17 00:00:00 2001 From: Peter Palfrader Date: Sat, 8 Dec 2012 14:44:34 +0100 Subject: [PATCH] static mirroring system --- modules/debian-org/misc/local.yaml | 4 + .../files/static-mirroring/static-master-run | 173 +++++++++++++++++ .../static-mirroring/static-master-ssh-wrap | 157 ++++++++++++++++ .../static-master-update-component | 130 +++++++++++++ .../files/static-mirroring/static-mirror-run | 174 ++++++++++++++++++ .../static-mirroring/static-mirror-ssh-wrap | 129 +++++++++++++ modules/roles/manifests/init.pp | 8 + modules/roles/manifests/static_master.pp | 22 +++ modules/roles/manifests/static_mirror.pp | 44 +++++ .../static-master-authorized_keys.erb | 42 +++++ .../static-mirror-authorized_keys.erb | 42 +++++ 11 files changed, 925 insertions(+) create mode 100755 modules/roles/files/static-mirroring/static-master-run create mode 100755 modules/roles/files/static-mirroring/static-master-ssh-wrap create mode 100755 modules/roles/files/static-mirroring/static-master-update-component create mode 100755 modules/roles/files/static-mirroring/static-mirror-run create mode 100755 modules/roles/files/static-mirroring/static-mirror-ssh-wrap create mode 100644 modules/roles/manifests/static_master.pp create mode 100644 modules/roles/manifests/static_mirror.pp create mode 100644 modules/roles/templates/static-master-authorized_keys.erb create mode 100644 modules/roles/templates/static-mirror-authorized_keys.erb diff --git a/modules/debian-org/misc/local.yaml b/modules/debian-org/misc/local.yaml index a8715867..9dbbe8d4 100644 --- a/modules/debian-org/misc/local.yaml +++ b/modules/debian-org/misc/local.yaml @@ -277,5 +277,9 @@ host_settings: - rautavaara.debian.org - salieri.debian.org - widor.debian.org + static_master: + - bizet.debian.org + static_mirror: + - klecker.debian.org # reservedaddrs: # ball.debian.org: "0.0.0.0/8 : 127.0.0.0/8 : 169.254.0.0/16 : 172.16.0.0/12 : 192.0.0.0/17 : 192.168.0.0/16 : 224.0.0.0/4 : 240.0.0.0/5 : 248.0.0.0/5" diff --git a/modules/roles/files/static-mirroring/static-master-run b/modules/roles/files/static-mirroring/static-master-run new file mode 100755 index 00000000..24ea3e11 --- /dev/null +++ b/modules/roles/files/static-mirroring/static-master-run @@ -0,0 +1,173 @@ +#!/usr/bin/python + +import fcntl +import os +import shutil +import subprocess +import string +import tempfile +import time + +base='/home/staticsync/static-master' +subdirs = { 'master': 'master', # where updates from off-site end up going, the source of everything we do here + 'cur': 'current-push', # where clients rsync from during a mirror push + 'live': 'current-live'} # what is currently on the mirrors, and what they rsync from when they come back from being down +serialname = '.serial' + +clients = [] +with open('/home/staticsync/etc/static-clients') as f: + for line in f: + clients.append(line.strip()) + +def log(m): + t = time.strftime("[%Y-%m-%d %H:%M:%S]", time.gmtime()) + print t, m + +def stage1(pipes, status): + for c in clients: + p = pipes[c] + while 1: + line = p.stdout.readline() + if line == '': + status[c] = 'failed' + p.stdout.close() + p.stdin.close() + p.wait() + log("%s: failed with returncode %d"%(c,p.returncode)) + break + + line = line.strip() + log("%s >> %s"%(c, line)) + if not line.startswith('[MSM]'): continue + kw = string.split(line, ' ', 2)[1] + + if kw == 'ALREADY-CURRENT': + pipes[c].stdout.close() + pipes[c].stdin.close() + p.wait() + if p.returncode == 0: + log("%s: already current"%(c,)) + status[c] = 'ok' + else: + log("%s: said ALREADY-CURRENT but returncode %d"%(c,p.returncode)) + status[c] = 'failed' + break + elif kw == 'STAGE1-DONE': + log("%s: waiting"%(c,)) + status[c] = 'waiting' + break + elif kw in ['STAGE1-START']: + pass + else: + log("%s: ignoring unknown line"%(c,)) + +def count_statuses(status): + cnt = {} + for k in status: + v = status[k] + if v not in cnt: cnt[v] = 1 + else: cnt[v] += 1 + return cnt + +def stage2(pipes, status, command): + for c in clients: + if status[c] != 'waiting': continue + log("%s << %s"%(c, command)) + pipes[c].stdin.write("%s\n"%(command,)) + + for c in clients: + if status[c] != 'waiting': continue + p = pipes[c] + + (o, dummy) = p.communicate('') + for l in string.split(o, "\n"): + log("%s >> %s"%(c, l)) + log("%s: returned %d"%(c, p.returncode)) + +def callout(serial): + log("Calling clients...") + pipes = {} + status = {} + for c in clients: + args = ['ssh', '-o', 'BatchMode=yes', c, 'mirror', "%d"%(serial,)] + p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pipes[c] = p + status[c] = 'in-progress' + + log("Stage 1...") + stage1(pipes, status) + log("Stage 1 done.") + cnt = count_statuses(status) + + if 'failed' in cnt > 0: + log("Some clients failed, aborting...") + stage2(pipes, status, 'abort') + return False + elif 'waiting' in cnt > 0: + log("Committing...") + stage2(pipes, status, 'go') + return True + else: + log("All clients up to date.") + return True + + +cleanup_dirs = [] +def run_mirror(): + # setup + master = os.path.join(base, subdirs['master']) + cur = os.path.join(base, subdirs['cur']) + live = os.path.join(base, subdirs['live']) + tmpdir_new = tempfile.mkdtemp(prefix='live.new-', dir=base); cleanup_dirs.append(tmpdir_new); + tmpdir_old = tempfile.mkdtemp(prefix='live.new-', dir=base); cleanup_dirs.append(tmpdir_old); + os.chmod(tmpdir_new, 0755) + + locks = [] + for p in (master, live, tmpdir_new): + if not os.path.exists(p): os.mkdir(p, 0755) + fd = os.open(p, os.O_RDONLY) + log("Acquiring lock for %s(%d)."%(p,fd)) + fcntl.flock(fd, fcntl.LOCK_EX) + locks.append(fd) + log("All locks acquired.") + + serialfile = os.path.join(master, serialname) + try: + with open(serialfile) as f: serial = int(f.read()) + except: + serial = int(time.time()) + with open(serialfile, "w") as f: f.write("%d\n"%(serial,)) + log("Serial is %s."%(serial,)) + + log("Populating %s."%(tmpdir_new,)) + subprocess.check_call(['cp', '-al', os.path.join(master, '.'), tmpdir_new]) + + if os.path.exists(cur): + log("Removing existing %s."%(cur,)) + shutil.rmtree(cur) + + log("Renaming %s to %s."%(tmpdir_new, cur)) + os.rename(tmpdir_new, cur) + + proceed = callout(serial) + + if proceed: + log("Moving %s aside."%(live,)) + os.rename(live, os.path.join(tmpdir_old, 'old')) + log("Renaming %s to %s."%(cur, live)) + os.rename(cur, live) + log("Cleaning up.") + shutil.rmtree(tmpdir_old) + log("Done.") + else: + log("Aborted.") + + +try: + run_mirror() +finally: + for p in cleanup_dirs: + if os.path.exists(p): shutil.rmtree(p) +# vim:set et: +# vim:set ts=2: +# vim:set shiftwidth=2: diff --git a/modules/roles/files/static-mirroring/static-master-ssh-wrap b/modules/roles/files/static-mirroring/static-master-ssh-wrap new file mode 100755 index 00000000..2b2ddea7 --- /dev/null +++ b/modules/roles/files/static-mirroring/static-master-ssh-wrap @@ -0,0 +1,157 @@ +#!/bin/bash + +# Copyright (c) 2009, 2010, 2012 Peter Palfrader +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +set -e +set -u + +MYLOGNAME="`basename "$0"`[$$]" +BASEDIR="/home/staticsync/static-master" + +usage() { + echo "local Usage: $0 " + echo "via ssh orig command:" + echo " rsync " + echo " static-master-update-component " +} + +one_more_arg() { + if [ "$#" -lt 1 ]; then + usage >&2 + exit 1 + fi +} + +info() { + logger -p daemon.info -t "$MYLOGNAME" "$1" +} + +croak() { + logger -s -p daemon.warn -t "$MYLOGNAME" "$1" + exit 1 +} + +lock() { + local fd="$1"; shift + local path="$1"; shift + local exclusive="$1"; shift + + eval "exec $fd< '$path'" + + if [ "$exclusive" -gt 0 ]; then + locktype="-e" + else + locktype="-s" + fi + + if ! flock "$locktype" "$fd"; then + echo >&2 "$0: Cannot acquire lock on $base (flock $locktype failed) - Very bad, we should have waited!" + exit 1 + fi +} + +serve_dir() { + local remote_host="$1"; shift + local path="$1"; shift + + local sender='rsync --server --sender -vlogDtprze.iLsf . ' + + if [ -e "$path" ]; then + info "serving $remote_host with $path" + $sender "$path/" + else + info "$remote_host wants non-existing $path" + echo >&2 "$path does not exist." + exit 1 + fi +} + +do_rsync() { + local remote_host="$1"; shift + + if [ "$*" = "--server --sender -vlogDtprze.iLsf . -new-/" ] ; then + serve_dir "$remote_host" "$BASEDIR/current-push" + elif [ "$*" = "--server --sender -vlogDtprze.iLsf . -live-/" ] ; then + local p="$BASEDIR/current-live" + info "host $remote_host wants $p, acquiring lock" + lock 200 "$p" 0 + serve_dir "$remote_host" "$p" + else + info "NOT allowed for $remote_host: rsync $*" + echo >&2 "This rsync command ($@) not allowed." + exit 1 + fi +} + +do_update_component() { + local remote_host="$1"; shift + + one_more_arg "$@" + component="$1" + shift + + #if [ "$component" = "www.torproject.org" ] && [ "$remote_host" = "vescum.torproject.org" ]; then + # exec static-master-update-component "$component" + # echo >&2 "Exec failed" + # croak "exec failed" + #else + info "Not whitelisted: $remote_host update $component" + echo >&2 "Not whitelisted: $remote_host update $component" + exit 1 + #fi +} + + +if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 +fi + +one_more_arg "$@" +remote_host="$1" +shift + + +# check/parse remote command line +if [ -z "${SSH_ORIGINAL_COMMAND:-}" ] ; then + croak "Did not find SSH_ORIGINAL_COMMAND" +fi +set "dummy" ${SSH_ORIGINAL_COMMAND} +shift + +info "host $remote_host called with $*" + +one_more_arg "$@" +action="$1" +shift + +case "$action" in + rsync) + do_rsync "$remote_host" "$@" + ;; + static-master-update-component) + do_update_component "$remote_host" "$@" + ;; + *) + croak "Invalid operation '$action'" + ;; +esac diff --git a/modules/roles/files/static-mirroring/static-master-update-component b/modules/roles/files/static-mirroring/static-master-update-component new file mode 100755 index 00000000..a79f342f --- /dev/null +++ b/modules/roles/files/static-mirroring/static-master-update-component @@ -0,0 +1,130 @@ +#!/bin/bash + +# Updates one component (i.e. subdirectory) in static-master/master + +# acquires a shared lock on the base directory (so that we know no updates are +# outgoing, as those acquire an exclusive one). Also acquired an exclusive lock +# on the component directory in question. +# +# The config file is a list of component source-directory pairs. + +# Copyright (c) 2012 Peter Palfrader +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +componentlist=/home/staticsync/etc/static-components +base=/home/staticsync/static-master/master + +set -e +set -u + + +lock() { + local fd="$1"; shift + local path="$1"; shift + local exclusive="$1"; shift + + eval "exec $fd< '$path'" + + if [ "$exclusive" -gt 0 ]; then + locktype="-e" + else + locktype="-s" + fi + + if ! flock "$locktype" "$fd"; then + echo >&2 "$0: Cannot acquire lock on $base (flock $locktype failed) - Very bad, we should have waited!" + exit 1 + fi +} + +unlock() { + local fd="$1"; shift + + if ! flock -o "$fd"; then + echo >&2 "$0: Cannot release lock on fd $fd - This should not have happened!" + exit 1 + fi + eval "exec $fd<&-" +} + +if [ "$#" != 1 ]; then + echo >&2 "Usage: $0 " + exit 1 +fi + +component="$1" + +if [ "${component%/*}" != "$component" ] ; then + echo >&2 "$0: Invalid component: $component"; + exit 1 +fi + +src="$(awk -v component="$component" '$1 == component {print $2; exit}' "$componentlist")" +if [ -z "$src" ]; then + echo >&2 "$0: Invalid component: $component (not found in $componentlist)"; + exit 1 +fi +tgt="$base/$component" +if ! [ -d "$tgt" ]; then + echo >&2 "$0: Invalid component: $component ($tgt does not exist)"; + exit 1 +fi + +echo "$0: Acquiring locks..." +lock 200 "$base" 0 +lock 201 "$tgt" 1 + +tmpdir_new="$(mktemp -d --tmpdir="$base" "${component}.new-XXXXXX")" +tmpdir_old="$(mktemp -d --tmpdir="$base" "${component}.old-XXXXXX")" +trap "rm -rf '$tmpdir_new' '$tmpdir_old'" EXIT +chmod 0755 "$tmpdir_new" + +lock 202 "$tmpdir_new" 1 +echo "$0: Got them." + +echo "$0: Updating master copy of $component..." +rsync --delete \ + -tr \ + --link-dest="$tgt" \ + "$src/." "$tmpdir_new/." +echo "$0: Done. Committing." + +mv "$tgt" "$tmpdir_old/old" +if ! mv "$tmpdir_new" "$tgt"; then + echo >&2 "$0: WARNING: could not move $tmpdir_new to $tgt. Trying to recover" + rm -rf "$tgt" + mv "$tmpdir_old/old" "$tgt" + echo >&2 "$0: Rolled back to old tree maybe successfully." + exit 1 +fi + +rm -rf "$tmpdir_new" "$tmpdir_old" +trap - EXIT + +date '+%s' > "$base/.serial" +unlock 201 +unlock 200 +echo "$0: Triggering mirror runs..." +exec static-master-run + +# vim:set et: +# vim:set ts=2: +# vim:set shiftwidth=2: diff --git a/modules/roles/files/static-mirroring/static-mirror-run b/modules/roles/files/static-mirroring/static-mirror-run new file mode 100755 index 00000000..bdcc9283 --- /dev/null +++ b/modules/roles/files/static-mirroring/static-mirror-run @@ -0,0 +1,174 @@ +#!/bin/bash + +# initiate a mirror staged mirror update from sync-source. +# +# if we have a serial file and we got a serial on the command line, only sync if the serial is different + +# Copyright (c) 2012 Peter Palfrader +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +set -e +set -u + +NAME="$(basename "$0")" + +usage() { + echo "Usage: $0 [--one-stage] []" +} + +if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then usage; exit 0; fi + +one_stage=0 +while :; do + case "${1:-}" in + --) + shift + break; + ;; + --one-stage) + shift + one_stage=1 + ;; + -*) usage >&2 + exit 1 + ;; + *) + break + ;; + esac +done + +BASEDIR=${1:-}; shift +SYNC_SOURCE=${1:-}; shift +SYNC_SERIAL=${1:-}; shift || true +if [ -z "$BASEDIR" ]; then usage >&2; exit 1; fi +if [ -z "$SYNC_SOURCE" ]; then usage >&2; exit 1; fi + +RSYNC="rsync" +RSYNC_BASE_OPTIONS="-avz --delete" +RSYNC_SSH_OPTIONS="ssh -o AddressFamily=inet -o BatchMode=yes" + +LOGDIR="$HOME/logs" +LOGFILE="$LOGDIR/$NAME-run.log" + + +ALPHA="tree-a" +BRAVO="tree-b" +ACTIVE="cur" + +CNF_FILE="$HOME/etc/$NAME.conf" +! [ -e "$CNF_FILE" ] || . "$CNF_FILE" + +SOURCE="${SYNC_SOURCE}/" +BASEDIR="${BASEDIR}/" + +############################################### + +# point stdout and stderr to the logfile if it's not a tty. +# save stdout to fd5 for communications with the master + +log_setup() { + mkdir -p "$LOGDIR" + if ! [ -t 1 ]; then + # move current stdout to fd5 and reopen to logfile + exec 5>&1- + exec 1>> "$LOGFILE" + else + # duplicate stdout to fd5 + exec 5>&1 + fi + if ! [ -t 2 ]; then + exec 2>> "$LOGFILE" + fi +} + +log() { + echo "[$(date)][$NAME][$$] $1" +} + +lock() { + exec 200< "$BASEDIR" + if ! flock -e 200; then + log "Cannot acquire lock." + echo >&5 "[MSM] LOCK-ERROR" + exit 1 + fi + log "Got the lock." +} + +############################################### + + +log_setup +#log "called with $* and ${SSH_ORIGINAL_COMMAND:-no ssh original command options}." +log "called with $*" +lock + +if [ -e "${BASEDIR}${ACTIVE}" ] && [ "$(readlink "${BASEDIR}${ACTIVE}")" = "$ALPHA" ] ; then + staging="$BRAVO" + active="$ALPHA" +else + staging="$ALPHA" + active="$BRAVO" +fi +log "active is $active; staging is $staging" + +rsync_source="${SOURCE}" +rsync_curactive="${BASEDIR}${active}/" +rsync_target="${BASEDIR}${staging}/" + +if [ -e "$rsync_curactive/.serial" ] && [ -n "$SYNC_SERIAL" ] && [ "$(cat $rsync_curactive/.serial)" = "$SYNC_SERIAL" ]; then + log "active is already at serial $SYNC_SERIAL. No action required." + echo >&5 "[MSM] ALREADY-CURRENT" + exit 0 +fi + +echo >&5 "[MSM] STAGE1-START" +log "Running $RSYNC $RSYNC_BASE_OPTIONS -e $RSYNC_SSH_OPTIONS --link-dest $rsync_curactive $rsync_source $rsync_target" +$RSYNC $RSYNC_BASE_OPTIONS -e "$RSYNC_SSH_OPTIONS" --link-dest "$rsync_curactive" "$rsync_source" "$rsync_target" +log "rsync done." +echo >&5 "[MSM] STAGE1-DONE" +if [ "$one_stage" -gt 0 ]; then + action="go" +else + read action +fi + +case "$action" in + go) + ln --symbolic --force --no-target-directory "$staging" "${BASEDIR}$ACTIVE" + rm -rf "$rsync_curactive" + echo >&5 "[MSM] STAGE2-DONE" + log "stage2 done" + ;; + abort) + echo >&5 "[MSM] STAGE2-ABORT" + log "stage2 abort" + ;; + *) + echo >&5 "[MSM] STAGE2-UNKNOWN-ACTION $action" + log "stage2 unknown action $action" + exit 1 + ;; +esac + +savelog "$LOGFILE" diff --git a/modules/roles/files/static-mirroring/static-mirror-ssh-wrap b/modules/roles/files/static-mirroring/static-mirror-ssh-wrap new file mode 100755 index 00000000..f7a6a81c --- /dev/null +++ b/modules/roles/files/static-mirroring/static-mirror-ssh-wrap @@ -0,0 +1,129 @@ +#!/bin/bash + +# Copyright (c) 2009, 2010, 2012 Peter Palfrader +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +set -e +set -u + +MYLOGNAME="`basename "$0"`[$$]" + +usage() { + echo "local Usage: $0 " + echo "via ssh orig command:" + echo " mirror " +} + +one_more_arg() { + if [ "$#" -lt 1 ]; then + usage >&2 + exit 1 + fi +} + +info() { + logger -p daemon.info -t "$MYLOGNAME" "$1" +} + +croak() { + logger -s -p daemon.warn -t "$MYLOGNAME" "$1" + exit 1 +} + +do_mirror() { + local basedir="$1"; shift + local remote_host="$1"; shift + one_more_arg "$@" + local serial="$1"; shift + + info "Host $remote_host triggered a mirror run for serial $serial" + exec /usr/local/bin/static-mirror-run "$basedir" "$remote_host:-new-" "$serial" + echo >&2 "Exec failed" + croak "exec failed" +} + +do_rsync() { + local remote_host="$1" + shift + + local allowed_rsyncs + allowed_rsyncs=() + + #case "`hostname`" in + # vescum) + # allowed_rsyncs=( + # '^--server --sender -tre\.iLsf \. /srv/www-master\.torproject\.org/htdocs/\.$' + # ) + # ;; + # *) + #esac + for cmd_idx in ${!allowed_rsyncs[*]}; do + allowed="${allowed_rsyncs[$cmd_idx]}" + if [[ "$*" =~ $allowed ]]; then # do !not! quote regex + info "Running for host $remote_host: rsync $*" + exec rsync "$@" + echo >&2 "Exec failed" + exit 1 + fi + done + + info "NOT allowed for $remote_host: rsync $*" + echo >&2 "This rsync command ($*) not allowed." + exit 1 +} + + +if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 +fi + +one_more_arg "$@" +basedir="$1" +shift + +one_more_arg "$@" +remote_host="$1" +shift + + +# check/parse remote command line +if [ -z "${SSH_ORIGINAL_COMMAND:-}" ] ; then + croak "Did not find SSH_ORIGINAL_COMMAND" +fi +set "dummy" ${SSH_ORIGINAL_COMMAND} +shift + +one_more_arg "$@" +action="$1" +shift + +case "$action" in + mirror) + do_mirror "$basedir" "$remote_host" "$@" + ;; + rsync) + do_rsync "$remote_host" "$@" + ;; + *) + croak "Invalid operation '$action'" + ;; +esac diff --git a/modules/roles/manifests/init.pp b/modules/roles/manifests/init.pp index 647f4576..ede7f9b1 100644 --- a/modules/roles/manifests/init.pp +++ b/modules/roles/manifests/init.pp @@ -66,4 +66,12 @@ class roles { if getfromhash($site::nodeinfo, 'apache2_ftp-upcoming_mirror') { include roles::ftp-upcoming_mirror } + + if getfromhash($site::nodeinfo, 'static_master') { + include roles::static_master + } + + if getfromhash($site::nodeinfo, 'static_mirror') { + include roles::static_mirror + } } diff --git a/modules/roles/manifests/static_master.pp b/modules/roles/manifests/static_master.pp new file mode 100644 index 00000000..6750aa6a --- /dev/null +++ b/modules/roles/manifests/static_master.pp @@ -0,0 +1,22 @@ +class roles::static_master inherits roles::mirror_base { + file { + '/etc/ssh/userkeys/staticsync': + content => template('roles/static-master-authorized_keys.erb'), + ; + '/usr/local/bin/static-master-run': + source => "puppet:///modules/roles/static-mirroring/static-master-run", + mode => 555, + ; + '/usr/local/bin/static-master-ssh-wrap': + source => "puppet:///modules/roles/static-mirroring/static-master-ssh-wrap", + mode => 555, + ; + '/usr/local/bin/static-master-update-component': + source => "puppet:///modules/roles/static-mirroring/static-master-update-component", + mode => 555, + ; + } +} +# vim:set et: +# vim:set sts=4 ts=4: +# vim:set shiftwidth=4: diff --git a/modules/roles/manifests/static_mirror.pp b/modules/roles/manifests/static_mirror.pp new file mode 100644 index 00000000..9b10b103 --- /dev/null +++ b/modules/roles/manifests/static_mirror.pp @@ -0,0 +1,44 @@ +class roles::static_mirror inherits roles::mirror_base { + file { + '/etc/ssh/userkeys/staticsync': + content => template('roles/static-mirror-authorized_keys.erb'), + ; + '/usr/local/bin/static-mirror-run': + source => "puppet:///modules/roles/static-mirroring/static-mirror-run", + mode => 555, + ; + '/usr/local/bin/static-mirror-ssh-wrap': + source => "puppet:///modules/roles/static-mirroring/static-mirror-ssh-wrap", + mode => 555, + ; + } + + file { + "/etc/cron.d/puppet-static-mirror": + content => "PATH=/usr/local/bin:/usr/bin:/bin\n@reboot staticsync sleep 60; static-mirror-run --one-stage /srv/static.debian.org bizet.debian.org:-live- > /dev/null\n", + ; + + #"/etc/apache2/sites-available/dist.torproject.org": + # source => "puppet:///modules/roles/static-mirroring/vhost/dist.torproject.org", + # require => Package["apache2"], + # notify => Exec["reload-apache2"], + # ; + #"/etc/apache2/sites-available/www.torproject.org": + # source => "puppet:///modules/roles/static-mirroring/vhost/www.torproject.org", + # require => Package["apache2"], + # notify => Exec["reload-apache2"], + # ; + } + + #apache2::activate_apache_site { + # "10-dist.torproject.org": + # site => "dist.torproject.org", + # require => File['/etc/ssl/certs/apache-wildcard.torproject.org.pem']; + # "10-www.torproject.org": + # site => "www.torproject.org", + # require => File['/etc/ssl/certs/apache-wildcard.torproject.org.pem']; + #} +} +# vim:set et: +# vim:set sts=4 ts=4: +# vim:set shiftwidth=4: diff --git a/modules/roles/templates/static-master-authorized_keys.erb b/modules/roles/templates/static-master-authorized_keys.erb new file mode 100644 index 00000000..27bd4488 --- /dev/null +++ b/modules/roles/templates/static-master-authorized_keys.erb @@ -0,0 +1,42 @@ +## +## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE. +## + +<%= +def getstaticsynckey(host) + key = nil + begin + facts = YAML.load(File.open("/var/lib/puppet/yaml/facts/#{host}.yaml").read) + return facts.values['staticsync_key'] + rescue Exception => e + end + return key +end + +mirrors = [] +scope.lookupvar('site::localinfo').keys.sort.each do |node| + if scope.lookupvar('site::localinfo')[node]['static_mirror'] + key = getstaticsynckey(node) + mirrors << { 'node' => node, 'addr' => scope.lookupvar('site::allnodeinfo')[node]['ipHostNumber'], 'key' => key} + end +end + + +lines = [] +for m in mirrors: + lines << '# ' + m['node'] + if m['key'].nil? + lines << "# no key for node" + else + lines << "command=\"/usr/local/bin/static-master-ssh-wrap #{m['node']}\"," + + 'no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-user-rc,' + + 'from="' + m['addr'].join(',') + '" ' + + m['key'] + end +end + +lines.join("\n") +# vim:set et: +# vim:set sts=4 ts=4: +# vim:set shiftwidth=4: +%> diff --git a/modules/roles/templates/static-mirror-authorized_keys.erb b/modules/roles/templates/static-mirror-authorized_keys.erb new file mode 100644 index 00000000..74bb7d59 --- /dev/null +++ b/modules/roles/templates/static-mirror-authorized_keys.erb @@ -0,0 +1,42 @@ +## +## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE. +## + +<%= +def getstaticsynckey(host) + key = nil + begin + facts = YAML.load(File.open("/var/lib/puppet/yaml/facts/#{host}.yaml").read) + return facts.values['staticsync_key'] + rescue Exception => e + end + return key +end + +masters = [] +scope.lookupvar('site::localinfo').keys.sort.each do |node| + if scope.lookupvar('site::localinfo')[node]['static_master'] + key = getstaticsynckey(node) + masters << { 'node' => node, 'addr' => scope.lookupvar('site::allnodeinfo')[node]['ipHostNumber'], 'key' => key} + end +end + + +lines = [] +for m in masters: + lines << '# ' + m['node'] + if m['key'].nil? + lines << "# no key for node" + else + lines << "command=\"/usr/local/bin/static-mirror-ssh-wrap /srv/static.debian.org #{m['node']}\"," + + 'no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-user-rc,' + + 'from="' + m['addr'].join(',') + '" ' + + m['key'] + end +end + +lines.join("\n") +# vim:set et: +# vim:set sts=4 ts=4: +# vim:set shiftwidth=4: +%> -- 2.39.2