Add debaday.debian.net

[dsa-puppet.git] / modules / roles / files / static-mirroring / static-master-run

diff --git a/modules/roles/files/static-mirroring/static-master-run b/modules/roles/files/static-mirroring/static-master-run
index 0d7003d45500e6fc98691c8da6b179f4b2509cdf..c85e984470ee7f4c436d87c3ca417a8152c7e260 100755 (executable)
--- a/modules/roles/files/static-mirroring/static-master-run
+++ b/modules/roles/files/static-mirroring/static-master-run
@@ -5,14 +5,13 @@ import os
 import shutil
 import subprocess
 import string
+import sys
 import tempfile
 import time
 
-base='/home/staticsync/static-master'
-subdirs = { 'master': 'master',       # where updates from off-site end up going, the source of everything we do here
-            'cur':    'current-push', # where clients rsync from during a mirror push
-            'live':   'current-live'} # what is currently on the mirrors, and what they rsync from when they come back from being down
+base="/srv/static.debian.org"
 serialname = '.serial'
+had_warnings = False
 
 clients = []
 with open('/etc/static-clients.conf') as f:
@@ -87,12 +86,12 @@ def stage2(pipes, status, command):
       log("%s >> %s"%(c, l))
     log("%s: returned %d"%(c, p.returncode))
 
-def callout(serial):
+def callout(component, serial):
   log("Calling clients...")
   pipes = {}
   status = {}
   for c in clients:
-    args = ['ssh', '-o', 'BatchMode=yes', c, 'mirror', "%d"%(serial,)]
+    args = ['ssh', '-o', 'BatchMode=yes', c, 'mirror', component, "%d"%(serial,)]
     p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     pipes[c] = p
     status[c] = 'in-progress'
@@ -102,11 +101,24 @@ def callout(serial):
   log("Stage 1 done.")
   cnt = count_statuses(status)
 
-  if 'failed' in cnt > 0:
-    log("Some clients failed, aborting...")
+  if 'failed' in cnt and cnt['failed'] >= 2:
+    log("%d clients failed, aborting..."%(cnt['failed'],))
     stage2(pipes, status, 'abort')
     return False
-  elif 'waiting' in cnt > 0:
+
+  failedmirrorsfile = os.path.join(base, 'master', component + "-failedmirrors")
+  if 'failed' in cnt:
+    log("WARNING: %d clients failed!  Continuing anyway!"%(cnt['failed'],))
+    global had_warnings
+    had_warnings = True
+    f = open(failedmirrorsfile, "w")
+    for c in status:
+      if status[c] == 'failed': f.write(c+"\n")
+    f.close()
+  else:
+    if os.path.exists(failedmirrorsfile): os.unlink(failedmirrorsfile)
+
+  if 'waiting' in cnt:
     log("Committing...")
     stage2(pipes, status, 'go')
     return True
@@ -116,17 +128,18 @@ def callout(serial):
 
 
 cleanup_dirs = []
-def run_mirror():
+def run_mirror(component):
   # setup
-  master = os.path.join(base, subdirs['master'])
-  cur = os.path.join(base, subdirs['cur'])
-  live = os.path.join(base, subdirs['live'])
-  tmpdir_new = tempfile.mkdtemp(prefix='live.new-', dir=base); cleanup_dirs.append(tmpdir_new);
-  tmpdir_old = tempfile.mkdtemp(prefix='live.new-', dir=base); cleanup_dirs.append(tmpdir_old);
+  basemaster = os.path.join(base, 'master')
+  componentdir = os.path.join(basemaster, component)
+  cur = componentdir + '-current-push'
+  live = componentdir + '-current-live'
+  tmpdir_new = tempfile.mkdtemp(prefix=component+'-live.new-', dir=basemaster); cleanup_dirs.append(tmpdir_new);
+  tmpdir_old = tempfile.mkdtemp(prefix=component+'-live.old-', dir=basemaster); cleanup_dirs.append(tmpdir_old);
   os.chmod(tmpdir_new, 0755)
 
   locks = []
-  for p in (master, live, tmpdir_new):
+  for p in (componentdir, live, tmpdir_new):
     if not os.path.exists(p): os.mkdir(p, 0755)
     fd = os.open(p, os.O_RDONLY)
     log("Acquiring lock for %s(%d)."%(p,fd))
@@ -134,7 +147,7 @@ def run_mirror():
     locks.append(fd)
   log("All locks acquired.")
 
-  serialfile = os.path.join(master, serialname)
+  serialfile = os.path.join(componentdir, serialname)
   try:
     with open(serialfile) as f: serial = int(f.read())
   except:
@@ -143,7 +156,7 @@ def run_mirror():
   log("Serial is %s."%(serial,))
 
   log("Populating %s."%(tmpdir_new,))
-  subprocess.check_call(['cp', '-al', os.path.join(master, '.'), tmpdir_new])
+  subprocess.check_call(['cp', '-al', os.path.join(componentdir, '.'), tmpdir_new])
 
   if os.path.exists(cur):
     log("Removing existing %s."%(cur,))
@@ -152,7 +165,7 @@ def run_mirror():
   log("Renaming %s to %s."%(tmpdir_new, cur))
   os.rename(tmpdir_new, cur)
 
-  proceed = callout(serial)
+  proceed = callout(component, serial)
 
   if proceed:
     log("Moving %s aside."%(live,))
@@ -161,16 +174,33 @@ def run_mirror():
     os.rename(cur, live)
     log("Cleaning up.")
     shutil.rmtree(tmpdir_old)
-    log("Done.")
+    if had_warnings: log("Done, with warnings.")
+    else: log("Done.")
+    ret = True
   else:
     log("Aborted.")
+    ret = False
+
+  for fd in locks:
+    os.close(fd)
 
+  return ret
 
+
+if len(sys.argv) != 2:
+  print >> sys.stderr, "Usage: %s <component>"%(sys.argv[0],)
+  sys.exit(1)
+component = sys.argv[1]
+
+ok = False
 try:
-  run_mirror()
+  ok = run_mirror(component)
 finally:
   for p in cleanup_dirs:
     if os.path.exists(p): shutil.rmtree(p)
+
+if not ok:
+  sys.exit(1)
 # vim:set et:
 # vim:set ts=2:
 # vim:set shiftwidth=2: