Try porting torproject changes: support excluding mirror for a single static component

[dsa-puppet.git] / modules / roles / files / static-mirroring / static-master-run

diff --git a/modules/roles/files/static-mirroring/static-master-run b/modules/roles/files/static-mirroring/static-master-run
index fab3baa404e41c6e4e2651f86afafa0ba462ad80..e2a583e9ee4fa5a325a6271e9b3099d9daec7540 100755 (executable)
--- a/modules/roles/files/static-mirroring/static-master-run
+++ b/modules/roles/files/static-mirroring/static-master-run
@@ -9,22 +9,23 @@ import sys
 import tempfile
 import time
 
-base='/home/staticsync/static-master'
+base="/srv/static.debian.org"
 serialname = '.serial'
+had_warnings = False
 
-clients = []
+allclients = set()
 with open('/etc/static-clients.conf') as f:
   for line in f:
     line = line.strip()
     if line == "": continue
     if line.startswith('#'): continue
-    clients.append(line)
+    allclients.add(line)
 
 def log(m):
   t = time.strftime("[%Y-%m-%d %H:%M:%S]", time.gmtime())
   print t, m
 
-def stage1(pipes, status):
+def stage1(pipes, status, clients):
   for c in clients:
     p = pipes[c]
     while 1:
@@ -70,7 +71,7 @@ def count_statuses(status):
     else: cnt[v] += 1
   return cnt
 
-def stage2(pipes, status, command):
+def stage2(pipes, status, command, clients):
   for c in clients:
     if status[c] != 'waiting': continue
     log("%s << %s"%(c, command))
@@ -85,7 +86,7 @@ def stage2(pipes, status, command):
       log("%s >> %s"%(c, l))
     log("%s: returned %d"%(c, p.returncode))
 
-def callout(component, serial):
+def callout(component, serial, clients):
   log("Calling clients...")
   pipes = {}
   status = {}
@@ -96,32 +97,67 @@ def callout(component, serial):
     status[c] = 'in-progress'
 
   log("Stage 1...")
-  stage1(pipes, status)
+  stage1(pipes, status, clients)
   log("Stage 1 done.")
   cnt = count_statuses(status)
 
-  if 'failed' in cnt > 0:
-    log("Some clients failed, aborting...")
-    stage2(pipes, status, 'abort')
+  if 'failed' in cnt and cnt['failed'] >= 2:
+    log("%d clients failed, aborting..."%(cnt['failed'],))
+    stage2(pipes, status, 'abort', clients)
     return False
-  elif 'waiting' in cnt > 0:
+
+  failedmirrorsfile = os.path.join(base, 'master', component + "-failedmirrors")
+  if 'failed' in cnt:
+    log("WARNING: %d clients failed!  Continuing anyway!"%(cnt['failed'],))
+    global had_warnings
+    had_warnings = True
+    f = open(failedmirrorsfile, "w")
+    for c in status:
+      if status[c] == 'failed': f.write(c+"\n")
+    f.close()
+  else:
+    if os.path.exists(failedmirrorsfile): os.unlink(failedmirrorsfile)
+
+  if 'waiting' in cnt:
     log("Committing...")
-    stage2(pipes, status, 'go')
+    stage2(pipes, status, 'go', clients)
     return True
   else:
     log("All clients up to date.")
     return True
 
+def load_component_info(component):
+  with open('/etc/static-components.conf') as f:
+    for line in f:
+      if line.startswith('#'): continue
+      field = line.strip().split()
+      if len(field) < 4: continue
+      if field[1] != component: continue
+      meta = {}
+      meta['master'] = field[0]
+      meta['sourcehost'] = field[2]
+      meta['sourcedir'] = field[3]
+      meta['extrapushhosts'] = set(field[4].split(',')) if len(field) > 4 else set()
+      meta['extraignoreclients'] = set(field[5].split(',')) if len(field) > 5 else set()
+      return meta
+    else:
+      return None
 
 cleanup_dirs = []
 def run_mirror(component):
+  meta = load_component_info(component)
+  if meta is None:
+    log("Component %s not found."%(component,))
+    return False
+  clients = allclients - meta['extraignoreclients']
+
   # setup
   basemaster = os.path.join(base, 'master')
   componentdir = os.path.join(basemaster, component)
   cur = componentdir + '-current-push'
   live = componentdir + '-current-live'
-  tmpdir_new = tempfile.mkdtemp(prefix='live.new-', dir=basemaster); cleanup_dirs.append(tmpdir_new);
-  tmpdir_old = tempfile.mkdtemp(prefix='live.old-', dir=basemaster); cleanup_dirs.append(tmpdir_old);
+  tmpdir_new = tempfile.mkdtemp(prefix=component+'-live.new-', dir=basemaster); cleanup_dirs.append(tmpdir_new);
+  tmpdir_old = tempfile.mkdtemp(prefix=component+'-live.old-', dir=basemaster); cleanup_dirs.append(tmpdir_old);
   os.chmod(tmpdir_new, 0755)
 
   locks = []
@@ -151,7 +187,7 @@ def run_mirror(component):
   log("Renaming %s to %s."%(tmpdir_new, cur))
   os.rename(tmpdir_new, cur)
 
-  proceed = callout(component, serial)
+  proceed = callout(component, serial, clients)
 
   if proceed:
     log("Moving %s aside."%(live,))
@@ -160,7 +196,8 @@ def run_mirror(component):
     os.rename(cur, live)
     log("Cleaning up.")
     shutil.rmtree(tmpdir_old)
-    log("Done.")
+    if had_warnings: log("Done, with warnings.")
+    else: log("Done.")
     ret = True
   else:
     log("Aborted.")