Try porting torproject changes: support excluding mirror for a single static component

diff --git a/modules/roles/files/static-mirroring/static-components.conf b/modules/roles/files/static-mirroring/static-components.conf
index 97850364271cbcb978bd862dd2fd1996b9295a66..12b67db433c259291dd1c3d76c18bddecdc42e79 100644 (file)
--- a/modules/roles/files/static-mirroring/static-components.conf
+++ b/modules/roles/files/static-mirroring/static-components.conf
@@ -1,5 +1,5 @@
 # puppetd maintained
-# <master>             <service>                       <source host>                   <directory>                             <extra push hosts, comma separated>
+# <master>             <service>                       <source host>                   <directory>                             <extra push hosts, comma separated>             <hosts to not mirror this component to>
 
 bizet.debian.org       bits.debian.org                 master.debian.org               /srv/bits-master.debian.org/htdocs
 bizet.debian.org       lintian.debian.org              lilburn.debian.org              /srv/lintian.debian.org/www

diff --git a/modules/roles/files/static-mirroring/static-master-run b/modules/roles/files/static-mirroring/static-master-run
index c85e984470ee7f4c436d87c3ca417a8152c7e260..e2a583e9ee4fa5a325a6271e9b3099d9daec7540 100755 (executable)
--- a/modules/roles/files/static-mirroring/static-master-run
+++ b/modules/roles/files/static-mirroring/static-master-run
@@ -13,19 +13,19 @@ base="/srv/static.debian.org"
 serialname = '.serial'
 had_warnings = False
 
-clients = []
+allclients = set()
 with open('/etc/static-clients.conf') as f:
   for line in f:
     line = line.strip()
     if line == "": continue
     if line.startswith('#'): continue
-    clients.append(line)
+    allclients.add(line)
 
 def log(m):
   t = time.strftime("[%Y-%m-%d %H:%M:%S]", time.gmtime())
   print t, m
 
-def stage1(pipes, status):
+def stage1(pipes, status, clients):
   for c in clients:
     p = pipes[c]
     while 1:
@@ -71,7 +71,7 @@ def count_statuses(status):
     else: cnt[v] += 1
   return cnt
 
-def stage2(pipes, status, command):
+def stage2(pipes, status, command, clients):
   for c in clients:
     if status[c] != 'waiting': continue
     log("%s << %s"%(c, command))
@@ -86,7 +86,7 @@ def stage2(pipes, status, command):
       log("%s >> %s"%(c, l))
     log("%s: returned %d"%(c, p.returncode))
 
-def callout(component, serial):
+def callout(component, serial, clients):
   log("Calling clients...")
   pipes = {}
   status = {}
@@ -97,13 +97,13 @@ def callout(component, serial):
     status[c] = 'in-progress'
 
   log("Stage 1...")
-  stage1(pipes, status)
+  stage1(pipes, status, clients)
   log("Stage 1 done.")
   cnt = count_statuses(status)
 
   if 'failed' in cnt and cnt['failed'] >= 2:
     log("%d clients failed, aborting..."%(cnt['failed'],))
-    stage2(pipes, status, 'abort')
+    stage2(pipes, status, 'abort', clients)
     return False
 
   failedmirrorsfile = os.path.join(base, 'master', component + "-failedmirrors")
@@ -120,15 +120,37 @@ def callout(component, serial):
 
   if 'waiting' in cnt:
     log("Committing...")
-    stage2(pipes, status, 'go')
+    stage2(pipes, status, 'go', clients)
     return True
   else:
     log("All clients up to date.")
     return True
 
+def load_component_info(component):
+  with open('/etc/static-components.conf') as f:
+    for line in f:
+      if line.startswith('#'): continue
+      field = line.strip().split()
+      if len(field) < 4: continue
+      if field[1] != component: continue
+      meta = {}
+      meta['master'] = field[0]
+      meta['sourcehost'] = field[2]
+      meta['sourcedir'] = field[3]
+      meta['extrapushhosts'] = set(field[4].split(',')) if len(field) > 4 else set()
+      meta['extraignoreclients'] = set(field[5].split(',')) if len(field) > 5 else set()
+      return meta
+    else:
+      return None
 
 cleanup_dirs = []
 def run_mirror(component):
+  meta = load_component_info(component)
+  if meta is None:
+    log("Component %s not found."%(component,))
+    return False
+  clients = allclients - meta['extraignoreclients']
+
   # setup
   basemaster = os.path.join(base, 'master')
   componentdir = os.path.join(basemaster, component)
@@ -165,7 +187,7 @@ def run_mirror(component):
   log("Renaming %s to %s."%(tmpdir_new, cur))
   os.rename(tmpdir_new, cur)
 
-  proceed = callout(component, serial)
+  proceed = callout(component, serial, clients)
 
   if proceed:
     log("Moving %s aside."%(live,))

diff --git a/modules/roles/files/static-mirroring/static-mirror-run-all b/modules/roles/files/static-mirroring/static-mirror-run-all
new file mode 100755 (executable)
index 0000000..11ec288
--- /dev/null
+++ b/modules/roles/files/static-mirroring/static-mirror-run-all
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# initiate a mirror run for all components on this host.
+
+# Copyright (c) 2012, 2015 Peter Palfrader
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+set -e
+set -u
+
+awk -v host="$(hostname -f)" '
+  !/^ *(#|$)/ {
+    split($6,ignorehosts,",")
+    for (i in ignorehosts) {
+      if (host == ignorehosts[i]) {
+        next
+      }
+    }
+    print $1, $2
+  }' /etc/static-components.conf |
+  while read master component ; do
+    static-mirror-run --one-stage "/srv/static.debian.org/mirrors/$component" "$master:$component/-live-"
+  done

diff --git a/modules/roles/manifests/static_mirror.pp b/modules/roles/manifests/static_mirror.pp
index 3a614a93ea02cf4114af619a63122fe4884567da..dd76b1cda36daa90360152d9c5e8f51b407f9eee 100644 (file)
--- a/modules/roles/manifests/static_mirror.pp
+++ b/modules/roles/manifests/static_mirror.pp
@@ -15,6 +15,11 @@ class roles::static_mirror {
                mode   => '0555',
        }
 
+       file { '/usr/local/bin/static-mirror-run-all':
+               source => 'puppet:///modules/roles/static-mirroring/static-mirror-run-all',
+               mode   => '0555',
+       }
+
        file { '/srv/static.debian.org':
                ensure => directory,
                owner  => staticsync,
@@ -23,7 +28,7 @@ class roles::static_mirror {
        }
 
        file { '/etc/cron.d/puppet-static-mirror':
-                       content => "PATH=/usr/local/bin:/usr/bin:/bin\n@reboot staticsync sleep 60; awk '!/^ *(#|$)/ {print \$1, \$2}' /etc/static-components.conf | while read master component; do static-mirror-run --one-stage /srv/static.debian.org/mirrors/\$component \"\$master:\$component/-live-\" > /dev/null; done\n",
+                       content => "MAILTO=root\nPATH=/usr/local/bin:/usr/bin:/bin\n@reboot staticsync sleep 60; chronic static-mirror-run-all\n",
        }
 
        $vhost_listen = $::hostname ? {