Part I of web log shipping

diff --git a/modules/apache2/files/logformat-privacy b/modules/apache2/files/logformat-privacy
index aede26b465f838bfcce010cb24cf05df0f9ac073..3740dbd50012080bf8f3bc8b95e76aa05f47bd1f 100644 (file)
--- a/modules/apache2/files/logformat-privacy
+++ b/modules/apache2/files/logformat-privacy
@@ -4,3 +4,6 @@
 
 LogFormat "0.0.0.0 - %u %{[%d/%b/%Y:00:00:00 %z]}t \"%r\" %>s %b \"%{Referer}i\" \"-\"" privacy
 LogFormat "0.0.0.1 - %u %{[%d/%b/%Y:00:00:00 %z]}t \"%r\" %>s %b \"%{Referer}i\" \"-\"" privacyssl
+<IfModule mod_geoip.c>
+LogFormat "0.0.0.0 - %u %{[%d/%b/%Y:00:00:00 %z]}t \"%r\" %>s %b \"%{Referer}i\" \"-\" %{GEOIP_COUNTRY_CODE}e" privacy+geo
+</IfModule>

diff --git a/modules/debian-org/lib/facter/roleaccounts.rb b/modules/debian-org/lib/facter/roleaccounts.rb
new file mode 100644 (file)
index 0000000..9726919
--- /dev/null
+++ b/modules/debian-org/lib/facter/roleaccounts.rb
@@ -0,0 +1,58 @@
+begin
+    require 'etc'
+
+    Facter.add("staticsync_key") do
+        setcode do
+            key = nil
+            keyfile = '/home/staticsync/.ssh/id_rsa.pub'
+            if FileTest.exist?(keyfile)
+                key = File.open(keyfile).read.chomp
+            end
+            key
+        end
+    end
+
+    Facter.add("staticsync_user_exists") do
+        setcode do
+            result = ''
+            begin
+                if Etc.getpwnam('staticsync')
+                    result = true
+                end
+            rescue ArgumentError
+            end
+            result
+        end
+    end
+
+
+
+    Facter.add("weblogsync_key") do
+        setcode do
+            key = nil
+            keyfile = '/home/weblogsync/.ssh/id_rsa.pub'
+            if FileTest.exist?(keyfile)
+                key = File.open(keyfile).read.chomp
+            end
+            key
+        end
+    end
+
+    Facter.add("weblogsync_user_exists") do
+        setcode do
+            result = ''
+            begin
+                if Etc.getpwnam('weblogsync')
+                    result = true
+                end
+            rescue ArgumentError
+            end
+            result
+        end
+    end
+
+rescue Exception => e
+end
+# vim:set et:
+# vim:set ts=4:
+# vim:set shiftwidth=4:

diff --git a/modules/debian-org/lib/facter/staticsync.rb b/modules/debian-org/lib/facter/staticsync.rb
deleted file mode 100644 (file)
index eccf697..0000000
--- a/modules/debian-org/lib/facter/staticsync.rb
+++ /dev/null
@@ -1,32 +0,0 @@
-begin
-    require 'etc'
-
-    Facter.add("staticsync_key") do
-        setcode do
-            key = nil
-            keyfile = '/home/staticsync/.ssh/id_rsa.pub'
-            if FileTest.exist?(keyfile)
-                key = File.open(keyfile).read.chomp
-            end
-            key
-        end
-    end
-
-    Facter.add("staticsync_user_exists") do
-        setcode do
-            result = ''
-            begin
-                if Etc.getpwnam('staticsync')
-                    result = true
-                end
-            rescue ArgumentError
-            end
-            result
-        end
-    end
-
-rescue Exception => e
-end
-# vim:set et:
-# vim:set ts=4:
-# vim:set shiftwidth=4:

diff --git a/modules/roles/manifests/weblog_provider.pp b/modules/roles/manifests/weblog_provider.pp
new file mode 100644 (file)
index 0000000..ea14b2b
--- /dev/null
+++ b/modules/roles/manifests/weblog_provider.pp
@@ -0,0 +1,12 @@
+class roles::weblog_provider {
+       if ! $::weblogsync_key {
+               exec { 'create-weblogsync-key':
+                       command => '/bin/su - weblogsync -c \'mkdir -p -m 02700 .ssh && ssh-keygen -C "`whoami`@`hostname` (`date +%Y-%m-%d`)" -P "" -f .ssh/id_rsa -q\'',
+                       onlyif  => '/usr/bin/getent passwd weblogsync > /dev/null && ! [ -e /home/weblogsync/.ssh/id_rsa ]'
+               }
+       } else {
+               file { '/etc/cron.d/puppet-weblog-provider':
+                       content => "SHELL=/bin/bash\n\n0 */4 * * * weblogsync sleep $((RANDOM % 1800)); rsync -a --delete /var/log/apache2/*-public-access.log-*gz weblogsync@ravel:-weblogs-incoming-\n",
+               }
+       }
+}

diff --git a/modules/roles/templates/static-mirroring/static-vhost.conf.erb b/modules/roles/templates/static-mirroring/static-vhost.conf.erb
index 7cd99394628e3b0d3ea0aaca913df7c1857d4106..4b8640cee89bfab416d4796191c27e00db9c0a36 100644 (file)
--- a/modules/roles/templates/static-mirroring/static-vhost.conf.erb
+++ b/modules/roles/templates/static-mirroring/static-vhost.conf.erb
@@ -1,7 +1,6 @@
 # puppet maintained
 
-<Macro common-static-vhost $name>
-<Macro common-$name>
+<Macro common-static-base $name>
        <IfModule mod_userdir.c>
                UserDir disabled
        </IfModule>
@@ -19,16 +18,33 @@
        AddOutputFilterByType DEFLATE text/html text/plain text/xml text/css
 </Macro>
 
-<Virtualhost <%= vhost_listen %> >
-       ServerName $name
-       ServerAdmin debian-admin@lists.debian.org
 
-       ErrorLog /var/log/apache2/$name-error.log
-       CustomLog /var/log/apache2/$name-access.log privacy
+<Macro common-static-vhost $name>
+       <Virtualhost <%= vhost_listen %> >
+               ServerName $name
+               ServerAdmin debian-admin@lists.debian.org
 
-       Use common-$name
-</VirtualHost>
+               ErrorLog /var/log/apache2/$name-error.log
+               CustomLog /var/log/apache2/$name-access.log privacy
 
+               Use common-static-base $name
+       </VirtualHost>
 </Macro>
 
+<Macro common-static-vhost-publiclog $name>
+       <Virtualhost <%= vhost_listen %> >
+               ServerName $name
+               ServerAdmin debian-admin@lists.debian.org
+
+               ErrorLog /var/log/apache2/$name-error.log
+               CustomLog /var/log/apache2/$name-access.log privacy
+               <IfModule mod_geoip.c>
+                       CustomLog /var/log/apache2/$name-public-access.log privacy+geo
+               </IfModule>
+
+               Use common-static-base $name
+       </VirtualHost>
+</Macro>
+
+
 # vim:ft=apache:

diff --git a/modules/roles/templates/static-mirroring/vhost/static-vhosts-simple.erb b/modules/roles/templates/static-mirroring/vhost/static-vhosts-simple.erb
index 94802696a99de78792f658b79f382a53711fe848..ea186f32446e66fdb86f1285e18360a9927f2418 100644 (file)
--- a/modules/roles/templates/static-mirroring/vhost/static-vhosts-simple.erb
+++ b/modules/roles/templates/static-mirroring/vhost/static-vhosts-simple.erb
@@ -1,8 +1,8 @@
 # puppet maintained
 
 Use common-static-vhost mozilla.debian.net
-Use common-static-vhost bits.debian.org
 Use common-static-vhost backports.debian.org
+Use common-static-vhost-publiclog bits.debian.org
 
 # www.backports.org is the historical place for the backports
 # website and archive.  It is now a CNAME to backports.debian.org: