Expire old dump files.

diff --git a/expire_dumps b/expire_dumps
new file mode 100755 (executable)
index 0000000..2530b3d
--- /dev/null
+++ b/expire_dumps
@@ -0,0 +1,158 @@
+#!/usr/bin/python
+
+# Copyright (C) 2007 Florian Reitmeir <florian@reitmeir.org>
+# Copyright (C) 2008 Joerg Jaspert <joerg@debian.org>
+
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+# requires: python-dateutil
+
+import glob, os, sys
+import time, datetime
+import re
+from datetime import datetime
+from datetime import timedelta
+from optparse import OptionParser
+
+RULES = [
+    {'days':14,   'interval':0},
+    {'days':31,   'interval':7},
+    {'days':365,  'interval':31},
+    {'days':3650, 'interval':365},
+
+    # keep 14 days, all each day
+    # keep 31 days, 1 each 7th day
+    # keep 365 days, 1 each 31th day
+    # keep 3650 days, 1 each 365th day
+]
+
+TODAY = datetime.today()
+VERBOSE = False
+NOACTION = False
+PRINT = False
+PREFIX = ''
+PATH = ''
+
+def all_files(pattern, search_path, pathsep=os.pathsep):
+    """ Given a search path, yield all files matching the pattern. """
+    for path in search_path.split(pathsep):
+        for match in glob.glob(os.path.join(path, pattern)):
+            yield match
+
+def parse_file_dates_pre(list):
+    out = []
+    # dump_2006.05.02-11:52:01.bz2
+    p = re.compile('^\./dump_pre_([0-9]{4})\.([0-9]{2})\.([0-9]{2})-([0-9]{2}):([0-9]{2}):([0-9]{2})(.bz2)?$')
+    for file in list:
+        m = p.search(file)
+        if m:
+            d = datetime(int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4)), int(m.group(5)), int(m.group(6)))
+            out.append({'name': file, 'date': d})
+    return out
+
+def parse_file_dates_post(list):
+    out = []
+    # dump_2006.05.02-11:52:01.bz2
+    p = re.compile('^\./dump_post_([0-9]{4})\.([0-9]{2})\.([0-9]{2})-([0-9]{2}):([0-9]{2}):([0-9]{2})(.bz2)?$')
+    for file in list:
+        m = p.search(file)
+        if m:
+            d = datetime(int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4)), int(m.group(5)), int(m.group(6)))
+            out.append({'name': file, 'date': d})
+    return out
+
+def prepare_rules(rules):
+    out = []
+    for rule in rules:
+        out.append(
+            {
+            'days':timedelta(days=rule['days']),
+            'interval':timedelta(days=rule['interval'])}
+            )
+    return out
+
+def expire(rules, list):
+    t_rules=prepare_rules(rules)
+    rule = t_rules.pop(0)
+    last = list.pop(0)
+
+    for file in list:
+        if VERBOSE:
+            print "current file to expire: " + file['name']
+            print file['date']
+
+        # check if rule applies
+        if (file['date'] < (TODAY-rule['days'])):
+            if VERBOSE:
+                print "move to next rule"
+            if t_rules:
+                rule = t_rules.pop(0)
+
+        if (last['date'] - file['date']) < rule['interval']:
+            if VERBOSE:
+                print "unlink file:" + file['name']
+            if PRINT:
+                print file['name']
+            if not NOACTION:
+                os.unlink(file['name'])
+        else:
+            last = file
+            if VERBOSE:
+                print "kept file:" + file['name']
+
+
+parser = OptionParser()
+parser.add_option("-d", "--directory", dest="directory",
+                  help="directory name", metavar="Name")
+parser.add_option("-f", "--pattern", dest="pattern",
+                  help="Pattern maybe some glob", metavar="*.backup")
+parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
+                  help="verbose")
+parser.add_option("-n", "--no-action", action="store_true", dest="noaction", default=False,
+                  help="just prints what would be done, this implies verbose")
+parser.add_option("-p", "--print", action="store_true", dest="printfiles", default=False,
+                  help="just print the filenames that should be deleted, this forbids verbose")
+
+(options, args) = parser.parse_args()
+
+if (not options.directory):
+    parser.error("no directory to check given")
+
+if options.noaction:
+    VERBOSE=True
+    NOACTION=True
+
+if options.verbose:
+    VERBOSE=True
+
+if options.printfiles:
+    VERBOSE=False
+    PRINT=True
+
+files = sorted( list(all_files(options.pattern,options.directory)), reverse=True );
+
+if not files:
+    sys.exit(0)
+
+files_dates =  parse_file_dates_pre(files);
+expire(RULES, files_dates)
+
+files_dates =  parse_file_dates_post(files);
+expire(RULES, files_dates)

diff --git a/trigger.daily b/trigger.daily
index 6cf74228e61b8909bccfa9ec661dc9cedb4f32cc..0926de133362ae03a5b6b56624dc1c57f8160ba0 100755 (executable)
--- a/trigger.daily
+++ b/trigger.daily
@@ -222,6 +222,8 @@ echo "merge ended: `date`"
 
 sudo -u postgres /usr/bin/pg_dumpall --cluster 8.4/wanna-build | bzip2 > /org/wanna-build/dumps/dump_post_$(date +%Y.%m.%d-%H:%M:%S).bz2
 
+(cd /org/wanna-build/dumps && /org/wanna-build/expire_dumps -d . -f "dump_*")
+
 #
 # Only update stats if it's been at least 20h since the last time.
 #