From eb11f44b4492e8350866992443d352be055eb7ba Mon Sep 17 00:00:00 2001 From: Kurt Roeckx Date: Sat, 12 Dec 2009 21:49:38 +0000 Subject: [PATCH] Expire old dump files. --- expire_dumps | 158 ++++++++++++++++++++++++++++++++++++++++++++++++++ trigger.daily | 2 + 2 files changed, 160 insertions(+) create mode 100755 expire_dumps diff --git a/expire_dumps b/expire_dumps new file mode 100755 index 0000000..2530b3d --- /dev/null +++ b/expire_dumps @@ -0,0 +1,158 @@ +#!/usr/bin/python + +# Copyright (C) 2007 Florian Reitmeir +# Copyright (C) 2008 Joerg Jaspert + +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# requires: python-dateutil + +import glob, os, sys +import time, datetime +import re +from datetime import datetime +from datetime import timedelta +from optparse import OptionParser + +RULES = [ + {'days':14, 'interval':0}, + {'days':31, 'interval':7}, + {'days':365, 'interval':31}, + {'days':3650, 'interval':365}, + + # keep 14 days, all each day + # keep 31 days, 1 each 7th day + # keep 365 days, 1 each 31th day + # keep 3650 days, 1 each 365th day +] + +TODAY = datetime.today() +VERBOSE = False +NOACTION = False +PRINT = False +PREFIX = '' +PATH = '' + +def all_files(pattern, search_path, pathsep=os.pathsep): + """ Given a search path, yield all files matching the pattern. """ + for path in search_path.split(pathsep): + for match in glob.glob(os.path.join(path, pattern)): + yield match + +def parse_file_dates_pre(list): + out = [] + # dump_2006.05.02-11:52:01.bz2 + p = re.compile('^\./dump_pre_([0-9]{4})\.([0-9]{2})\.([0-9]{2})-([0-9]{2}):([0-9]{2}):([0-9]{2})(.bz2)?$') + for file in list: + m = p.search(file) + if m: + d = datetime(int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4)), int(m.group(5)), int(m.group(6))) + out.append({'name': file, 'date': d}) + return out + +def parse_file_dates_post(list): + out = [] + # dump_2006.05.02-11:52:01.bz2 + p = re.compile('^\./dump_post_([0-9]{4})\.([0-9]{2})\.([0-9]{2})-([0-9]{2}):([0-9]{2}):([0-9]{2})(.bz2)?$') + for file in list: + m = p.search(file) + if m: + d = datetime(int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4)), int(m.group(5)), int(m.group(6))) + out.append({'name': file, 'date': d}) + return out + +def prepare_rules(rules): + out = [] + for rule in rules: + out.append( + { + 'days':timedelta(days=rule['days']), + 'interval':timedelta(days=rule['interval'])} + ) + return out + +def expire(rules, list): + t_rules=prepare_rules(rules) + rule = t_rules.pop(0) + last = list.pop(0) + + for file in list: + if VERBOSE: + print "current file to expire: " + file['name'] + print file['date'] + + # check if rule applies + if (file['date'] < (TODAY-rule['days'])): + if VERBOSE: + print "move to next rule" + if t_rules: + rule = t_rules.pop(0) + + if (last['date'] - file['date']) < rule['interval']: + if VERBOSE: + print "unlink file:" + file['name'] + if PRINT: + print file['name'] + if not NOACTION: + os.unlink(file['name']) + else: + last = file + if VERBOSE: + print "kept file:" + file['name'] + + +parser = OptionParser() +parser.add_option("-d", "--directory", dest="directory", + help="directory name", metavar="Name") +parser.add_option("-f", "--pattern", dest="pattern", + help="Pattern maybe some glob", metavar="*.backup") +parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, + help="verbose") +parser.add_option("-n", "--no-action", action="store_true", dest="noaction", default=False, + help="just prints what would be done, this implies verbose") +parser.add_option("-p", "--print", action="store_true", dest="printfiles", default=False, + help="just print the filenames that should be deleted, this forbids verbose") + +(options, args) = parser.parse_args() + +if (not options.directory): + parser.error("no directory to check given") + +if options.noaction: + VERBOSE=True + NOACTION=True + +if options.verbose: + VERBOSE=True + +if options.printfiles: + VERBOSE=False + PRINT=True + +files = sorted( list(all_files(options.pattern,options.directory)), reverse=True ); + +if not files: + sys.exit(0) + +files_dates = parse_file_dates_pre(files); +expire(RULES, files_dates) + +files_dates = parse_file_dates_post(files); +expire(RULES, files_dates) diff --git a/trigger.daily b/trigger.daily index 6cf7422..0926de1 100755 --- a/trigger.daily +++ b/trigger.daily @@ -222,6 +222,8 @@ echo "merge ended: `date`" sudo -u postgres /usr/bin/pg_dumpall --cluster 8.4/wanna-build | bzip2 > /org/wanna-build/dumps/dump_post_$(date +%Y.%m.%d-%H:%M:%S).bz2 +(cd /org/wanna-build/dumps && /org/wanna-build/expire_dumps -d . -f "dump_*") + # # Only update stats if it's been at least 20h since the last time. # -- 2.39.2