#!/usr/bin/python ## vim:set et ts=2 sw=2 ai: # Send email reminders to users having sizable homedirs. ## # Copyright (c) 2013 Philipp Kern # Copyright (c) 2013, 2014 Peter Palfrader # Copyright (c) 2013 Luca Filipozzi # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. from collections import defaultdict from dsa_mq.connection import Connection from dsa_mq.config import Config import email import email.mime.text import glob import logging from optparse import OptionParser import os.path import platform import pwd import subprocess import struct import time import StringIO # avoid base64 encoding for utf-8 email.charset.add_charset('utf-8', email.charset.SHORTEST, email.charset.QP) parser = OptionParser() parser.add_option("-D", "--dryrun", action="store_true", default=False, help="Dry run mode") parser.add_option("-d", "--debug", action="store_true", default=False, help="Enable debug output") (options, args) = parser.parse_args() options.section = 'dsa-homedirs' options.config = '/etc/dsa/pubsub.conf' config = Config(options) mq_conf = { 'rabbit_userid': config.username, 'rabbit_password': config.password, 'rabbit_virtual_host': config.vhost, 'rabbit_hosts': ['pubsub02.debian.org', 'pubsub01.debian.org'], 'use_ssl': False } if options.dryrun: SENDMAIL_COMMAND = ['/bin/cat'] RM_COMMAND = ['/bin/echo', 'Would remove'] else: SENDMAIL_COMMAND = ['/usr/sbin/sendmail', '-t', '-oi'] RM_COMMAND = ['/bin/rm', '-rf'] CRITERIA = [ { 'size': 10240, 'notifyafter': 5, 'deleteafter': 40 }, { 'size': 1024, 'notifyafter': 10, 'deleteafter': 50 }, { 'size': 100, 'notifyafter': 30, 'deleteafter': 90 }, { 'size': 20, 'notifyafter': 90, 'deleteafter': 150 }, { 'size': 5, 'deleteafter': 700 } ] EXCLUDED_USERNAMES = ['lost+found', 'debian', 'buildd', 'd-i'] MAIL_FROM = 'debian-admin (via Cron) ' MAIL_TO = '{username}@{hostname}.debian.org' MAIL_CC = 'debian-admin (bulk sink) ' MAIL_REPLYTO = 'debian-admin ' MAIL_SUBJECT = 'Please clean up ~{username} on {hostname}.debian.org' MAIL_MESSAGE = u"""\ Hi {realname}! Thanks for your porting effort on {hostname}! Please note that, on most porterboxes, /home is quite small, so please remove files that you do not need anymore. For your information, you last logged into {hostname} {days_ago} days ago, and your home directory there is {homedir_size} MB in size. If you currently do not use {hostname}, please keep ~{username} under 10 MB, if possible. Please assist us in freeing up space by deleting schroots, also. Thanks, Debian System Administration Team via Cron PS: A reply is not required. """ class Error(Exception): pass class SendmailError(Error): pass class LastlogTimes(dict): LASTLOG_STRUCT_32 = '=L32s256s' LASTLOG_STRUCT_64 = '=Q32s256s' def __init__(self): record_size_32 = struct.calcsize(self.LASTLOG_STRUCT_32) record_size_64 = struct.calcsize(self.LASTLOG_STRUCT_64) lastlog_size = os.path.getsize('/var/log/lastlog') if 0 == (lastlog_size % record_size_32): self.LASTLOG_STRUCT = self.LASTLOG_STRUCT_32 record_size = record_size_32 elif 0 == (lastlog_size % record_size_64): self.LASTLOG_STRUCT = self.LASTLOG_STRUCT_64 record_size = record_size_64 else: raise RuntimeError('Unknown architecture, cannot interpret /var/log/lastlog file size (%d)' % lastlog_size) with open('/var/log/lastlog', 'r') as fp: uid = -1 # there is one record per uid in lastlog for record in iter(lambda: fp.read(record_size), ''): uid += 1 # so keep incrementing uid for each record read lastlog_time, _, _ = list(struct.unpack(self.LASTLOG_STRUCT, record)) try: self[pwd.getpwuid(uid).pw_name] = lastlog_time except KeyError: # this is a normal condition continue class HomedirSizes(dict): def __init__(self): for direntry in glob.glob('/home/*'): username = os.path.basename(direntry) if username in EXCLUDED_USERNAMES: continue try: pwinfo = pwd.getpwnam(username) except KeyError: if os.path.isdir(direntry): logging.warning('directory %s exists on %s but there is no %s user', direntry, platform.node(), username) continue if pwinfo.pw_dir != direntry: logging.warning('home directory for %s is not %s, but that exists. confused.', username, direntry) continue command = ['/usr/bin/du', '-ms', pwinfo.pw_dir] p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = p.communicate() if p.returncode != 0: # ignore errors from du logging.info('%s failed:', ' '.join(command)) logging.info(stderr) continue try: self[username] = int(stdout.split('\t')[0]) except ValueError: logging.error('could not convert size output from %s: %s', ' '.join(command), stdout) continue class HomedirReminder(object): def __init__(self): self.lastlog_times = LastlogTimes() self.homedir_sizes = HomedirSizes() def notify(self, **kwargs): msg = email.mime.text.MIMEText(MAIL_MESSAGE.format(**kwargs), _charset='UTF-8') msg['From'] = MAIL_FROM.format(**kwargs) msg['To'] = MAIL_TO.format(**kwargs) if MAIL_CC != "": msg['Cc'] = MAIL_CC.format(**kwargs) if MAIL_REPLYTO != "": msg['Reply-To'] = MAIL_REPLYTO.format(**kwargs) msg['Subject'] = MAIL_SUBJECT.format(**kwargs) msg['Precedence'] = "bulk" msg['Auto-Submitted'] = "auto-generated by mail-big-homedirs" p = subprocess.Popen(SENDMAIL_COMMAND, stdin=subprocess.PIPE) p.communicate(msg.as_string()) logging.debug(msg.as_string()) if p.returncode != 0: raise SendmailError def remove(self, **kwargs): try: pwinfo = pwd.getpwnam(kwargs.get('username')) except KeyError: return command = RM_COMMAND + [pwinfo.pw_dir] p = subprocess.check_call(command) def run(self): current_time = time.time() conn = None try: data = {} for user in set(self.homedir_sizes.keys()) | \ set(self.lastlog_times.keys()): data[user] = { 'homedir': self.homedir_sizes.get(user, 0), 'lastlog': self.lastlog_times.get(user, 0), } msg = { 'timestamp': current_time, 'data': data, 'host': platform.node(), } conn = Connection(conf=mq_conf) conn.topic_send(config.topic, msg, exchange_name=config.exchange, timeout=5) except Exception, e: logging.error("Error sending: %s" % e) finally: if conn: conn.close() for username, homedir_size in self.homedir_sizes.iteritems(): try: realname = pwd.getpwnam(username).pw_gecos.decode('utf-8').split(',', 1)[0] except: realname = username lastlog_time = self.lastlog_times.get(username, 0) days_ago = int( (current_time - lastlog_time) / 3600 / 24 ) kwargs = { 'hostname': platform.node(), 'username': username, 'realname': realname, 'homedir_size': homedir_size, 'days_ago': days_ago } notify = False remove = False for x in CRITERIA: if homedir_size > x['size'] and 'notifyafter' in x and days_ago >= x['notifyafter']: notify = True if homedir_size > x['size'] and 'deleteafter' in x and days_ago >= x['deleteafter']: remove = True if remove: self.remove(**kwargs) elif notify: self.notify(**kwargs) if __name__ == '__main__': lvl = logging.ERROR if options.debug: lvl = logging.DEBUG logging.basicConfig(level=lvl) HomedirReminder().run()