#!/usr/bin/python
## vim:set et ts=2 sw=2 ai:
-# homedir_reminder.py - Reminds users about sizable homedirs.
+# Send email reminders to users having sizable homedirs.
##
-# Copyright (c) 2013 Philipp Kern <phil@philkern.de>
-# Copyright (c) 2013 Peter Palfrader <peter@palfrader.org>
+# Copyright (c) 2013 Philipp Kern <pkern@debian.org>
+# Copyright (c) 2013, 2014 Peter Palfrader <peter@palfrader.org>
+# Copyright (c) 2013 Luca Filipozzi <lfilipoz@debian.org>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
-from __future__ import print_function
-
from collections import defaultdict
+from dsa_mq.connection import Connection
+from dsa_mq.config import Config
import email
import email.mime.text
import glob
import logging
+from optparse import OptionParser
import os.path
import platform
import pwd
import struct
import time
import StringIO
-import pwd
# avoid base64 encoding for utf-8
email.charset.add_charset('utf-8', email.charset.SHORTEST, email.charset.QP)
+parser = OptionParser()
+parser.add_option("-D", "--dryrun",
+ action="store_true", default=False,
+ help="Dry run mode")
+
+parser.add_option("-d", "--debug",
+ action="store_true", default=False,
+ help="Enable debug output")
+
+(options, args) = parser.parse_args()
+options.section = 'dsa-homedirs'
+options.config = '/etc/dsa/pubsub.conf'
+config = Config(options)
+mq_conf = {
+ 'rabbit_userid': config.username,
+ 'rabbit_password': config.password,
+ 'rabbit_virtual_host': config.vhost,
+ 'rabbit_hosts': ['pubsub02.debian.org', 'pubsub01.debian.org'],
+ 'use_ssl': False
+}
-SENDMAIL = ['/usr/sbin/sendmail', '-t', '-oi']
-#SENDMAIL = ['/bin/cat']
+if options.dryrun:
+ SENDMAIL_COMMAND = ['/bin/cat']
+ RM_COMMAND = ['/bin/echo', 'Would remove']
+else:
+ SENDMAIL_COMMAND = ['/usr/sbin/sendmail', '-t', '-oi']
+ RM_COMMAND = ['/bin/rm', '-rf']
-REPORT_SIZES = [
- { 'days': 5, 'size': 10240 },
- { 'days': 10, 'size': 1024 },
- { 'days': 30, 'size': 100 },
- { 'days': 60, 'size': 20 },
- { 'days': 150, 'size': 10 }
+CRITERIA = [
+ { 'size': 10240, 'notifyafter': 5, 'deleteafter': 40 },
+ { 'size': 1024, 'notifyafter': 10, 'deleteafter': 50 },
+ { 'size': 100, 'notifyafter': 30, 'deleteafter': 90 },
+ { 'size': 20, 'notifyafter': 90, 'deleteafter': 150 },
+ { 'size': 5, 'deleteafter': 700 }
]
-USER_EXCLUSION_LIST = ['lost+found']
-MAIL_FROM = 'dsa@debian.org'
+EXCLUDED_USERNAMES = ['lost+found', 'debian', 'buildd', 'd-i']
+MAIL_FROM = 'debian-admin (via Cron) <bulk@admin.debian.org>'
MAIL_TO = '{username}@{hostname}.debian.org'
-MAIL_CC = ''
+MAIL_CC = 'debian-admin (bulk sink) <bulk@admin.debian.org>'
+MAIL_REPLYTO = 'debian-admin <dsa@debian.org>'
MAIL_SUBJECT = 'Please clean up ~{username} on {hostname}.debian.org'
-MAIL_TEXT = u"""\
-Hi {name},
+MAIL_MESSAGE = u"""\
+Hi {realname}!
-you last logged into {hostname} {days_ago} days ago, and your home
-directory there is {homedir_size}MB in size.
+Thanks for your porting effort on {hostname}!
-Disk space on porter boxes is often limited. Please respect your fellow
-porters by cleaning up after yourself and deleting schroots and source/build
-trees in your home directory as soon as feasible.
+Please note that, on most porterboxes, /home is quite small, so please
+remove files that you do not need anymore.
-If you currently do not use {hostname}, please keep ~{username} under 10MB.
+For your information, you last logged into {hostname} {days_ago} days
+ago, and your home directory there is {homedir_size} MB in size.
+
+If you currently do not use {hostname}, please keep ~{username} under
+10 MB, if possible.
+
+Please assist us in freeing up space by deleting schroots, also.
Thanks,
-Cron, on behalf of your catherders/admins
+
+Debian System Administration Team via Cron
+
+PS: A reply is not required.
"""
class Error(Exception):
class SendmailError(Error):
pass
-class LastLog(object):
- LASTLOG_STRUCT = '=L32s256s'
+class LastlogTimes(dict):
+ LASTLOG_STRUCT_32 = '=L32s256s'
+ LASTLOG_STRUCT_64 = '=Q32s256s'
- def __init__(self, fname='/var/log/lastlog'):
- record_size = struct.calcsize(self.LASTLOG_STRUCT)
- self.records = {}
- with open(fname, 'r') as fp:
- uid = -1
+ def __init__(self):
+ record_size_32 = struct.calcsize(self.LASTLOG_STRUCT_32)
+ record_size_64 = struct.calcsize(self.LASTLOG_STRUCT_64)
+ lastlog_size = os.path.getsize('/var/log/lastlog')
+ if 0 == (lastlog_size % record_size_32):
+ self.LASTLOG_STRUCT = self.LASTLOG_STRUCT_32
+ record_size = record_size_32
+ elif 0 == (lastlog_size % record_size_64):
+ self.LASTLOG_STRUCT = self.LASTLOG_STRUCT_64
+ record_size = record_size_64
+ else:
+ raise RuntimeError('Unknown architecture, cannot interpret /var/log/lastlog file size (%d)' % lastlog_size)
+ with open('/var/log/lastlog', 'r') as fp:
+ uid = -1 # there is one record per uid in lastlog
for record in iter(lambda: fp.read(record_size), ''):
- uid += 1
- last_login, _, _ = list(struct.unpack(self.LASTLOG_STRUCT, record))
- if last_login == 0:
- continue
+ uid += 1 # so keep incrementing uid for each record read
+ lastlog_time, _, _ = list(struct.unpack(self.LASTLOG_STRUCT, record))
try:
- self.records[pwd.getpwuid(uid).pw_name] = last_login
+ self[pwd.getpwuid(uid).pw_name] = lastlog_time
except KeyError:
+ # this is a normal condition
continue
- def last_login_for_user(self, username):
- return self.records.get(username, 0)
-
-class HomedirReminder(object):
+class HomedirSizes(dict):
def __init__(self):
- self.lastlog = LastLog()
- self.generate_homedir_list()
-
- def parse_utmp(self):
- self.utmp_records = defaultdict(list)
- for wtmpfile in glob.glob('/var/log/wtmp*'):
- for entry in utmp.UtmpRecord(wtmpfile):
- # TODO: Login, does not account for non-idle sessions.
- self.utmp_records[entry.ut_user].append(entry.ut_tv[0])
- for username, timestamps in self.utmp_records.iteritems():
- self.utmp_records[username] = sorted(timestamps)[-1]
-
- def last_login_for_user(self, username):
- return self.lastlog.last_login_for_user(username)
-
- def generate_homedir_list(self):
- self.homedir_sizes = {}
for direntry in glob.glob('/home/*'):
username = os.path.basename(direntry)
+
+ if username in EXCLUDED_USERNAMES:
+ continue
+
try:
pwinfo = pwd.getpwnam(username)
except KeyError:
if os.path.isdir(direntry):
- logging.warning('Directory %s exists on %s but there is no %s user', direntry, platform.node(), username)
+ logging.warning('directory %s exists on %s but there is no %s user', direntry, platform.node(), username)
continue
- homedir = pwinfo.pw_dir
- if username in USER_EXCLUSION_LIST:
+ if pwinfo.pw_dir != direntry:
+ logging.warning('home directory for %s is not %s, but that exists. confused.', username, direntry)
continue
- # Ignore errors from du.
- command = ['/usr/bin/du', '-ms', homedir]
- p = subprocess.Popen(command,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
+
+ command = ['/usr/bin/du', '-ms', pwinfo.pw_dir]
+ p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(stdout, stderr) = p.communicate()
- if p.returncode != 0:
+ if p.returncode != 0: # ignore errors from du
logging.info('%s failed:', ' '.join(command))
logging.info(stderr)
+ continue
try:
- size = int(stdout.split('\t')[0])
+ self[username] = int(stdout.split('\t')[0])
except ValueError:
- logging.error('Could not convert size output from %s: %s',
- ' '.join(command), stdout)
+ logging.error('could not convert size output from %s: %s', ' '.join(command), stdout)
continue
- self.homedir_sizes[username] = size
- def send_mail(self, **kwargs):
- msg = email.mime.text.MIMEText(MAIL_TEXT.format(**kwargs), _charset='UTF-8')
+class HomedirReminder(object):
+ def __init__(self):
+ self.lastlog_times = LastlogTimes()
+ self.homedir_sizes = HomedirSizes()
+
+ def notify(self, **kwargs):
+ msg = email.mime.text.MIMEText(MAIL_MESSAGE.format(**kwargs), _charset='UTF-8')
msg['From'] = MAIL_FROM.format(**kwargs)
msg['To'] = MAIL_TO.format(**kwargs)
- msg['Cc'] = MAIL_CC.format(**kwargs)
+ if MAIL_CC != "":
+ msg['Cc'] = MAIL_CC.format(**kwargs)
+ if MAIL_REPLYTO != "":
+ msg['Reply-To'] = MAIL_REPLYTO.format(**kwargs)
msg['Subject'] = MAIL_SUBJECT.format(**kwargs)
- p = subprocess.Popen(SENDMAIL, stdin=subprocess.PIPE)
+ msg['Precedence'] = "bulk"
+ msg['Auto-Submitted'] = "auto-generated by mail-big-homedirs"
+ p = subprocess.Popen(SENDMAIL_COMMAND, stdin=subprocess.PIPE)
p.communicate(msg.as_string())
logging.debug(msg.as_string())
if p.returncode != 0:
raise SendmailError
+ def remove(self, **kwargs):
+ try:
+ pwinfo = pwd.getpwnam(kwargs.get('username'))
+ except KeyError:
+ return
+
+ command = RM_COMMAND + [pwinfo.pw_dir]
+ p = subprocess.check_call(command)
+
def run(self):
current_time = time.time()
+ conn = None
+ try:
+ data = {}
+ for user in set(self.homedir_sizes.keys()) | \
+ set(self.lastlog_times.keys()):
+ data[user] = {
+ 'homedir': self.homedir_sizes.get(user, 0),
+ 'lastlog': self.lastlog_times.get(user, 0),
+ }
+
+ msg = {
+ 'timestamp': current_time,
+ 'data': data,
+ 'host': platform.node(),
+ }
+ conn = Connection(conf=mq_conf)
+ conn.topic_send(config.topic,
+ msg,
+ exchange_name=config.exchange,
+ timeout=5)
+ except Exception, e:
+ logging.error("Error sending: %s" % e)
+ finally:
+ if conn:
+ conn.close()
+
for username, homedir_size in self.homedir_sizes.iteritems():
- last_login = self.last_login_for_user(username)
- logging.info('user %s: size %dMB, last login: %d', username, homedir_size, last_login)
- days_ago = int( (current_time - last_login) / 3600 / 24 )
+ try:
+ realname = pwd.getpwnam(username).pw_gecos.decode('utf-8').split(',', 1)[0]
+ except:
+ realname = username
+ lastlog_time = self.lastlog_times.get(username, 0)
+ days_ago = int( (current_time - lastlog_time) / 3600 / 24 )
+ kwargs = {
+ 'hostname': platform.node(),
+ 'username': username,
+ 'realname': realname,
+ 'homedir_size': homedir_size,
+ 'days_ago': days_ago
+ }
- reportsize = None
- for e in REPORT_SIZES:
- if days_ago > e['days']: reportsize = e['size']
+ notify = False
+ remove = False
+ for x in CRITERIA:
+ if homedir_size > x['size'] and 'notifyafter' in x and days_ago >= x['notifyafter']:
+ notify = True
+ if homedir_size > x['size'] and 'deleteafter' in x and days_ago >= x['deleteafter']:
+ remove = True
- if reportsize is not None and homedir_size > reportsize:
- logging.warning('Homedir of user %s is %d and did not login for a while', username, homedir_size)
- try:
- name = pwd.getpwnam(username).pw_gecos.decode('utf-8')
- name = name.split(',', 1)[0]
- except:
- name = username
- self.send_mail(hostname=platform.node(),
- username=username,
- name=name,
- homedir_size=homedir_size,
- days_ago=days_ago)
+ if remove:
+ self.remove(**kwargs)
+ elif notify:
+ self.notify(**kwargs)
if __name__ == '__main__':
- logging.basicConfig()
- # DEBUG for debugging, ERROR for production.
- logging.getLogger().setLevel(logging.ERROR)
+ lvl = logging.ERROR
+ if options.debug:
+ lvl = logging.DEBUG
+ logging.basicConfig(level=lvl)
HomedirReminder().run()