Merge remote-tracking branch 'drkranz/changelogs' into merge

* drkranz/changelogs:
  Define changelog export directory for backports
  Automatically remove empty directories
  Generate a listing of the exported files
  Fix number of removed files when rmtree'ing directories

diff --git a/config/debian/dak.conf-backports b/config/debian/dak.conf-backports
index 8a7a9f36e94b3c3c981fa6adb15e72f8f080bf83..40582b09034a23a58eb408448df8728e386b95ac 100644 (file)
--- a/config/debian/dak.conf-backports
+++ b/config/debian/dak.conf-backports
@@ -5,6 +5,7 @@ Dinstall {
 Dir {
   Log "/srv/backports-master.debian.org/log/";
   Lock "/srv/backports-master.debian.org/lock/";
+  Export "/srv/backports-master.debian.org/export/";
 };
 
 Rm {

diff --git a/dak/make_changelog.py b/dak/make_changelog.py
index 50307b158250271769b595e8315421fb03f4efd3..2d52d0693a461dd071b42a30ddff8e4cb0366d6f 100755 (executable)
--- a/dak/make_changelog.py
+++ b/dak/make_changelog.py
@@ -54,6 +54,7 @@ import sys
 import apt_pkg
 from glob import glob
 from shutil import rmtree
+from yaml import safe_dump
 from daklib.dbconn import *
 from daklib import utils
 from daklib.config import Config
@@ -62,6 +63,8 @@ from daklib.regexes import re_no_epoch
 
 ################################################################################
 
+filelist = 'filelist.yaml'
+
 def usage (exit_code=0):
     print """Generate changelog between two suites
 
@@ -240,20 +243,27 @@ def export_files(session, archive, clpool, progress=False):
             print 'make-changelog: unable to unpack %s\n%s' % (p, e)
             stats['errors'] += 1
 
-    for root, dirs, files in os.walk(clpool):
+    for root, dirs, files in os.walk(clpool, topdown=False):
+        files = [f for f in files if f != filelist]
         if len(files):
-            if root.split('/')[-1] not in sources.keys():
-                if os.path.exists(root):
-                    rmtree(root)
-                    stats['removed'] += 1
+            if root != clpool:
+                if root.split('/')[-1] not in sources.keys():
+                    if os.path.exists(root):
+                        stats['removed'] += len(os.listdir(root))
+                        rmtree(root)
             for file in files:
                 if os.path.exists(os.path.join(root, file)):
                     if os.stat(os.path.join(root, file)).st_nlink ==  1:
-                        os.unlink(os.path.join(root, file))
                         stats['removed'] += 1
-
-    for root, dirs, files in os.walk(clpool):
+                        os.unlink(os.path.join(root, file))
+        for dir in dirs:
+            try:
+                os.rmdir(os.path.join(root, dir))
+            except OSError:
+                pass
         stats['files'] += len(files)
+    stats['files'] -= stats['removed']
+
     print 'make-changelog: file exporting finished'
     print '  * New packages unpacked: %d' % stats['unpack']
     print '  * New files created: %d' % stats['created']
@@ -261,6 +271,26 @@ def export_files(session, archive, clpool, progress=False):
     print '  * Unpack errors: %d' % stats['errors']
     print '  * Files available into changelog pool: %d' % stats['files']
 
+def generate_export_filelist(clpool):
+    clfiles = {}
+    for root, dirs, files in os.walk(clpool):
+        for file in [f for f in files if f != filelist]:
+            clpath = os.path.join(root, file).replace(clpool, '').strip('/')
+            source = clpath.split('/')[2]
+            elements = clpath.split('/')[3].split('_')
+            if source not in clfiles:
+                clfiles[source] = {}
+            if elements[0] == source:
+                if elements[1] not in clfiles[source]:
+                    clfiles[source][elements[1]] = []
+                clfiles[source][elements[1]].append(clpath)
+            else:
+                if elements[0] not in clfiles[source]:
+                    clfiles[source][elements[0]] = []
+                clfiles[source][elements[0]].append(clpath)
+    with open(os.path.join(clpool, filelist), 'w+') as fd:
+        safe_dump(clfiles, fd, default_flow_style=False)
+
 def main():
     Cnf = utils.get_conf()
     cnf = Config()
@@ -298,6 +328,7 @@ def main():
             archive = session.query(Archive).filter_by(archive_name=Options['Archive']).one()
             exportpath = os.path.join(Cnf['Dir::Export'], cnf.exportpath)
             export_files(session, archive, exportpath, progress)
+            generate_export_filelist(exportpath)
         else:
             utils.fubar('No changelog export path defined')
     elif binnmu: