X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=dak%2Fgenerate_filelist.py;h=faf08bd7a8680e59802d4bdf8c12d4ec0a68a21b;hb=905d361b3e8c03fe4b0ece18f856bbf7a1e2eec5;hp=1a31ed310cd0e3812f32530355c104b30087ed3f;hpb=5e0cbd293ee595359c856ea5d8b2dfc3b84f6d9b;p=dak.git diff --git a/dak/generate_filelist.py b/dak/generate_filelist.py index 1a31ed31..faf08bd7 100755 --- a/dak/generate_filelist.py +++ b/dak/generate_filelist.py @@ -5,6 +5,7 @@ Generate file lists for apt-ftparchive. @contact: Debian FTP Master @copyright: 2009 Torsten Werner +@copyright: 2011 Ansgar Burchardt @license: GNU General Public License version 2 or later """ @@ -37,94 +38,11 @@ Generate file lists for apt-ftparchive. from daklib.dbconn import * from daklib.config import Config -from daklib.threadpool import ThreadPool -from daklib import utils +from daklib import utils, daklog +from daklib.dakmultiprocessing import DakProcessPool, PROC_STATUS_SUCCESS, PROC_STATUS_SIGNALRAISED import apt_pkg, os, stat, sys -def fetch(query, args, session): - return [path + filename for (path, filename) in \ - session.execute(query, args).fetchall()] - -def getSources(suite, component, session, timestamp): - extra_cond = "" - if timestamp: - extra_cond = "AND extract(epoch from sa.created) > %d" % timestamp - query = """ - SELECT l.path, f.filename - FROM source s - JOIN src_associations sa - ON s.id = sa.source AND sa.suite = :suite %s - JOIN files f - ON s.file = f.id - JOIN location l - ON f.location = l.id AND l.component = :component - ORDER BY filename - """ % extra_cond - args = { 'suite': suite.suite_id, - 'component': component.component_id } - return fetch(query, args, session) - -def getBinaries(suite, component, architecture, type, session, timestamp): - extra_cond = "" - if timestamp: - extra_cond = "AND extract(epoch from ba.created) > %d" % timestamp - query = """ -CREATE TEMP TABLE b_candidates ( - source integer, - file integer, - architecture integer); - -INSERT INTO b_candidates (source, file, architecture) - SELECT b.source, b.file, b.architecture - FROM binaries b - JOIN bin_associations ba ON b.id = ba.bin - WHERE b.type = :type AND ba.suite = :suite AND - b.architecture IN (2, :architecture) %s; - -CREATE TEMP TABLE gf_candidates ( - filename text, - path text, - architecture integer, - src integer, - source text); - -INSERT INTO gf_candidates (filename, path, architecture, src, source) - SELECT f.filename, l.path, bc.architecture, bc.source as src, s.source - FROM b_candidates bc - JOIN source s ON bc.source = s.id - JOIN files f ON bc.file = f.id - JOIN location l ON f.location = l.id - WHERE l.component = :component; - -WITH arch_any AS - - (SELECT path, filename FROM gf_candidates - WHERE architecture > 2), - - arch_all_with_any AS - (SELECT path, filename FROM gf_candidates - WHERE architecture = 2 AND - src IN (SELECT src FROM gf_candidates WHERE architecture > 2)), - - arch_all_without_any AS - (SELECT path, filename FROM gf_candidates - WHERE architecture = 2 AND - source NOT IN (SELECT DISTINCT source FROM gf_candidates WHERE architecture > 2)), - - filelist AS - (SELECT * FROM arch_any - UNION - SELECT * FROM arch_all_with_any - UNION - SELECT * FROM arch_all_without_any) - - SELECT * FROM filelist ORDER BY filename - """ % extra_cond - args = { 'suite': suite.suite_id, - 'component': component.component_id, - 'architecture': architecture.arch_id, - 'type': type } - return fetch(query, args, session) +from daklib.lists import getSources, getBinaries, getArchAll def listPath(suite, component, architecture = None, type = None, incremental_mode = False): @@ -147,26 +65,54 @@ def listPath(suite, component, architecture = None, type = None, file.truncate() return (file, timestamp) -def writeSourceList(args): - (suite, component, incremental_mode) = args +def writeSourceList(suite_id, component_id, incremental_mode): + session = DBConn().session() + suite = Suite.get(suite_id, session) + component = Component.get(component_id, session) (file, timestamp) = listPath(suite, component, incremental_mode = incremental_mode) - session = DBConn().session() - for filename in getSources(suite, component, session, timestamp): + + message = "sources list for %s %s" % (suite.suite_name, component.component_name) + + for _, filename in getSources(suite, component, session, timestamp): file.write(filename + '\n') - session.close() + session.rollback() file.close() + return (PROC_STATUS_SUCCESS, message) -def writeBinaryList(args): - (suite, component, architecture, type, incremental_mode) = args +def writeAllList(suite_id, component_id, architecture_id, type, incremental_mode): + session = DBConn().session() + suite = Suite.get(suite_id, session) + component = Component.get(component_id, session) + architecture = Architecture.get(architecture_id, session) (file, timestamp) = listPath(suite, component, architecture, type, incremental_mode) + + message = "all list for %s %s (arch=%s, type=%s)" % (suite.suite_name, component.component_name, architecture.arch_string, type) + + for _, filename in getArchAll(suite, component, architecture, type, + session, timestamp): + file.write(filename + '\n') + session.rollback() + file.close() + return (PROC_STATUS_SUCCESS, message) + +def writeBinaryList(suite_id, component_id, architecture_id, type, incremental_mode): session = DBConn().session() - for filename in getBinaries(suite, component, architecture, type, + suite = Suite.get(suite_id, session) + component = Component.get(component_id, session) + architecture = Architecture.get(architecture_id, session) + (file, timestamp) = listPath(suite, component, architecture, type, + incremental_mode) + + message = "binary list for %s %s (arch=%s, type=%s)" % (suite.suite_name, component.component_name, architecture.arch_string, type) + + for _, filename in getBinaries(suite, component, architecture, type, session, timestamp): file.write(filename + '\n') - session.close() + session.rollback() file.close() + return (PROC_STATUS_SUCCESS, message) def usage(): print """Usage: dak generate_filelist [OPTIONS] @@ -181,64 +127,91 @@ Create filename lists for apt-ftparchive. ARCH, COMPONENT and SUITE can be comma (or space) separated list, e.g. --suite=testing,unstable -Incremental mode appends only newer files to exising lists.""" +Incremental mode appends only newer files to existing lists.""" sys.exit() def main(): cnf = Config() + Logger = daklog.Logger('generate-filelist') Arguments = [('h', "help", "Filelist::Options::Help"), ('s', "suite", "Filelist::Options::Suite", "HasArg"), ('c', "component", "Filelist::Options::Component", "HasArg"), ('a', "architecture", "Filelist::Options::Architecture", "HasArg"), ('i', "incremental", "Filelist::Options::Incremental")] - query_suites = DBConn().session().query(Suite) - suites = [suite.suite_name for suite in query_suites.all()] + session = DBConn().session() + query_suites = session.query(Suite) + suites = [suite.suite_name for suite in query_suites] if not cnf.has_key('Filelist::Options::Suite'): - cnf['Filelist::Options::Suite'] = ','.join(suites) - # we can ask the database for components if 'mixed' is gone + cnf['Filelist::Options::Suite'] = ','.join(suites).encode() + query_components = session.query(Component) + components = \ + [component.component_name for component in query_components] if not cnf.has_key('Filelist::Options::Component'): - cnf['Filelist::Options::Component'] = 'main,contrib,non-free' - query_architectures = DBConn().session().query(Architecture) + cnf['Filelist::Options::Component'] = ','.join(components).encode() + query_architectures = session.query(Architecture) architectures = \ - [architecture.arch_string for architecture in query_architectures.all()] + [architecture.arch_string for architecture in query_architectures] if not cnf.has_key('Filelist::Options::Architecture'): - cnf['Filelist::Options::Architecture'] = ','.join(architectures) + cnf['Filelist::Options::Architecture'] = ','.join(architectures).encode() cnf['Filelist::Options::Help'] = '' cnf['Filelist::Options::Incremental'] = '' - apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv) - Options = cnf.SubTree("Filelist::Options") + apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) + Options = cnf.subtree("Filelist::Options") if Options['Help']: usage() - session = DBConn().session() - suite_arch = session.query(SuiteArchitecture) - threadpool = ThreadPool() - for suite_name in utils.split_args(Options['Suite']): - suite = query_suites.filter_by(suite_name = suite_name).one() - join = suite_arch.filter_by(suite_id = suite.suite_id) - for component_name in utils.split_args(Options['Component']): - component = session.query(Component).\ - filter_by(component_name = component_name).one() - for architecture_name in utils.split_args(Options['Architecture']): - architecture = query_architectures.\ - filter_by(arch_string = architecture_name).one() - try: - join.filter_by(arch_id = architecture.arch_id).one() - if architecture_name == 'source': - threadpool.queueTask(writeSourceList, - (suite, component, Options['Incremental'])) - elif architecture_name != 'all': - threadpool.queueTask(writeBinaryList, - (suite, component, architecture, 'deb', - Options['Incremental'])) - threadpool.queueTask(writeBinaryList, - (suite, component, architecture, 'udeb', - Options['Incremental'])) - except: + pool = DakProcessPool() + query_suites = query_suites. \ + filter(Suite.suite_name.in_(utils.split_args(Options['Suite']))) + query_components = query_components. \ + filter(Component.component_name.in_(utils.split_args(Options['Component']))) + query_architectures = query_architectures. \ + filter(Architecture.arch_string.in_(utils.split_args(Options['Architecture']))) + + def parse_results(message): + # Split out into (code, msg) + code, msg = message + if code == PROC_STATUS_SUCCESS: + Logger.log([msg]) + elif code == PROC_STATUS_SIGNALRAISED: + Logger.log(['E: Subprocess recieved signal ', msg]) + else: + Logger.log(['E: ', msg]) + + for suite in query_suites: + suite_id = suite.suite_id + for component in query_components: + component_id = component.component_id + for architecture in query_architectures: + architecture_id = architecture.arch_id + if architecture not in suite.architectures: pass - threadpool.joinAll() + elif architecture.arch_string == 'source': + pool.apply_async(writeSourceList, + (suite_id, component_id, Options['Incremental']), callback=parse_results) + elif architecture.arch_string == 'all': + pool.apply_async(writeAllList, + (suite_id, component_id, architecture_id, 'deb', + Options['Incremental']), callback=parse_results) + pool.apply_async(writeAllList, + (suite_id, component_id, architecture_id, 'udeb', + Options['Incremental']), callback=parse_results) + else: # arch any + pool.apply_async(writeBinaryList, + (suite_id, component_id, architecture_id, 'deb', + Options['Incremental']), callback=parse_results) + pool.apply_async(writeBinaryList, + (suite_id, component_id, architecture_id, 'udeb', + Options['Incremental']), callback=parse_results) + pool.close() + pool.join() + # this script doesn't change the database session.close() + Logger.close() + + sys.exit(pool.overall_status()) + if __name__ == '__main__': main()