from daklib.dbconn import *
from daklib import utils
from daklib.dak_exceptions import *
-from daklib.regexes import re_strip_source_version, re_build_dep_arch, re_bin_only_nmu
+from daklib.regexes import re_strip_source_version, re_bin_only_nmu
import debianbts as bts
################################################################################
Remove PACKAGE(s) from suite(s).
-a, --architecture=ARCH only act on this architecture
- -b, --binary remove binaries only
+ -b, --binary PACKAGE are binary packages to remove
+ -B, --binary-only remove binaries only
-c, --component=COMPONENT act on this component
-C, --carbon-copy=EMAIL send a CC of removal message to EMAIL
-d, --done=BUG# send removal message as closure to bug#
################################################################################
def reverse_depends_check(removals, suite, arches=None, session=None):
- dbsuite = get_suite(suite, session)
- cnf = Config()
-
print "Checking reverse dependencies..."
- dep_problem = 0
- p2c = {}
- all_broken = {}
- if arches:
- all_arches = set(arches)
- else:
- all_arches = set([x.arch_string for x in get_suite_architectures(suite)])
- all_arches -= set(["source", "all"])
- metakey_d = get_or_set_metadatakey("Depends", session)
- metakey_p = get_or_set_metadatakey("Provides", session)
- params = {
- 'suite_id': dbsuite.suite_id,
- 'metakey_d_id': metakey_d.key_id,
- 'metakey_p_id': metakey_p.key_id,
- }
- for architecture in all_arches | set(['all']):
- deps = {}
- sources = {}
- virtual_packages = {}
- params['arch_id'] = get_architecture(architecture, session).arch_id
-
- statement = '''
- SELECT b.id, b.package, s.source, c.name as component,
- (SELECT bmd.value FROM binaries_metadata bmd WHERE bmd.bin_id = b.id AND bmd.key_id = :metakey_d_id) AS depends,
- (SELECT bmp.value FROM binaries_metadata bmp WHERE bmp.bin_id = b.id AND bmp.key_id = :metakey_p_id) AS provides
- FROM binaries b
- JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :suite_id
- JOIN source s ON b.source = s.id
- JOIN files f ON b.file = f.id
- JOIN location l ON f.location = l.id
- JOIN component c ON l.component = c.id
- WHERE b.architecture = :arch_id'''
- query = session.query('id', 'package', 'source', 'component', 'depends', 'provides'). \
- from_statement(statement).params(params)
- for binary_id, package, source, component, depends, provides in query:
- sources[package] = source
- p2c[package] = component
- if depends is not None:
- deps[package] = depends
- # Maintain a counter for each virtual package. If a
- # Provides: exists, set the counter to 0 and count all
- # provides by a package not in the list for removal.
- # If the counter stays 0 at the end, we know that only
- # the to-be-removed packages provided this virtual
- # package.
- if provides is not None:
- for virtual_pkg in provides.split(","):
- virtual_pkg = virtual_pkg.strip()
- if virtual_pkg == package: continue
- if not virtual_packages.has_key(virtual_pkg):
- virtual_packages[virtual_pkg] = 0
- if package not in removals:
- virtual_packages[virtual_pkg] += 1
-
- # If a virtual package is only provided by the to-be-removed
- # packages, treat the virtual package as to-be-removed too.
- for virtual_pkg in virtual_packages.keys():
- if virtual_packages[virtual_pkg] == 0:
- removals.append(virtual_pkg)
-
- # Check binary dependencies (Depends)
- for package in deps.keys():
- if package in removals: continue
- parsed_dep = []
- try:
- parsed_dep += apt_pkg.ParseDepends(deps[package])
- except ValueError as e:
- print "Error for package %s: %s" % (package, e)
- for dep in parsed_dep:
- # Check for partial breakage. If a package has a ORed
- # dependency, there is only a dependency problem if all
- # packages in the ORed depends will be removed.
- unsat = 0
- for dep_package, _, _ in dep:
- if dep_package in removals:
- unsat += 1
- if unsat == len(dep):
- component = p2c[package]
- source = sources[package]
- if component != "main":
- source = "%s/%s" % (source, component)
- all_broken.setdefault(source, {}).setdefault(package, set()).add(architecture)
- dep_problem = 1
-
- if all_broken:
- print "# Broken Depends:"
- for source, bindict in sorted(all_broken.items()):
- lines = []
- for binary, arches in sorted(bindict.items()):
- if arches == all_arches or 'all' in arches:
- lines.append(binary)
- else:
- lines.append('%s [%s]' % (binary, ' '.join(sorted(arches))))
- print '%s: %s' % (source, lines[0])
- for line in lines[1:]:
- print ' ' * (len(source) + 2) + line
- print
-
- # Check source dependencies (Build-Depends and Build-Depends-Indep)
- all_broken.clear()
- metakey_bd = get_or_set_metadatakey("Build-Depends", session)
- metakey_bdi = get_or_set_metadatakey("Build-Depends-Indep", session)
- params = {
- 'suite_id': dbsuite.suite_id,
- 'metakey_ids': (metakey_bd.key_id, metakey_bdi.key_id),
- }
- statement = '''
- SELECT s.id, s.source, string_agg(sm.value, ', ') as build_dep
- FROM source s
- JOIN source_metadata sm ON s.id = sm.src_id
- WHERE s.id in
- (SELECT source FROM src_associations
- WHERE suite = :suite_id)
- AND sm.key_id in :metakey_ids
- GROUP BY s.id, s.source'''
- query = session.query('id', 'source', 'build_dep').from_statement(statement). \
- params(params)
- for source_id, source, build_dep in query:
- if source in removals: continue
- parsed_dep = []
- if build_dep is not None:
- # Remove [arch] information since we want to see breakage on all arches
- build_dep = re_build_dep_arch.sub("", build_dep)
- try:
- parsed_dep += apt_pkg.ParseDepends(build_dep)
- except ValueError as e:
- print "Error for source %s: %s" % (source, e)
- for dep in parsed_dep:
- unsat = 0
- for dep_package, _, _ in dep:
- if dep_package in removals:
- unsat += 1
- if unsat == len(dep):
- component = DBSource.get(source_id, session).get_component_name()
- if component != "main":
- source = "%s/%s" % (source, component)
- all_broken.setdefault(source, set()).add(utils.pp_deps(dep))
- dep_problem = 1
-
- if all_broken:
- print "# Broken Build-Depends:"
- for source, bdeps in sorted(all_broken.items()):
- bdeps = sorted(bdeps)
- print '%s: %s' % (source, bdeps[0])
- for bdep in bdeps[1:]:
- print ' ' * (len(source) + 2) + bdep
- print
-
- if dep_problem:
+ if utils.check_reverse_depends(removals, suite, arches, session):
print "Dependency problem found."
if not Options["No-Action"]:
game_over()
Arguments = [('h',"help","Rm::Options::Help"),
('a',"architecture","Rm::Options::Architecture", "HasArg"),
- ('b',"binary", "Rm::Options::Binary-Only"),
+ ('b',"binary", "Rm::Options::Binary"),
+ ('B',"binary-only", "Rm::Options::Binary-Only"),
('c',"component", "Rm::Options::Component", "HasArg"),
('C',"carbon-copy", "Rm::Options::Carbon-Copy", "HasArg"), # Bugs to Cc
('d',"done","Rm::Options::Done", "HasArg"), # Bugs fixed
('S',"source-only", "Rm::Options::Source-Only"),
]
- for i in [ "architecture", "binary-only", "carbon-copy", "component",
+ for i in [ "architecture", "binary", "binary-only", "carbon-copy", "component",
"done", "help", "no-action", "partial", "rdep-check", "reason",
"source-only", "Do-Close" ]:
if not cnf.has_key("Rm::Options::%s" % (i)):
if not cnf.has_key("Rm::Options::Suite"):
cnf["Rm::Options::Suite"] = "unstable"
- arguments = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
- Options = cnf.SubTree("Rm::Options")
+ arguments = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.subtree("Rm::Options")
if Options["Help"]:
usage()
utils.fubar("need at least one package name as an argument.")
if Options["Architecture"] and Options["Source-Only"]:
utils.fubar("can't use -a/--architecture and -S/--source-only options simultaneously.")
- if Options["Binary-Only"] and Options["Source-Only"]:
- utils.fubar("can't use -b/--binary-only and -S/--source-only options simultaneously.")
+ if ((Options["Binary"] and Options["Source-Only"])
+ or (Options["Binary"] and Options["Binary-Only"])
+ or (Options["Binary-Only"] and Options["Source-Only"])):
+ utils.fubar("Only one of -b/--binary, -B/--binary-only and -S/--source-only can be used.")
if Options.has_key("Carbon-Copy") and not Options.has_key("Done"):
utils.fubar("can't use -C/--carbon-copy without also using -d/--done option.")
if Options["Architecture"] and not Options["Partial"]:
Options["Partial"] = "true"
if Options["Do-Close"] and not Options["Done"]:
utils.fubar("No.")
- if Options["Do-Close"] and Options["Binary-Only"]:
- utils.fubar("No.")
- if Options["Do-Close"] and Options["Source-Only"]:
- utils.fubar("No.")
- if Options["Do-Close"] and Options["Suite"] != 'unstable':
+ if (Options["Do-Close"]
+ and (Options["Binary"] or Options["Binary-Only"] or Options["Source-Only"])):
utils.fubar("No.")
# Force the admin to tell someone if we're not doing a 'dak
else:
utils.fubar("Invalid -C/--carbon-copy argument '%s'; not a bug number, 'package' or email address." % (copy_to))
- if Options["Binary-Only"]:
+ if Options["Binary"]:
field = "b.package"
else:
field = "s.source"
# Additional suite checks
suite_ids_list = []
+ whitelists = []
suites = utils.split_args(Options["Suite"])
suites_list = utils.join_with_commas_and(suites)
if not Options["No-Action"]:
s = get_suite(suite, session=session)
if s is not None:
suite_ids_list.append(s.suite_id)
+ whitelists.append(s.mail_whitelist)
if suite in ("oldstable", "stable"):
print "**WARNING** About to remove from the (old)stable suite!"
print "This should only be done just prior to a (point) release and not at"
if Options["Rdep-Check"] and len(suites) > 1:
utils.fubar("Reverse dependency check on multiple suites is not implemented.")
- print "Working...",
- sys.stdout.flush()
to_remove = []
maintainers = {}
- # We have 3 modes of package selection: binary-only, source-only
- # and source+binary. The first two are trivial and obvious; the
- # latter is a nasty mess, but very nice from a UI perspective so
- # we try to support it.
+ # We have 3 modes of package selection: binary, source-only, binary-only
+ # and source+binary.
# XXX: TODO: This all needs converting to use placeholders or the object
# API. It's an SQL injection dream at the moment
- if Options["Binary-Only"]:
- # Binary-only
- q = session.execute("SELECT b.package, b.version, a.arch_string, b.id, b.maintainer FROM binaries b, bin_associations ba, architecture a, suite su, files f, location l, component c WHERE ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id AND b.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s %s" % (con_packages, con_suites, con_components, con_architectures))
- for i in q.fetchall():
- to_remove.append(i)
+ if Options["Binary"]:
+ # Removal by binary package name
+ q = session.execute("SELECT b.package, b.version, a.arch_string, b.id, b.maintainer FROM binaries b, bin_associations ba, architecture a, suite su, files f, files_archive_map af, component c WHERE ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id AND b.file = f.id AND af.file_id = f.id AND af.archive_id = su.archive_id AND af.component_id = c.id %s %s %s %s" % (con_packages, con_suites, con_components, con_architectures))
+ to_remove.extend(q)
else:
# Source-only
- source_packages = {}
- q = session.execute("SELECT l.path, f.filename, s.source, s.version, 'source', s.id, s.maintainer FROM source s, src_associations sa, suite su, files f, location l, component c WHERE sa.source = s.id AND sa.suite = su.id AND s.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s" % (con_packages, con_suites, con_components))
- for i in q.fetchall():
- source_packages[i[2]] = i[:2]
- to_remove.append(i[2:])
+ if not Options["Binary-Only"]:
+ q = session.execute("SELECT s.source, s.version, 'source', s.id, s.maintainer FROM source s, src_associations sa, suite su, archive, files f, files_archive_map af, component c WHERE sa.source = s.id AND sa.suite = su.id AND archive.id = su.archive_id AND s.file = f.id AND af.file_id = f.id AND af.archive_id = su.archive_id AND af.component_id = c.id %s %s %s" % (con_packages, con_suites, con_components))
+ to_remove.extend(q)
if not Options["Source-Only"]:
# Source + Binary
- binary_packages = {}
- # First get a list of binary package names we suspect are linked to the source
- q = session.execute("SELECT DISTINCT b.package FROM binaries b, source s, src_associations sa, suite su, files f, location l, component c WHERE b.source = s.id AND sa.source = s.id AND sa.suite = su.id AND s.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s" % (con_packages, con_suites, con_components))
- for i in q.fetchall():
- binary_packages[i[0]] = ""
- # Then parse each .dsc that we found earlier to see what binary packages it thinks it produces
- for i in source_packages.keys():
- filename = "/".join(source_packages[i])
- try:
- dsc = utils.parse_changes(filename, dsc_file=1)
- except CantOpenError:
- utils.warn("couldn't open '%s'." % (filename))
- continue
- for package in dsc.get("binary").split(','):
- package = package.strip()
- binary_packages[package] = ""
- # Then for each binary package: find any version in
- # unstable, check the Source: field in the deb matches our
- # source package and if so add it to the list of packages
- # to be removed.
- for package in binary_packages.keys():
- q = session.execute("SELECT l.path, f.filename, b.package, b.version, a.arch_string, b.id, b.maintainer FROM binaries b, bin_associations ba, architecture a, suite su, files f, location l, component c WHERE ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id AND b.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s AND b.package = '%s'" % (con_suites, con_components, con_architectures, package))
- for i in q.fetchall():
- filename = "/".join(i[:2])
- control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(filename)))
- source = control.Find("Source", control.Find("Package"))
- source = re_strip_source_version.sub('', source)
- if source_packages.has_key(source):
- to_remove.append(i[2:])
- print "done."
+ q = session.execute("""
+ SELECT b.package, b.version, a.arch_string, b.id, b.maintainer
+ FROM binaries b
+ JOIN bin_associations ba ON b.id = ba.bin
+ JOIN architecture a ON b.architecture = a.id
+ JOIN suite su ON ba.suite = su.id
+ JOIN archive ON archive.id = su.archive_id
+ JOIN files_archive_map af ON b.file = af.file_id AND af.archive_id = archive.id
+ JOIN component c ON af.component_id = c.id
+ JOIN source s ON b.source = s.id
+ JOIN src_associations sa ON s.id = sa.source AND sa.suite = su.id
+ WHERE TRUE %s %s %s %s""" % (con_packages, con_suites, con_components, con_architectures))
+ to_remove.extend(q)
if not to_remove:
print "Nothing to do."
versions = []
for package in removals:
versions = d[package].keys()
- versions.sort(apt_pkg.VersionCompare)
+ versions.sort(apt_pkg.version_compare)
for version in versions:
d[package][version].sort(utils.arch_compare_sw)
summary += "%10s | %10s | %s\n" % (package, version, ", ".join(d[package][version]))
if Options["Done"]:
Subst_close_rm = Subst_common
bcc = []
- if cnf.Find("Dinstall::Bcc") != "":
+ if cnf.find("Dinstall::Bcc") != "":
bcc.append(cnf["Dinstall::Bcc"])
- if cnf.Find("Rm::Bcc") != "":
+ if cnf.find("Rm::Bcc") != "":
bcc.append(cnf["Rm::Bcc"])
if bcc:
Subst_close_rm["__BCC__"] = "Bcc: " + ", ".join(bcc)
summarymail += "----------------------------------------------\n"
Subst_close_rm["__SUMMARY__"] = summarymail
- whereami = utils.where_am_i()
- Archive = get_archive(whereami, session)
- if Archive is None:
- utils.warn("Cannot find archive %s. Setting blank values for origin" % whereami)
- Subst_close_rm["__MASTER_ARCHIVE__"] = ""
- Subst_close_rm["__PRIMARY_MIRROR__"] = ""
- else:
- Subst_close_rm["__MASTER_ARCHIVE__"] = Archive.origin_server
- Subst_close_rm["__PRIMARY_MIRROR__"] = Archive.primary_mirror
-
for bug in utils.split_args(Options["Done"]):
Subst_close_rm["__BUG_NUMBER__"] = bug
if Options["Do-Close"]:
mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close-with-related")
else:
mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close")
- utils.send_mail(mail_message)
+ utils.send_mail(mail_message, whitelists=whitelists)
# close associated bug reports
if Options["Do-Close"]:
if len(sources) == 1:
source_pkg = source.split("_", 1)[0]
else:
- utils.fubar("Closing bugs for multiple source pakcages is not supported. Do it yourself.")
+ utils.fubar("Closing bugs for multiple source packages is not supported. Do it yourself.")
Subst_close_other["__BUG_NUMBER_ALSO__"] = ""
Subst_close_other["__SOURCE__"] = source_pkg
- other_bugs = bts.get_bugs('src', source_pkg, 'status', 'open')
+ merged_bugs = set()
+ other_bugs = bts.get_bugs('src', source_pkg, 'status', 'open', 'status', 'forwarded')
if other_bugs:
+ for bugno in other_bugs:
+ if bugno not in merged_bugs:
+ for bug in bts.get_status(bugno):
+ for merged in bug.mergedwith:
+ other_bugs.remove(merged)
+ merged_bugs.add(merged)
logfile.write("Also closing bug(s):")
logfile822.write("Also-Bugs:")
for bug in other_bugs:
projects / dak.git / blobdiff