"""General purpose package removal code for ftpmaster
@contact: Debian FTP Master <ftpmaster@debian.org>
@copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
@copyright: 2010 Alexander Reichle-Schmehl <tolimar@debian.org>
@copyright: 2015 Niels Thykier <niels@thykier.net>
@license: GNU General Public License version 2 or later
"""
# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
# Copyright (C) 2010 Alexander Reichle-Schmehl <tolimar@debian.org>
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
################################################################################
# From: Andrew Morton <akpm@osdl.org>
# Subject: 2.6.6-mm5
# To: linux-kernel@vger.kernel.org
# Date: Sat, 22 May 2004 01:36:36 -0700
# X-Mailer: Sylpheed version 0.9.7 (GTK+ 1.2.10; i386-redhat-linux-gnu)
#
# [...]
#
# Although this feature has been around for a while it is new code, and the
# usual cautions apply. If it munches all your files please tell Jens and
# he'll type them in again for you.
################################################################################
import apt_pkg
import fcntl
import functools
import sqlalchemy.sql as sql
import email.utils
from re import sub
from collections import defaultdict
from .regexes import re_build_dep_arch
from typing import Optional, Union
from daklib.dbconn import *
from daklib import utils
from daklib.regexes import re_bin_only_nmu
import debianbts as bts
################################################################################
[docs]class ReverseDependencyChecker:
"""A bulk tester for reverse dependency checks
This class is similar to the check_reverse_depends method from "utils". However,
it is primarily focused on facilitating bulk testing of reverse dependencies.
It caches the state of the suite and then uses that as basis for answering queries.
This saves a significant amount of time if multiple reverse dependency checks are
required.
"""
def __init__(self, session, suite: str):
"""Creates a new ReverseDependencyChecker instance
This will spend a significant amount of time caching data.
:param session: The database session in use
:param suite: The name of the suite that is used as basis for removal tests.
"""
self._session = session
dbsuite = get_suite(suite, session)
suite_archs2id = dict((x.arch_string, x.arch_id) for x in get_suite_architectures(suite))
package_dependencies, arch_providers_of, arch_provided_by = self._load_package_information(session,
dbsuite.suite_id,
suite_archs2id)
self._package_dependencies = package_dependencies
self._arch_providers_of = arch_providers_of
self._arch_provided_by = arch_provided_by
self._archs_in_suite = set(suite_archs2id)
[docs] def check_reverse_depends(self, removal_requests: Union[dict, list[tuple]]) -> dict:
"""Bulk check reverse dependencies
Example:
removal_request = {
"eclipse-rcp": None, # means ALL architectures (incl. source)
"eclipse": None, # means ALL architectures (incl. source)
"lintian": ["source", "all"], # Only these two "architectures".
}
obj.check_reverse_depends(removal_request)
:param removal_requests: A dictionary mapping a package name to a list of architectures. The list of
architectures decides from which the package will be removed - if the list is empty the package will
be removed on ALL architectures in the suite (including "source").
:return: A mapping of "removed package" (as a "(pkg, arch)"-tuple) to a set of broken
broken packages (also as "(pkg, arch)"-tuple). Note that the architecture values
in these tuples /can/ be "source" to reflect a breakage in build-dependencies.
"""
archs_in_suite = self._archs_in_suite
removals_by_arch = defaultdict(set)
affected_virtual_by_arch = defaultdict(set)
package_dependencies = self._package_dependencies
arch_providers_of = self._arch_providers_of
arch_provided_by = self._arch_provided_by
arch_provides2removal = defaultdict(lambda: defaultdict(set))
dep_problems = defaultdict(set)
src_deps = package_dependencies['source']
src_removals = set()
arch_all_removals = set()
if isinstance(removal_requests, dict):
removal_requests = removal_requests.items()
for pkg, arch_list in removal_requests:
if not arch_list:
arch_list = archs_in_suite
for arch in arch_list:
if arch == 'source':
src_removals.add(pkg)
continue
if arch == 'all':
arch_all_removals.add(pkg)
continue
removals_by_arch[arch].add(pkg)
if pkg in arch_providers_of[arch]:
affected_virtual_by_arch[arch].add(pkg)
if arch_all_removals:
for arch in archs_in_suite:
if arch in ('all', 'source'):
continue
removals_by_arch[arch].update(arch_all_removals)
for pkg in arch_all_removals:
if pkg in arch_providers_of[arch]:
affected_virtual_by_arch[arch].add(pkg)
if not removals_by_arch:
# Nothing to remove => no problems
return dep_problems
for arch, removed_providers in affected_virtual_by_arch.items():
provides2removal = arch_provides2removal[arch]
removals = removals_by_arch[arch]
for virtual_pkg, virtual_providers in arch_provided_by[arch].items():
v = virtual_providers & removed_providers
if len(v) == len(virtual_providers):
# We removed all the providers of virtual_pkg
removals.add(virtual_pkg)
# Pick one to take the blame for the removal
# - we sort for determinism, optimally we would prefer to blame the same package
# to minimise the number of blamed packages.
provides2removal[virtual_pkg] = sorted(v)[0]
for arch, removals in removals_by_arch.items():
deps = package_dependencies[arch]
provides2removal = arch_provides2removal[arch]
# Check binary dependencies (Depends)
for package, dependencies in deps.items():
if package in removals:
continue
for clause in dependencies:
if not (clause <= removals):
# Something probably still satisfies this relation
continue
# whoops, we seemed to have removed all packages that could possibly satisfy
# this relation. Lets blame something for it
for dep_package in clause:
removal = dep_package
if dep_package in provides2removal:
removal = provides2removal[dep_package]
dep_problems[(removal, arch)].add((package, arch))
for source, build_dependencies in src_deps.items():
if source in src_removals:
continue
for clause in build_dependencies:
if not (clause <= removals):
# Something probably still satisfies this relation
continue
# whoops, we seemed to have removed all packages that could possibly satisfy
# this relation. Lets blame something for it
for dep_package in clause:
removal = dep_package
if dep_package in provides2removal:
removal = provides2removal[dep_package]
dep_problems[(removal, arch)].add((source, 'source'))
return dep_problems
[docs]def remove(session, reason: str, suites: list, removals: list,
whoami: Optional[str] = None, partial: bool = False,
components: Optional[list] = None, done_bugs: Optional[list] = None,
date: Optional[str] = None, carbon_copy: Optional[list[str]] = None,
close_related_bugs: bool = False) -> None:
"""Batch remove a number of packages
Verify that the files listed in the Files field of the .dsc are
those expected given the announced Format.
:param session: The database session in use
:param reason: The reason for the removal (e.g. "[auto-cruft] NBS (no longer built by <source>)")
:param suites: A list of the suite names in which the removal should occur
:param removals: A list of the removals. Each element should be a tuple (or list) of at least the following
for 4 items from the database (in order): package, version, architecture, (database) id.
For source packages, the "architecture" should be set to "source".
:param whoami: The person (or entity) doing the removal. Defaults to utils.whoami()
:param partial: Whether the removal is "partial" (e.g. architecture specific).
:param components: List of components involved in a partial removal. Can be an empty list to not restrict the
removal to any components.
:param done_bugs: A list of bugs to be closed when doing this removal.
:param date: The date of the removal. Defaults to `date -R`
:param carbon_copy: A list of mail addresses to CC when doing removals. NB: all items are taken "as-is" unlike
"dak rm".
:param close_related_bugs: Whether bugs related to the package being removed should be closed as well. NB: Not implemented
for more than one suite.
"""
# Generate the summary of what's to be removed
d = {}
summary = ""
affected_sources = set()
sources = []
binaries = []
whitelists = []
versions = []
newest_source = ''
suite_ids_list = []
suites_list = utils.join_with_commas_and(suites)
cnf = utils.get_conf()
con_components = ''
#######################################################################################################
if not reason:
raise ValueError("Empty removal reason not permitted")
reason = reason.strip()
if not removals:
raise ValueError("Nothing to remove!?")
if not suites:
raise ValueError("Removals without a suite!?")
if whoami is None:
whoami = utils.whoami()
if date is None:
date = email.utils.formatdate()
if partial and components:
component_ids_list = []
for componentname in components:
component = get_component(componentname, session=session)
if component is None:
raise ValueError("component '%s' not recognised." % componentname)
else:
component_ids_list.append(component.component_id)
if component_ids_list:
con_components = "AND component IN (%s)" % ", ".join([str(i) for i in component_ids_list])
for i in removals:
package = i[0]
version = i[1]
architecture = i[2]
if package not in d:
d[package] = {}
if version not in d[package]:
d[package][version] = []
if architecture not in d[package][version]:
d[package][version].append(architecture)
for package in sorted(d):
versions = sorted(d[package], key=functools.cmp_to_key(apt_pkg.version_compare))
for version in versions:
d[package][version].sort(key=utils.ArchKey)
summary += "%10s | %10s | %s\n" % (package, version, ", ".join(d[package][version]))
if apt_pkg.version_compare(version, newest_source) > 0:
newest_source = version
for package in summary.split("\n"):
for row in package.split("\n"):
element = row.split("|")
if len(element) == 3:
if element[2].find("source") > 0:
sources.append("%s_%s" % tuple(elem.strip(" ") for elem in element[:2]))
element[2] = sub(r"source\s?,?", "", element[2]).strip(" ")
if element[2]:
binaries.append("%s_%s [%s]" % tuple(elem.strip(" ") for elem in element))
dsc_type_id = get_override_type('dsc', session).overridetype_id
deb_type_id = get_override_type('deb', session).overridetype_id
for suite in suites:
s = get_suite(suite, session=session)
if s is not None:
suite_ids_list.append(s.suite_id)
whitelists.append(s.mail_whitelist)
#######################################################################################################
log_filename = cnf["Rm::LogFile"]
log822_filename = cnf["Rm::LogFile822"]
with open(log_filename, "a") as logfile, open(log822_filename, "a") as logfile822:
fcntl.lockf(logfile, fcntl.LOCK_EX)
fcntl.lockf(logfile822, fcntl.LOCK_EX)
logfile.write("=========================================================================\n")
logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami))
logfile.write("Removed the following packages from %s:\n\n%s" % (suites_list, summary))
if done_bugs:
logfile.write("Closed bugs: %s\n" % (", ".join(done_bugs)))
logfile.write("\n------------------- Reason -------------------\n%s\n" % reason)
logfile.write("----------------------------------------------\n")
logfile822.write("Date: %s\n" % date)
logfile822.write("Ftpmaster: %s\n" % whoami)
logfile822.write("Suite: %s\n" % suites_list)
if sources:
logfile822.write("Sources:\n")
for source in sources:
logfile822.write(" %s\n" % source)
if binaries:
logfile822.write("Binaries:\n")
for binary in binaries:
logfile822.write(" %s\n" % binary)
logfile822.write("Reason: %s\n" % reason.replace('\n', '\n '))
if done_bugs:
logfile822.write("Bug: %s\n" % (", ".join(done_bugs)))
for i in removals:
package = i[0]
architecture = i[2]
package_id = i[3]
for suite_id in suite_ids_list:
if architecture == "source":
q = session.execute("DELETE FROM src_associations sa USING source s WHERE sa.source = s.id AND sa.source = :packageid AND sa.suite = :suiteid RETURNING s.source",
{'packageid': package_id, 'suiteid': suite_id})
affected_sources.add(q.scalar())
else:
q = session.execute("DELETE FROM bin_associations ba USING binaries b, source s WHERE ba.bin = b.id AND b.source = s.id AND ba.bin = :packageid AND ba.suite = :suiteid RETURNING s.source",
{'packageid': package_id, 'suiteid': suite_id})
affected_sources.add(q.scalar())
# Delete from the override file
if not partial:
if architecture == "source":
type_id = dsc_type_id
else:
type_id = deb_type_id
# TODO: Fix this properly to remove the remaining non-bind argument
session.execute("DELETE FROM override WHERE package = :package AND type = :typeid AND suite = :suiteid %s" % (con_components), {'package': package, 'typeid': type_id, 'suiteid': suite_id})
session.commit()
# ### REMOVAL COMPLETE - send mail time ### #
# If we don't have a Bug server configured, we're done
if "Dinstall::BugServer" not in cnf:
if done_bugs or close_related_bugs:
utils.warn("Cannot send mail to BugServer as Dinstall::BugServer is not configured")
logfile.write("=========================================================================\n")
logfile822.write("\n")
return
# read common subst variables for all bug closure mails
Subst_common = {}
Subst_common["__RM_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
Subst_common["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
Subst_common["__CC__"] = "X-DAK: dak rm"
if carbon_copy:
Subst_common["__CC__"] += "\nCc: " + ", ".join(carbon_copy)
Subst_common["__SOURCES__"] = ", ".join(sorted(affected_sources))
Subst_common["__SUITE_LIST__"] = suites_list
Subst_common["__SUITES__"] = ", ".join(sorted(suites))
Subst_common["__SUBJECT__"] = "Removed package(s) from %s" % (suites_list)
Subst_common["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
Subst_common["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
Subst_common["__WHOAMI__"] = whoami
# Send the bug closing messages
if done_bugs:
Subst_close_rm = Subst_common
bcc = []
if cnf.find("Dinstall::Bcc") != "":
bcc.append(cnf["Dinstall::Bcc"])
if cnf.find("Rm::Bcc") != "":
bcc.append(cnf["Rm::Bcc"])
if bcc:
Subst_close_rm["__BCC__"] = "Bcc: " + ", ".join(bcc)
else:
Subst_close_rm["__BCC__"] = "X-Filler: 42"
summarymail = "%s\n------------------- Reason -------------------\n%s\n" % (summary, reason)
summarymail += "----------------------------------------------\n"
Subst_close_rm["__SUMMARY__"] = summarymail
for bug in done_bugs:
Subst_close_rm["__BUG_NUMBER__"] = bug
if close_related_bugs:
mail_message = utils.TemplateSubst(Subst_close_rm, cnf["Dir::Templates"] + "/rm.bug-close-with-related")
else:
mail_message = utils.TemplateSubst(Subst_close_rm, cnf["Dir::Templates"] + "/rm.bug-close")
utils.send_mail(mail_message, whitelists=whitelists)
# close associated bug reports
if close_related_bugs:
Subst_close_other = Subst_common
bcc = []
wnpp = utils.parse_wnpp_bug_file()
newest_source = re_bin_only_nmu.sub('', newest_source)
if len(set(s.split("_", 1)[0] for s in sources)) == 1:
source_pkg = source.split("_", 1)[0]
else:
logfile.write("=========================================================================\n")
logfile822.write("\n")
raise ValueError("Closing bugs for multiple source packages is not supported. Please do it yourself.")
if newest_source != '':
Subst_close_other["__VERSION__"] = newest_source
else:
logfile.write("=========================================================================\n")
logfile822.write("\n")
raise ValueError("No versions can be found. Close bugs yourself.")
if bcc:
Subst_close_other["__BCC__"] = "Bcc: " + ", ".join(bcc)
else:
Subst_close_other["__BCC__"] = "X-Filler: 42"
# at this point, I just assume, that the first closed bug gives
# some useful information on why the package got removed
Subst_close_other["__BUG_NUMBER__"] = done_bugs[0]
Subst_close_other["__BUG_NUMBER_ALSO__"] = ""
Subst_close_other["__SOURCE__"] = source_pkg
merged_bugs = set()
other_bugs = bts.get_bugs(src=source_pkg, status=('open', 'forwarded'))
if other_bugs:
for bugno in other_bugs:
if bugno not in merged_bugs:
for bug in bts.get_status(bugno):
for merged in bug.mergedwith:
other_bugs.remove(merged)
merged_bugs.add(merged)
logfile.write("Also closing bug(s):")
logfile822.write("Also-Bugs:")
for bug in other_bugs:
Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
logfile.write(" " + str(bug))
logfile822.write(" " + str(bug))
logfile.write("\n")
logfile822.write("\n")
if source_pkg in wnpp:
logfile.write("Also closing WNPP bug(s):")
logfile822.write("Also-WNPP:")
for bug in wnpp[source_pkg]:
# the wnpp-rm file we parse also contains our removal
# bugs, filtering that out
if bug != Subst_close_other["__BUG_NUMBER__"]:
Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
logfile.write(" " + str(bug))
logfile822.write(" " + str(bug))
logfile.write("\n")
logfile822.write("\n")
mail_message = utils.TemplateSubst(Subst_close_other, cnf["Dir::Templates"] + "/rm.bug-close-related")
if Subst_close_other["__BUG_NUMBER_ALSO__"]:
utils.send_mail(mail_message)
logfile.write("=========================================================================\n")
logfile822.write("\n")