1#! /usr/bin/env python3
3"""
4Check for obsolete binary packages
6@contact: Debian FTP Master <ftpmaster@debian.org>
7@copyright: 2000-2006 James Troup <james@nocrew.org>
8@copyright: 2009 Torsten Werner <twerner@debian.org>
9@copyright: 2015 Niels Thykier <niels@thykier.net>
10@license: GNU General Public License version 2 or later
11"""
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27################################################################################
29# | priviledged positions? What privilege? The honour of working harder
30# | than most people for absolutely no recognition?
31#
32# Manoj Srivastava <srivasta@debian.org> in <87lln8aqfm.fsf@glaurung.internal.golden-gryphon.com>
34################################################################################
36import sqlalchemy.sql as sql
37import sys
38import apt_pkg
39from itertools import chain, product
40from collections import defaultdict
41from collections.abc import Iterable
43from daklib.config import Config
44from daklib.dbconn import *
45from daklib import utils
46from daklib.cruft import *
47from daklib.rm import remove, ReverseDependencyChecker
49################################################################################
52def usage(exit_code=0):
53 print("""Usage: dak auto-decruft
54Automatic removal of common kinds of cruft
56 -h, --help show this help and exit.
57 -n, --dry-run don't do anything, just show what would have been done
58 -s, --suite=SUITE check suite SUITE.
59 --if-newer-version-in OS remove all packages in SUITE with a lower version than
60 in OS (e.g. -s experimental --if-newer-version-in
61 unstable)
62 --if-newer-version-in-rm-msg RMMSG
63 use RMMSG in the removal message (e.g. "NVIU")
64 --decruft-equal-versions use with --if-newer-version-in to also decruft versions
65 that are identical in both suites.
66 """)
67 sys.exit(exit_code)
69################################################################################
72def compute_sourceless_groups(suite_id: int, session):
73 """Find binaries without a source
75 :param suite_id: The id of the suite denoted by suite_name
76 :param session: The database session in use
77 """""
78 rows = query_without_source(suite_id, session)
79 message = '[auto-cruft] no longer built from source, no reverse dependencies'
80 arch = get_architecture('all', session=session)
81 arch_all_id_tuple = tuple([arch.arch_id])
82 arch_all_list = ["all"]
83 for row in rows: 83 ↛ 84line 83 didn't jump to line 84, because the loop on line 83 never started
84 package = row[0]
85 group_info = {
86 "name": "sourceless:%s" % package,
87 "packages": tuple([package]),
88 "architectures": arch_all_list,
89 "architecture_ids": arch_all_id_tuple,
90 "message": message,
91 "removal_request": {
92 package: arch_all_list,
93 },
94 }
95 yield group_info
98def compute_nbs_groups(suite_id: int, suite_name: str, session):
99 """Find binaries no longer built
101 :param suite_id: The id of the suite denoted by suite_name
102 :param suite_name: The name of the suite to remove from
103 :param session: The database session in use
104 """""
105 rows = queryNBS(suite_id, session)
106 arch2ids = dict((a.arch_string, a.arch_id) for a in get_suite_architectures(suite_name))
108 for row in rows: 108 ↛ 109line 108 didn't jump to line 109, because the loop on line 108 never started
109 (pkg_list, arch_list, source, _) = row
110 message = '[auto-cruft] NBS (no longer built by %s, no reverse dependencies)' % source
111 removal_request = dict((pkg, arch_list) for pkg in pkg_list)
112 group_info = {
113 "name": "NBS:%s" % source,
114 "packages": tuple(sorted(pkg_list)),
115 "architectures": sorted(arch_list),
116 "architecture_ids": tuple(arch2ids[arch] for arch in arch_list),
117 "message": message,
118 "removal_request": removal_request,
119 }
120 yield group_info
123def remove_groups(groups, suite_id, suite_name, session):
124 for group in groups:
125 message = group["message"]
126 params = {
127 "architecture_ids": group["architecture_ids"],
128 "packages": group["packages"],
129 "suite_id": suite_id
130 }
131 q = session.execute(sql.text("""
132 SELECT b.package, b.version, a.arch_string, b.id
133 FROM binaries b
134 JOIN bin_associations ba ON b.id = ba.bin
135 JOIN architecture a ON b.architecture = a.id
136 JOIN suite su ON ba.suite = su.id
137 WHERE a.id IN :architecture_ids AND b.package IN :packages AND su.id = :suite_id
138 """), params)
140 remove(session, message, [suite_name], list(q), partial=True, whoami="DAK's auto-decrufter")
143def dedup(*args):
144 seen = set()
145 for iterable in args:
146 for value in iterable:
147 if value not in seen:
148 seen.add(value)
149 yield value
152def merge_group(groupA: dict, groupB: dict) -> dict:
153 """Merges two removal groups into one
155 Note that some values are taken entirely from groupA (e.g. name and message)
157 :param groupA: A removal group
158 :param groupB: Another removal group
159 :return: A merged group
160 """
161 pkg_list = sorted(dedup(groupA["packages"], groupB["packages"]))
162 arch_list = sorted(dedup(groupA["architectures"], groupB["architectures"]))
163 arch_list_id = dedup(groupA["architecture_ids"], groupB["architecture_ids"])
164 removalA = groupA["removal_request"]
165 removalB = groupB["removal_request"]
166 new_removal = {}
167 for pkg in dedup(removalA, removalB):
168 listA = removalA[pkg] if pkg in removalA else []
169 listB = removalB[pkg] if pkg in removalB else []
170 new_removal[pkg] = sorted(dedup(listA, listB))
172 merged_group = {
173 "name": groupA["name"],
174 "packages": tuple(pkg_list),
175 "architectures": arch_list,
176 "architecture_ids": tuple(arch_list_id),
177 "message": groupA["message"],
178 "removal_request": new_removal,
179 }
181 return merged_group
184def auto_decruft_suite(suite_name: str, suite_id: int, session, dryrun: bool, debug: bool):
185 """Run the auto-decrufter on a given suite
187 :param suite_name: The name of the suite to remove from
188 :param suite_id: The id of the suite denoted by suite_name
189 :param session: The database session in use
190 :param dryrun: If True, just print the actions rather than actually doing them
191 :param debug: If True, print some extra information
192 """
193 all_architectures = [a.arch_string for a in get_suite_architectures(suite_name)]
194 pkg_arch2groups = defaultdict(set)
195 group_order = []
196 groups = {}
197 full_removal_request = []
198 group_generator = chain(
199 compute_sourceless_groups(suite_id, session),
200 compute_nbs_groups(suite_id, suite_name, session)
201 )
202 for group in group_generator: 202 ↛ 203line 202 didn't jump to line 203, because the loop on line 202 never started
203 group_name = group["name"]
204 pkgs = group["packages"]
205 affected_archs = group["architectures"]
206 # If we remove an arch:all package, then the breakage can occur on any
207 # of the architectures.
208 if "all" in affected_archs:
209 affected_archs = all_architectures
210 for pkg_arch in product(pkgs, affected_archs):
211 pkg_arch2groups[pkg_arch].add(group_name)
212 if group_name not in groups:
213 groups[group_name] = group
214 group_order.append(group_name)
215 else:
216 # This case usually happens when versions differ between architectures...
217 if debug:
218 print("N: Merging group %s" % (group_name))
219 groups[group_name] = merge_group(groups[group_name], group)
221 for group_name in group_order: 221 ↛ 222line 221 didn't jump to line 222, because the loop on line 221 never started
222 removal_request = groups[group_name]["removal_request"]
223 full_removal_request.extend(removal_request.items())
225 if not groups: 225 ↛ 230line 225 didn't jump to line 230, because the condition on line 225 was never false
226 if debug: 226 ↛ 227line 226 didn't jump to line 227, because the condition on line 226 was never true
227 print("N: Found no candidates")
228 return
230 if debug:
231 print("N: Considering to remove the following packages:")
232 for group_name in sorted(groups):
233 group_info = groups[group_name]
234 pkgs = group_info["packages"]
235 archs = group_info["architectures"]
236 print("N: * %s: %s [%s]" % (group_name, ", ".join(pkgs), " ".join(archs)))
238 if debug:
239 print("N: Compiling ReverseDependencyChecker (RDC) - please hold ...")
240 rdc = ReverseDependencyChecker(session, suite_name)
241 if debug:
242 print("N: Computing initial breakage...")
244 breakage = rdc.check_reverse_depends(full_removal_request)
245 while breakage:
246 by_breakers = [(len(breakage[x]), x, breakage[x]) for x in breakage]
247 by_breakers.sort(reverse=True)
248 if debug:
249 print("N: - Removal would break %s (package, architecture)-pairs" % (len(breakage)))
250 print("N: - full breakage:")
251 for _, breaker, broken in by_breakers:
252 bname = "%s/%s" % breaker
253 broken_str = ", ".join("%s/%s" % b for b in sorted(broken))
254 print("N: * %s => %s" % (bname, broken_str))
256 averted_breakage = set()
258 for _, package_arch, breakage in by_breakers:
259 if breakage <= averted_breakage:
260 # We already avoided this break
261 continue
262 guilty_groups = pkg_arch2groups[package_arch]
264 if not guilty_groups:
265 utils.fubar("Cannot figure what group provided %s" % str(package_arch))
267 if debug:
268 # Only output it, if it truly a new group being discarded
269 # - a group can reach this part multiple times, if it breaks things on
270 # more than one architecture. This being rather common in fact.
271 already_discard = True
272 if any(group_name for group_name in guilty_groups if group_name in groups):
273 already_discard = False
275 if not already_discard:
276 avoided = sorted(breakage - averted_breakage)
277 print("N: - skipping removal of %s (breakage: %s)" % (", ".join(sorted(guilty_groups)), str(avoided)))
279 averted_breakage |= breakage
280 for group_name in guilty_groups:
281 if group_name in groups:
282 del groups[group_name]
284 if not groups:
285 if debug:
286 print("N: Nothing left to remove")
287 return
289 if debug:
290 print("N: Now considering to remove: %s" % str(", ".join(sorted(groups.keys()))))
292 # Rebuild the removal request with the remaining groups and off
293 # we go to (not) break the world once more time
294 full_removal_request = []
295 for group_info in groups.values():
296 full_removal_request.extend(group_info["removal_request"].items())
297 breakage = rdc.check_reverse_depends(full_removal_request)
299 if debug:
300 print("N: Removal looks good")
302 if dryrun:
303 print("Would remove the equivalent of:")
304 for group_name in group_order:
305 if group_name not in groups:
306 continue
307 group_info = groups[group_name]
308 pkgs = group_info["packages"]
309 archs = group_info["architectures"]
310 message = group_info["message"]
312 # Embed the -R just in case someone wants to run it manually later
313 print(' dak rm -m "{message}" -s {suite} -a {architectures} -p -R -b {packages}'.format(
314 message=message, suite=suite_name,
315 architectures=",".join(archs), packages=" ".join(pkgs),
316 ))
318 print()
319 print("Note: The removals may be interdependent. A non-breaking result may require the execution of all")
320 print("of the removals")
321 else:
322 remove_groups(groups.values(), suite_id, suite_name, session)
325def sources2removals(source_list: Iterable[str], suite_id: int, session) -> list:
326 """Compute removals items given a list of names of source packages
328 :param source_list: A list of names of source packages
329 :param suite_id: The id of the suite from which these sources should be removed
330 :param session: The database session in use
331 :return: A list of items to be removed to remove all sources and their binaries from the given suite
332 """
333 to_remove = []
334 params = {"suite_id": suite_id, "sources": tuple(source_list)}
335 q = session.execute(sql.text("""
336 SELECT s.source, s.version, 'source', s.id
337 FROM source s
338 JOIN src_associations sa ON sa.source = s.id
339 WHERE sa.suite = :suite_id AND s.source IN :sources"""), params)
340 to_remove.extend(q)
341 q = session.execute(sql.text("""
342 SELECT b.package, b.version, a.arch_string, b.id
343 FROM binaries b
344 JOIN bin_associations ba ON b.id = ba.bin
345 JOIN architecture a ON b.architecture = a.id
346 JOIN source s ON b.source = s.id
347 WHERE ba.suite = :suite_id AND s.source IN :sources"""), params)
348 to_remove.extend(q)
349 return to_remove
352def decruft_newer_version_in(othersuite: str, suite_name: str, suite_id: int, rm_msg: str, session, dryrun: bool, decruft_equal_versions: bool) -> None:
353 """Compute removals items given a list of names of source packages
355 :param othersuite: The name of the suite to compare with (e.g. "unstable" for "NVIU")
356 :param suite: The name of the suite from which to do removals (e.g. "experimental" for "NVIU")
357 :param suite_id: The id of the suite from which these sources should be removed
358 :param rm_msg: The removal message (or tag, e.g. "NVIU")
359 :param session: The database session in use
360 :param dryrun: If True, just print the actions rather than actually doing them
361 :param decruft_equal_versions: If True, use >= instead of > for finding decruftable packages.
362 """
363 nvi_list = [x[0] for x in newer_version(othersuite, suite_name, session, include_equal=decruft_equal_versions)]
364 if nvi_list:
365 message = "[auto-cruft] %s" % rm_msg
366 if dryrun: 366 ↛ 367line 366 didn't jump to line 367, because the condition on line 366 was never true
367 print(" dak rm -m \"%s\" -s %s %s" % (message, suite_name, " ".join(nvi_list)))
368 else:
369 removals = sources2removals(nvi_list, suite_id, session)
370 remove(session, message, [suite_name], removals, whoami="DAK's auto-decrufter")
372################################################################################
375def main():
376 global Options
377 cnf = Config()
379 Arguments = [('h', "help", "Auto-Decruft::Options::Help"),
380 ('n', "dry-run", "Auto-Decruft::Options::Dry-Run"),
381 ('d', "debug", "Auto-Decruft::Options::Debug"),
382 ('s', "suite", "Auto-Decruft::Options::Suite", "HasArg"),
383 # The "\0" seems to be the only way to disable short options.
384 ("\0", 'if-newer-version-in', "Auto-Decruft::Options::OtherSuite", "HasArg"),
385 ("\0", 'if-newer-version-in-rm-msg', "Auto-Decruft::Options::OtherSuiteRMMsg", "HasArg"),
386 ("\0", 'decruft-equal-versions', "Auto-Decruft::Options::OtherSuiteDecruftEqual")
387 ]
388 for i in ["help", "Dry-Run", "Debug", "OtherSuite", "OtherSuiteRMMsg", "OtherSuiteDecruftEqual"]:
389 key = "Auto-Decruft::Options::%s" % i
390 if key not in cnf: 390 ↛ 388line 390 didn't jump to line 388, because the condition on line 390 was never false
391 cnf[key] = ""
393 cnf["Auto-Decruft::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable")
395 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
397 Options = cnf.subtree("Auto-Decruft::Options")
398 if Options["Help"]:
399 usage()
401 debug = False
402 dryrun = False
403 decruft_equal_versions = False
404 if Options["Dry-Run"]: 404 ↛ 405line 404 didn't jump to line 405, because the condition on line 404 was never true
405 dryrun = True
406 if Options["Debug"]: 406 ↛ 407line 406 didn't jump to line 407, because the condition on line 406 was never true
407 debug = True
408 if Options["OtherSuiteDecruftEqual"]:
409 decruft_equal_versions = True
411 if Options["OtherSuite"] and not Options["OtherSuiteRMMsg"]: 411 ↛ 412line 411 didn't jump to line 412, because the condition on line 411 was never true
412 utils.fubar("--if-newer-version-in requires --if-newer-version-in-rm-msg")
414 session = DBConn().session()
416 suite = get_suite(Options["Suite"].lower(), session)
417 if not suite: 417 ↛ 418line 417 didn't jump to line 418, because the condition on line 417 was never true
418 utils.fubar("Cannot find suite %s" % Options["Suite"].lower())
420 suite_id = suite.suite_id
421 suite_name = suite.suite_name.lower()
423 auto_decruft_suite(suite_name, suite_id, session, dryrun, debug)
425 if Options["OtherSuite"]:
426 osuite = get_suite(Options["OtherSuite"].lower(), session).suite_name
427 decruft_newer_version_in(osuite, suite_name, suite_id, Options["OtherSuiteRMMsg"], session, dryrun, decruft_equal_versions)
429 if not dryrun: 429 ↛ exitline 429 didn't return from function 'main', because the condition on line 429 was never false
430 session.commit()
432################################################################################
435if __name__ == '__main__':
436 main()