Coverage for dak/auto_decruft.py: 37%
211 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-08-03 16:46 +0000
1"""
2Check for obsolete binary packages
4@contact: Debian FTP Master <ftpmaster@debian.org>
5@copyright: 2000-2006 James Troup <james@nocrew.org>
6@copyright: 2009 Torsten Werner <twerner@debian.org>
7@copyright: 2015 Niels Thykier <niels@thykier.net>
8@license: GNU General Public License version 2 or later
9"""
11# This program is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation; either version 2 of the License, or
14# (at your option) any later version.
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
21# You should have received a copy of the GNU General Public License
22# along with this program; if not, write to the Free Software
23# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25################################################################################
27# | priviledged positions? What privilege? The honour of working harder
28# | than most people for absolutely no recognition?
29#
30# Manoj Srivastava <srivasta@debian.org> in <87lln8aqfm.fsf@glaurung.internal.golden-gryphon.com>
32################################################################################
34import sys
35from collections import defaultdict
36from collections.abc import Hashable, Iterable
37from itertools import chain, product
38from typing import TYPE_CHECKING, NoReturn, TypedDict
40import apt_pkg
41from sqlalchemy import sql
42from sqlalchemy.engine import Result
44from daklib import utils
45from daklib.config import Config
46from daklib.cruft import newer_version, query_without_source, queryNBS
47from daklib.dbconn import DBConn, get_architecture, get_suite, get_suite_architectures
48from daklib.rm import ReverseDependencyChecker, remove
50if TYPE_CHECKING:
51 from sqlalchemy.orm import Session
53Options: apt_pkg.Configuration
55################################################################################
58def usage(exit_code=0) -> NoReturn:
59 print(
60 """Usage: dak auto-decruft
61Automatic removal of common kinds of cruft
63 -h, --help show this help and exit.
64 -n, --dry-run don't do anything, just show what would have been done
65 -s, --suite=SUITE check suite SUITE.
66 --if-newer-version-in OS remove all packages in SUITE with a lower version than
67 in OS (e.g. -s experimental --if-newer-version-in
68 unstable)
69 --if-newer-version-in-rm-msg RMMSG
70 use RMMSG in the removal message (e.g. "NVIU")
71 --decruft-equal-versions use with --if-newer-version-in to also decruft versions
72 that are identical in both suites.
73 """
74 )
75 sys.exit(exit_code)
78################################################################################
81class Group(TypedDict):
82 name: str
83 packages: tuple[str, ...]
84 architectures: list[str]
85 architecture_ids: tuple[int, ...]
86 message: str
87 removal_request: dict[str, list[str]]
90def compute_sourceless_groups(suite_id: int, session: "Session") -> Iterable[Group]:
91 """Find binaries without a source
93 :param suite_id: The id of the suite denoted by suite_name
94 :param session: The database session in use
95 """
96 rows = query_without_source(suite_id, session)
97 message = "[auto-cruft] no longer built from source, no reverse dependencies"
98 arch = get_architecture("all", session=session)
99 assert arch is not None
100 arch_all_id_tuple = (arch.arch_id,)
101 arch_all_list = ["all"]
102 for row in rows: 102 ↛ 103line 102 didn't jump to line 103 because the loop on line 102 never started
103 package = row[0]
104 group_info: Group = {
105 "name": "sourceless:%s" % package,
106 "packages": (package,),
107 "architectures": arch_all_list,
108 "architecture_ids": arch_all_id_tuple,
109 "message": message,
110 "removal_request": {
111 package: arch_all_list,
112 },
113 }
114 yield group_info
117def compute_nbs_groups(
118 suite_id: int, suite_name: str, session: "Session"
119) -> Iterable[Group]:
120 """Find binaries no longer built
122 :param suite_id: The id of the suite denoted by suite_name
123 :param suite_name: The name of the suite to remove from
124 :param session: The database session in use
125 """
126 rows = queryNBS(suite_id, session)
127 arch2ids = {a.arch_string: a.arch_id for a in get_suite_architectures(suite_name)}
129 for row in rows: 129 ↛ 130line 129 didn't jump to line 130 because the loop on line 129 never started
130 (pkg_list, arch_list, source, _) = row
131 message = (
132 "[auto-cruft] NBS (no longer built by %s, no reverse dependencies)" % source
133 )
134 removal_request = {pkg: arch_list for pkg in pkg_list}
135 group_info: Group = {
136 "name": "NBS:%s" % source,
137 "packages": tuple(sorted(pkg_list)),
138 "architectures": sorted(arch_list),
139 "architecture_ids": tuple(arch2ids[arch] for arch in arch_list),
140 "message": message,
141 "removal_request": removal_request,
142 }
143 yield group_info
146def remove_groups(
147 groups: Iterable[Group], suite_id: int, suite_name: str, session: "Session"
148) -> None:
149 for group in groups:
150 message = group["message"]
151 params = {
152 "architecture_ids": group["architecture_ids"],
153 "packages": group["packages"],
154 "suite_id": suite_id,
155 }
156 q: Result[tuple[str, str, str, int]] = session.execute(
157 sql.text(
158 """
159 SELECT b.package, b.version, a.arch_string, b.id
160 FROM binaries b
161 JOIN bin_associations ba ON b.id = ba.bin
162 JOIN architecture a ON b.architecture = a.id
163 JOIN suite su ON ba.suite = su.id
164 WHERE a.id IN :architecture_ids AND b.package IN :packages AND su.id = :suite_id
165 """
166 ),
167 params,
168 )
170 remove(
171 session,
172 message,
173 [suite_name],
174 [*q],
175 partial=True,
176 whoami="DAK's auto-decrufter",
177 )
180def dedup[T: Hashable](*args: Iterable[T]) -> Iterable[T]:
181 seen = set()
182 for iterable in args:
183 for value in iterable:
184 if value not in seen:
185 seen.add(value)
186 yield value
189def merge_group(groupA: Group, groupB: Group) -> Group:
190 """Merges two removal groups into one
192 Note that some values are taken entirely from groupA (e.g. name and message)
194 :param groupA: A removal group
195 :param groupB: Another removal group
196 :return: A merged group
197 """
198 pkg_list = sorted(dedup(groupA["packages"], groupB["packages"]))
199 arch_list = sorted(dedup(groupA["architectures"], groupB["architectures"]))
200 arch_list_id = (*dedup(groupA["architecture_ids"], groupB["architecture_ids"]),)
201 removalA = groupA["removal_request"]
202 removalB = groupB["removal_request"]
203 new_removal = {}
204 for pkg in dedup(removalA, removalB):
205 listA = removalA.get(pkg, [])
206 listB = removalB.get(pkg, [])
207 new_removal[pkg] = sorted(dedup(listA, listB))
209 return {
210 "name": groupA["name"],
211 "packages": tuple(pkg_list),
212 "architectures": arch_list,
213 "architecture_ids": arch_list_id,
214 "message": groupA["message"],
215 "removal_request": new_removal,
216 }
219def auto_decruft_suite(
220 suite_name: str, suite_id: int, session: "Session", dryrun: bool, debug: bool
221) -> None:
222 """Run the auto-decrufter on a given suite
224 :param suite_name: The name of the suite to remove from
225 :param suite_id: The id of the suite denoted by suite_name
226 :param session: The database session in use
227 :param dryrun: If True, just print the actions rather than actually doing them
228 :param debug: If True, print some extra information
229 """
230 all_architectures = [a.arch_string for a in get_suite_architectures(suite_name)]
231 pkg_arch2groups = defaultdict(set)
232 group_order = []
233 groups = {}
234 full_removal_request: list[tuple[str, list[str]]] = []
235 group_generator = chain(
236 compute_sourceless_groups(suite_id, session),
237 compute_nbs_groups(suite_id, suite_name, session),
238 )
239 for group in group_generator: 239 ↛ 240line 239 didn't jump to line 240 because the loop on line 239 never started
240 group_name = group["name"]
241 pkgs = group["packages"]
242 affected_archs = group["architectures"]
243 # If we remove an arch:all package, then the breakage can occur on any
244 # of the architectures.
245 if "all" in affected_archs:
246 affected_archs = all_architectures
247 for pkg_arch in product(pkgs, affected_archs):
248 pkg_arch2groups[pkg_arch].add(group_name)
249 if group_name not in groups:
250 groups[group_name] = group
251 group_order.append(group_name)
252 else:
253 # This case usually happens when versions differ between architectures...
254 if debug:
255 print("N: Merging group %s" % (group_name))
256 groups[group_name] = merge_group(groups[group_name], group)
258 for group_name in group_order: 258 ↛ 259line 258 didn't jump to line 259 because the loop on line 258 never started
259 removal_request = groups[group_name]["removal_request"]
260 full_removal_request.extend(removal_request.items())
262 if not groups: 262 ↛ 267line 262 didn't jump to line 267 because the condition on line 262 was always true
263 if debug: 263 ↛ 264line 263 didn't jump to line 264 because the condition on line 263 was never true
264 print("N: Found no candidates")
265 return
267 if debug:
268 print("N: Considering to remove the following packages:")
269 for group_name in sorted(groups):
270 group_info = groups[group_name]
271 pkgs = group_info["packages"]
272 archs = group_info["architectures"]
273 print("N: * %s: %s [%s]" % (group_name, ", ".join(pkgs), " ".join(archs)))
275 if debug:
276 print("N: Compiling ReverseDependencyChecker (RDC) - please hold ...")
277 rdc = ReverseDependencyChecker(session, suite_name)
278 if debug:
279 print("N: Computing initial breakage...")
281 breakage = rdc.check_reverse_depends(full_removal_request)
282 while breakage:
283 by_breakers = [(len(breakage[x]), x, breakage[x]) for x in breakage]
284 by_breakers.sort(reverse=True)
285 if debug:
286 print(
287 "N: - Removal would break %s (package, architecture)-pairs"
288 % (len(breakage))
289 )
290 print("N: - full breakage:")
291 for _, breaker, broken in by_breakers:
292 bname = "%s/%s" % breaker
293 broken_str = ", ".join("%s/%s" % b for b in sorted(broken))
294 print("N: * %s => %s" % (bname, broken_str))
296 averted_breakage: set[tuple[str, str]] = set()
298 for _, package_arch, breakage2 in by_breakers:
299 if breakage2 <= averted_breakage:
300 # We already avoided this break
301 continue
302 guilty_groups = pkg_arch2groups[package_arch]
304 if not guilty_groups:
305 utils.fubar("Cannot figure what group provided %s" % str(package_arch))
307 if debug:
308 # Only output it, if it truly a new group being discarded
309 # - a group can reach this part multiple times, if it breaks things on
310 # more than one architecture. This being rather common in fact.
311 already_discard = True
312 if any(
313 group_name for group_name in guilty_groups if group_name in groups
314 ):
315 already_discard = False
317 if not already_discard:
318 avoided = sorted(breakage2 - averted_breakage)
319 print(
320 "N: - skipping removal of %s (breakage: %s)"
321 % (", ".join(sorted(guilty_groups)), str(avoided))
322 )
324 averted_breakage |= breakage2
325 for group_name in guilty_groups:
326 groups.pop(group_name, None)
328 if not groups:
329 if debug:
330 print("N: Nothing left to remove")
331 return
333 if debug:
334 print(
335 "N: Now considering to remove: %s"
336 % str(", ".join(sorted(groups.keys())))
337 )
339 # Rebuild the removal request with the remaining groups and off
340 # we go to (not) break the world once more time
341 full_removal_request = []
342 for group_info in groups.values():
343 full_removal_request.extend(group_info["removal_request"].items())
344 breakage = rdc.check_reverse_depends(full_removal_request)
346 if debug:
347 print("N: Removal looks good")
349 if dryrun:
350 print("Would remove the equivalent of:")
351 for group_name in group_order:
352 if group_name not in groups:
353 continue
354 group_info = groups[group_name]
355 pkgs = group_info["packages"]
356 archs = group_info["architectures"]
357 message = group_info["message"]
359 # Embed the -R just in case someone wants to run it manually later
360 print(
361 ' dak rm -m "{message}" -s {suite} -a {architectures} -p -R -b {packages}'.format(
362 message=message,
363 suite=suite_name,
364 architectures=",".join(archs),
365 packages=" ".join(pkgs),
366 )
367 )
369 print()
370 print(
371 "Note: The removals may be interdependent. A non-breaking result may require the execution of all"
372 )
373 print("of the removals")
374 else:
375 remove_groups(groups.values(), suite_id, suite_name, session)
378def sources2removals(
379 source_list: Iterable[str], suite_id: int, session: "Session"
380) -> list[tuple[str, str, str, int]]:
381 """Compute removals items given a list of names of source packages
383 :param source_list: A list of names of source packages
384 :param suite_id: The id of the suite from which these sources should be removed
385 :param session: The database session in use
386 :return: A list of items to be removed to remove all sources and their binaries from the given suite
387 """
388 params = {"suite_id": suite_id, "sources": tuple(source_list)}
389 return [
390 *session.execute(
391 sql.text(
392 """
393 SELECT s.source, s.version, 'source', s.id
394 FROM source s
395 JOIN src_associations sa ON sa.source = s.id
396 WHERE sa.suite = :suite_id AND s.source IN :sources"""
397 ),
398 params,
399 ),
400 *session.execute(
401 sql.text(
402 """
403 SELECT b.package, b.version, a.arch_string, b.id
404 FROM binaries b
405 JOIN bin_associations ba ON b.id = ba.bin
406 JOIN architecture a ON b.architecture = a.id
407 JOIN source s ON b.source = s.id
408 WHERE ba.suite = :suite_id AND s.source IN :sources"""
409 ),
410 params,
411 ),
412 ]
415def decruft_newer_version_in(
416 othersuite: str,
417 suite_name: str,
418 suite_id: int,
419 rm_msg: str,
420 session: "Session",
421 dryrun: bool,
422 decruft_equal_versions: bool,
423) -> None:
424 """Compute removals items given a list of names of source packages
426 :param othersuite: The name of the suite to compare with (e.g. "unstable" for "NVIU")
427 :param suite: The name of the suite from which to do removals (e.g. "experimental" for "NVIU")
428 :param suite_id: The id of the suite from which these sources should be removed
429 :param rm_msg: The removal message (or tag, e.g. "NVIU")
430 :param session: The database session in use
431 :param dryrun: If True, just print the actions rather than actually doing them
432 :param decruft_equal_versions: If True, use >= instead of > for finding decruftable packages.
433 """
434 nvi_list = [
435 x[0]
436 for x in newer_version(
437 othersuite, suite_name, session, include_equal=decruft_equal_versions
438 )
439 ]
440 if nvi_list:
441 message = "[auto-cruft] %s" % rm_msg
442 if dryrun: 442 ↛ 443line 442 didn't jump to line 443 because the condition on line 442 was never true
443 print(
444 ' dak rm -m "%s" -s %s %s'
445 % (message, suite_name, " ".join(nvi_list))
446 )
447 else:
448 removals = sources2removals(nvi_list, suite_id, session)
449 remove(
450 session, message, [suite_name], removals, whoami="DAK's auto-decrufter"
451 )
454################################################################################
457def main() -> None:
458 global Options
459 cnf = Config()
461 Arguments = [
462 ("h", "help", "Auto-Decruft::Options::Help"),
463 ("n", "dry-run", "Auto-Decruft::Options::Dry-Run"),
464 ("d", "debug", "Auto-Decruft::Options::Debug"),
465 ("s", "suite", "Auto-Decruft::Options::Suite", "HasArg"),
466 # The "\0" seems to be the only way to disable short options.
467 ("\0", "if-newer-version-in", "Auto-Decruft::Options::OtherSuite", "HasArg"),
468 (
469 "\0",
470 "if-newer-version-in-rm-msg",
471 "Auto-Decruft::Options::OtherSuiteRMMsg",
472 "HasArg",
473 ),
474 (
475 "\0",
476 "decruft-equal-versions",
477 "Auto-Decruft::Options::OtherSuiteDecruftEqual",
478 ),
479 ]
480 for i in [
481 "help",
482 "Dry-Run",
483 "Debug",
484 "OtherSuite",
485 "OtherSuiteRMMsg",
486 "OtherSuiteDecruftEqual",
487 ]:
488 key = "Auto-Decruft::Options::%s" % i
489 if key not in cnf: 489 ↛ 480line 489 didn't jump to line 480
490 cnf[key] = ""
492 cnf["Auto-Decruft::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable")
494 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) # type: ignore[attr-defined]
496 Options = cnf.subtree("Auto-Decruft::Options")
497 if Options["Help"]:
498 usage()
500 debug = False
501 dryrun = False
502 decruft_equal_versions = False
503 if Options["Dry-Run"]: 503 ↛ 504line 503 didn't jump to line 504 because the condition on line 503 was never true
504 dryrun = True
505 if Options["Debug"]: 505 ↛ 506line 505 didn't jump to line 506 because the condition on line 505 was never true
506 debug = True
507 if Options["OtherSuiteDecruftEqual"]:
508 decruft_equal_versions = True
510 if Options["OtherSuite"] and not Options["OtherSuiteRMMsg"]: 510 ↛ 511line 510 didn't jump to line 511 because the condition on line 510 was never true
511 utils.fubar("--if-newer-version-in requires --if-newer-version-in-rm-msg")
513 session = DBConn().session()
515 suite = get_suite(Options["Suite"].lower(), session)
516 if not suite: 516 ↛ 517line 516 didn't jump to line 517 because the condition on line 516 was never true
517 utils.fubar("Cannot find suite %s" % Options["Suite"].lower())
519 suite_id = suite.suite_id
520 suite_name = suite.suite_name.lower()
522 auto_decruft_suite(suite_name, suite_id, session, dryrun, debug)
524 if Options["OtherSuite"]:
525 osuite = get_suite(Options["OtherSuite"].lower(), session)
526 assert osuite is not None
527 osuite_name = osuite.suite_name
528 decruft_newer_version_in(
529 osuite_name,
530 suite_name,
531 suite_id,
532 Options["OtherSuiteRMMsg"],
533 session,
534 dryrun,
535 decruft_equal_versions,
536 )
538 if not dryrun: 538 ↛ exitline 538 didn't return from function 'main' because the condition on line 538 was always true
539 session.commit()