1#! /usr/bin/env python3
3"""
4Check for obsolete binary packages
6@contact: Debian FTP Master <ftpmaster@debian.org>
7@copyright: 2000-2006 James Troup <james@nocrew.org>
8@copyright: 2009 Torsten Werner <twerner@debian.org>
9@copyright: 2015 Niels Thykier <niels@thykier.net>
10@license: GNU General Public License version 2 or later
11"""
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27################################################################################
29# | priviledged positions? What privilege? The honour of working harder
30# | than most people for absolutely no recognition?
31#
32# Manoj Srivastava <srivasta@debian.org> in <87lln8aqfm.fsf@glaurung.internal.golden-gryphon.com>
34################################################################################
36import sys
37from collections import defaultdict
38from collections.abc import Iterable
39from itertools import chain, product
41import apt_pkg
42import sqlalchemy.sql as sql
44from daklib import utils
45from daklib.config import Config
46from daklib.cruft import newer_version, query_without_source, queryNBS
47from daklib.dbconn import DBConn, get_architecture, get_suite, get_suite_architectures
48from daklib.rm import ReverseDependencyChecker, remove
50################################################################################
53def usage(exit_code=0):
54 print(
55 """Usage: dak auto-decruft
56Automatic removal of common kinds of cruft
58 -h, --help show this help and exit.
59 -n, --dry-run don't do anything, just show what would have been done
60 -s, --suite=SUITE check suite SUITE.
61 --if-newer-version-in OS remove all packages in SUITE with a lower version than
62 in OS (e.g. -s experimental --if-newer-version-in
63 unstable)
64 --if-newer-version-in-rm-msg RMMSG
65 use RMMSG in the removal message (e.g. "NVIU")
66 --decruft-equal-versions use with --if-newer-version-in to also decruft versions
67 that are identical in both suites.
68 """
69 )
70 sys.exit(exit_code)
73################################################################################
76def compute_sourceless_groups(suite_id: int, session):
77 """Find binaries without a source
79 :param suite_id: The id of the suite denoted by suite_name
80 :param session: The database session in use
81 """ ""
82 rows = query_without_source(suite_id, session)
83 message = "[auto-cruft] no longer built from source, no reverse dependencies"
84 arch = get_architecture("all", session=session)
85 arch_all_id_tuple = tuple([arch.arch_id])
86 arch_all_list = ["all"]
87 for row in rows: 87 ↛ 88line 87 didn't jump to line 88, because the loop on line 87 never started
88 package = row[0]
89 group_info = {
90 "name": "sourceless:%s" % package,
91 "packages": tuple([package]),
92 "architectures": arch_all_list,
93 "architecture_ids": arch_all_id_tuple,
94 "message": message,
95 "removal_request": {
96 package: arch_all_list,
97 },
98 }
99 yield group_info
102def compute_nbs_groups(suite_id: int, suite_name: str, session):
103 """Find binaries no longer built
105 :param suite_id: The id of the suite denoted by suite_name
106 :param suite_name: The name of the suite to remove from
107 :param session: The database session in use
108 """ ""
109 rows = queryNBS(suite_id, session)
110 arch2ids = dict(
111 (a.arch_string, a.arch_id) for a in get_suite_architectures(suite_name)
112 )
114 for row in rows: 114 ↛ 115line 114 didn't jump to line 115, because the loop on line 114 never started
115 (pkg_list, arch_list, source, _) = row
116 message = (
117 "[auto-cruft] NBS (no longer built by %s, no reverse dependencies)" % source
118 )
119 removal_request = dict((pkg, arch_list) for pkg in pkg_list)
120 group_info = {
121 "name": "NBS:%s" % source,
122 "packages": tuple(sorted(pkg_list)),
123 "architectures": sorted(arch_list),
124 "architecture_ids": tuple(arch2ids[arch] for arch in arch_list),
125 "message": message,
126 "removal_request": removal_request,
127 }
128 yield group_info
131def remove_groups(groups, suite_id, suite_name, session):
132 for group in groups:
133 message = group["message"]
134 params = {
135 "architecture_ids": group["architecture_ids"],
136 "packages": group["packages"],
137 "suite_id": suite_id,
138 }
139 q = session.execute(
140 sql.text(
141 """
142 SELECT b.package, b.version, a.arch_string, b.id
143 FROM binaries b
144 JOIN bin_associations ba ON b.id = ba.bin
145 JOIN architecture a ON b.architecture = a.id
146 JOIN suite su ON ba.suite = su.id
147 WHERE a.id IN :architecture_ids AND b.package IN :packages AND su.id = :suite_id
148 """
149 ),
150 params,
151 )
153 remove(
154 session,
155 message,
156 [suite_name],
157 list(q),
158 partial=True,
159 whoami="DAK's auto-decrufter",
160 )
163def dedup(*args):
164 seen = set()
165 for iterable in args:
166 for value in iterable:
167 if value not in seen:
168 seen.add(value)
169 yield value
172def merge_group(groupA: dict, groupB: dict) -> dict:
173 """Merges two removal groups into one
175 Note that some values are taken entirely from groupA (e.g. name and message)
177 :param groupA: A removal group
178 :param groupB: Another removal group
179 :return: A merged group
180 """
181 pkg_list = sorted(dedup(groupA["packages"], groupB["packages"]))
182 arch_list = sorted(dedup(groupA["architectures"], groupB["architectures"]))
183 arch_list_id = dedup(groupA["architecture_ids"], groupB["architecture_ids"])
184 removalA = groupA["removal_request"]
185 removalB = groupB["removal_request"]
186 new_removal = {}
187 for pkg in dedup(removalA, removalB):
188 listA = removalA[pkg] if pkg in removalA else []
189 listB = removalB[pkg] if pkg in removalB else []
190 new_removal[pkg] = sorted(dedup(listA, listB))
192 merged_group = {
193 "name": groupA["name"],
194 "packages": tuple(pkg_list),
195 "architectures": arch_list,
196 "architecture_ids": tuple(arch_list_id),
197 "message": groupA["message"],
198 "removal_request": new_removal,
199 }
201 return merged_group
204def auto_decruft_suite(
205 suite_name: str, suite_id: int, session, dryrun: bool, debug: bool
206):
207 """Run the auto-decrufter on a given suite
209 :param suite_name: The name of the suite to remove from
210 :param suite_id: The id of the suite denoted by suite_name
211 :param session: The database session in use
212 :param dryrun: If True, just print the actions rather than actually doing them
213 :param debug: If True, print some extra information
214 """
215 all_architectures = [a.arch_string for a in get_suite_architectures(suite_name)]
216 pkg_arch2groups = defaultdict(set)
217 group_order = []
218 groups = {}
219 full_removal_request = []
220 group_generator = chain(
221 compute_sourceless_groups(suite_id, session),
222 compute_nbs_groups(suite_id, suite_name, session),
223 )
224 for group in group_generator: 224 ↛ 225line 224 didn't jump to line 225, because the loop on line 224 never started
225 group_name = group["name"]
226 pkgs = group["packages"]
227 affected_archs = group["architectures"]
228 # If we remove an arch:all package, then the breakage can occur on any
229 # of the architectures.
230 if "all" in affected_archs:
231 affected_archs = all_architectures
232 for pkg_arch in product(pkgs, affected_archs):
233 pkg_arch2groups[pkg_arch].add(group_name)
234 if group_name not in groups:
235 groups[group_name] = group
236 group_order.append(group_name)
237 else:
238 # This case usually happens when versions differ between architectures...
239 if debug:
240 print("N: Merging group %s" % (group_name))
241 groups[group_name] = merge_group(groups[group_name], group)
243 for group_name in group_order: 243 ↛ 244line 243 didn't jump to line 244, because the loop on line 243 never started
244 removal_request = groups[group_name]["removal_request"]
245 full_removal_request.extend(removal_request.items())
247 if not groups: 247 ↛ 252line 247 didn't jump to line 252, because the condition on line 247 was never false
248 if debug: 248 ↛ 249line 248 didn't jump to line 249, because the condition on line 248 was never true
249 print("N: Found no candidates")
250 return
252 if debug:
253 print("N: Considering to remove the following packages:")
254 for group_name in sorted(groups):
255 group_info = groups[group_name]
256 pkgs = group_info["packages"]
257 archs = group_info["architectures"]
258 print("N: * %s: %s [%s]" % (group_name, ", ".join(pkgs), " ".join(archs)))
260 if debug:
261 print("N: Compiling ReverseDependencyChecker (RDC) - please hold ...")
262 rdc = ReverseDependencyChecker(session, suite_name)
263 if debug:
264 print("N: Computing initial breakage...")
266 breakage = rdc.check_reverse_depends(full_removal_request)
267 while breakage:
268 by_breakers = [(len(breakage[x]), x, breakage[x]) for x in breakage]
269 by_breakers.sort(reverse=True)
270 if debug:
271 print(
272 "N: - Removal would break %s (package, architecture)-pairs"
273 % (len(breakage))
274 )
275 print("N: - full breakage:")
276 for _, breaker, broken in by_breakers:
277 bname = "%s/%s" % breaker
278 broken_str = ", ".join("%s/%s" % b for b in sorted(broken))
279 print("N: * %s => %s" % (bname, broken_str))
281 averted_breakage = set()
283 for _, package_arch, breakage in by_breakers:
284 if breakage <= averted_breakage:
285 # We already avoided this break
286 continue
287 guilty_groups = pkg_arch2groups[package_arch]
289 if not guilty_groups:
290 utils.fubar("Cannot figure what group provided %s" % str(package_arch))
292 if debug:
293 # Only output it, if it truly a new group being discarded
294 # - a group can reach this part multiple times, if it breaks things on
295 # more than one architecture. This being rather common in fact.
296 already_discard = True
297 if any(
298 group_name for group_name in guilty_groups if group_name in groups
299 ):
300 already_discard = False
302 if not already_discard:
303 avoided = sorted(breakage - averted_breakage)
304 print(
305 "N: - skipping removal of %s (breakage: %s)"
306 % (", ".join(sorted(guilty_groups)), str(avoided))
307 )
309 averted_breakage |= breakage
310 for group_name in guilty_groups:
311 if group_name in groups:
312 del groups[group_name]
314 if not groups:
315 if debug:
316 print("N: Nothing left to remove")
317 return
319 if debug:
320 print(
321 "N: Now considering to remove: %s"
322 % str(", ".join(sorted(groups.keys())))
323 )
325 # Rebuild the removal request with the remaining groups and off
326 # we go to (not) break the world once more time
327 full_removal_request = []
328 for group_info in groups.values():
329 full_removal_request.extend(group_info["removal_request"].items())
330 breakage = rdc.check_reverse_depends(full_removal_request)
332 if debug:
333 print("N: Removal looks good")
335 if dryrun:
336 print("Would remove the equivalent of:")
337 for group_name in group_order:
338 if group_name not in groups:
339 continue
340 group_info = groups[group_name]
341 pkgs = group_info["packages"]
342 archs = group_info["architectures"]
343 message = group_info["message"]
345 # Embed the -R just in case someone wants to run it manually later
346 print(
347 ' dak rm -m "{message}" -s {suite} -a {architectures} -p -R -b {packages}'.format(
348 message=message,
349 suite=suite_name,
350 architectures=",".join(archs),
351 packages=" ".join(pkgs),
352 )
353 )
355 print()
356 print(
357 "Note: The removals may be interdependent. A non-breaking result may require the execution of all"
358 )
359 print("of the removals")
360 else:
361 remove_groups(groups.values(), suite_id, suite_name, session)
364def sources2removals(source_list: Iterable[str], suite_id: int, session) -> list:
365 """Compute removals items given a list of names of source packages
367 :param source_list: A list of names of source packages
368 :param suite_id: The id of the suite from which these sources should be removed
369 :param session: The database session in use
370 :return: A list of items to be removed to remove all sources and their binaries from the given suite
371 """
372 to_remove = []
373 params = {"suite_id": suite_id, "sources": tuple(source_list)}
374 q = session.execute(
375 sql.text(
376 """
377 SELECT s.source, s.version, 'source', s.id
378 FROM source s
379 JOIN src_associations sa ON sa.source = s.id
380 WHERE sa.suite = :suite_id AND s.source IN :sources"""
381 ),
382 params,
383 )
384 to_remove.extend(q)
385 q = session.execute(
386 sql.text(
387 """
388 SELECT b.package, b.version, a.arch_string, b.id
389 FROM binaries b
390 JOIN bin_associations ba ON b.id = ba.bin
391 JOIN architecture a ON b.architecture = a.id
392 JOIN source s ON b.source = s.id
393 WHERE ba.suite = :suite_id AND s.source IN :sources"""
394 ),
395 params,
396 )
397 to_remove.extend(q)
398 return to_remove
401def decruft_newer_version_in(
402 othersuite: str,
403 suite_name: str,
404 suite_id: int,
405 rm_msg: str,
406 session,
407 dryrun: bool,
408 decruft_equal_versions: bool,
409) -> None:
410 """Compute removals items given a list of names of source packages
412 :param othersuite: The name of the suite to compare with (e.g. "unstable" for "NVIU")
413 :param suite: The name of the suite from which to do removals (e.g. "experimental" for "NVIU")
414 :param suite_id: The id of the suite from which these sources should be removed
415 :param rm_msg: The removal message (or tag, e.g. "NVIU")
416 :param session: The database session in use
417 :param dryrun: If True, just print the actions rather than actually doing them
418 :param decruft_equal_versions: If True, use >= instead of > for finding decruftable packages.
419 """
420 nvi_list = [
421 x[0]
422 for x in newer_version(
423 othersuite, suite_name, session, include_equal=decruft_equal_versions
424 )
425 ]
426 if nvi_list:
427 message = "[auto-cruft] %s" % rm_msg
428 if dryrun: 428 ↛ 429line 428 didn't jump to line 429, because the condition on line 428 was never true
429 print(
430 ' dak rm -m "%s" -s %s %s'
431 % (message, suite_name, " ".join(nvi_list))
432 )
433 else:
434 removals = sources2removals(nvi_list, suite_id, session)
435 remove(
436 session, message, [suite_name], removals, whoami="DAK's auto-decrufter"
437 )
440################################################################################
443def main():
444 global Options
445 cnf = Config()
447 Arguments = [
448 ("h", "help", "Auto-Decruft::Options::Help"),
449 ("n", "dry-run", "Auto-Decruft::Options::Dry-Run"),
450 ("d", "debug", "Auto-Decruft::Options::Debug"),
451 ("s", "suite", "Auto-Decruft::Options::Suite", "HasArg"),
452 # The "\0" seems to be the only way to disable short options.
453 ("\0", "if-newer-version-in", "Auto-Decruft::Options::OtherSuite", "HasArg"),
454 (
455 "\0",
456 "if-newer-version-in-rm-msg",
457 "Auto-Decruft::Options::OtherSuiteRMMsg",
458 "HasArg",
459 ),
460 (
461 "\0",
462 "decruft-equal-versions",
463 "Auto-Decruft::Options::OtherSuiteDecruftEqual",
464 ),
465 ]
466 for i in [
467 "help",
468 "Dry-Run",
469 "Debug",
470 "OtherSuite",
471 "OtherSuiteRMMsg",
472 "OtherSuiteDecruftEqual",
473 ]:
474 key = "Auto-Decruft::Options::%s" % i
475 if key not in cnf: 475 ↛ 466line 475 didn't jump to line 466
476 cnf[key] = ""
478 cnf["Auto-Decruft::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable")
480 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
482 Options = cnf.subtree("Auto-Decruft::Options")
483 if Options["Help"]:
484 usage()
486 debug = False
487 dryrun = False
488 decruft_equal_versions = False
489 if Options["Dry-Run"]: 489 ↛ 490line 489 didn't jump to line 490, because the condition on line 489 was never true
490 dryrun = True
491 if Options["Debug"]: 491 ↛ 492line 491 didn't jump to line 492, because the condition on line 491 was never true
492 debug = True
493 if Options["OtherSuiteDecruftEqual"]:
494 decruft_equal_versions = True
496 if Options["OtherSuite"] and not Options["OtherSuiteRMMsg"]: 496 ↛ 497line 496 didn't jump to line 497, because the condition on line 496 was never true
497 utils.fubar("--if-newer-version-in requires --if-newer-version-in-rm-msg")
499 session = DBConn().session()
501 suite = get_suite(Options["Suite"].lower(), session)
502 if not suite: 502 ↛ 503line 502 didn't jump to line 503, because the condition on line 502 was never true
503 utils.fubar("Cannot find suite %s" % Options["Suite"].lower())
505 suite_id = suite.suite_id
506 suite_name = suite.suite_name.lower()
508 auto_decruft_suite(suite_name, suite_id, session, dryrun, debug)
510 if Options["OtherSuite"]:
511 osuite = get_suite(Options["OtherSuite"].lower(), session).suite_name
512 decruft_newer_version_in(
513 osuite,
514 suite_name,
515 suite_id,
516 Options["OtherSuiteRMMsg"],
517 session,
518 dryrun,
519 decruft_equal_versions,
520 )
522 if not dryrun: 522 ↛ exitline 522 didn't return from function 'main', because the condition on line 522 was never false
523 session.commit()
526################################################################################
529if __name__ == "__main__":
530 main()