Package dak :: Module auto_decruft
[hide private]
[frames] | no frames]

Source Code for Module dak.auto_decruft

  1  #! /usr/bin/env python3 
  2   
  3  """ 
  4  Check for obsolete binary packages 
  5   
  6  @contact: Debian FTP Master <ftpmaster@debian.org> 
  7  @copyright: 2000-2006 James Troup <james@nocrew.org> 
  8  @copyright: 2009      Torsten Werner <twerner@debian.org> 
  9  @copyright: 2015      Niels Thykier <niels@thykier.net> 
 10  @license: GNU General Public License version 2 or later 
 11  """ 
 12   
 13  # This program is free software; you can redistribute it and/or modify 
 14  # it under the terms of the GNU General Public License as published by 
 15  # the Free Software Foundation; either version 2 of the License, or 
 16  # (at your option) any later version. 
 17   
 18  # This program is distributed in the hope that it will be useful, 
 19  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 20  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 21  # GNU General Public License for more details. 
 22   
 23  # You should have received a copy of the GNU General Public License 
 24  # along with this program; if not, write to the Free Software 
 25  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
 26   
 27  ################################################################################ 
 28   
 29  # | priviledged positions? What privilege? The honour of working harder 
 30  # | than most people for absolutely no recognition? 
 31  # 
 32  # Manoj Srivastava <srivasta@debian.org> in <87lln8aqfm.fsf@glaurung.internal.golden-gryphon.com> 
 33   
 34  ################################################################################ 
 35   
 36  import sqlalchemy.sql as sql 
 37  import sys 
 38  import apt_pkg 
 39  from itertools import chain, product 
 40  from collections import defaultdict 
 41   
 42  from daklib.config import Config 
 43  from daklib.dbconn import * 
 44  from daklib import utils 
 45  from daklib.cruft import * 
 46  from daklib.rm import remove, ReverseDependencyChecker 
 47   
 48  ################################################################################ 
 49   
 50   
51 -def usage(exit_code=0):
52 print("""Usage: dak auto-decruft 53 Automatic removal of common kinds of cruft 54 55 -h, --help show this help and exit. 56 -n, --dry-run don't do anything, just show what would have been done 57 -s, --suite=SUITE check suite SUITE. 58 --if-newer-version-in OS remove all packages in SUITE with a lower version than 59 in OS (e.g. -s experimental --if-newer-version-in 60 unstable) 61 --if-newer-version-in-rm-msg RMMSG 62 use RMMSG in the removal message (e.g. "NVIU") 63 --decruft-equal-versions use with --if-newer-version-in to also decruft versions 64 that are identical in both suites. 65 """) 66 sys.exit(exit_code)
67 68 ################################################################################ 69 70
71 -def compute_sourceless_groups(suite_id, session):
72 """Find binaries without a source 73 74 @type suite_id: int 75 @param suite_id: The id of the suite denoted by suite_name 76 77 @type session: SQLA Session 78 @param session: The database session in use 79 """"" 80 rows = query_without_source(suite_id, session) 81 message = '[auto-cruft] no longer built from source, no reverse dependencies' 82 arch = get_architecture('all', session=session) 83 arch_all_id_tuple = tuple([arch.arch_id]) 84 arch_all_list = ["all"] 85 for row in rows: 86 package = row[0] 87 group_info = { 88 "name": "sourceless:%s" % package, 89 "packages": tuple([package]), 90 "architectures": arch_all_list, 91 "architecture_ids": arch_all_id_tuple, 92 "message": message, 93 "removal_request": { 94 package: arch_all_list, 95 }, 96 } 97 yield group_info
98 99
100 -def compute_nbs_groups(suite_id, suite_name, session):
101 """Find binaries no longer built 102 103 @type suite_id: int 104 @param suite_id: The id of the suite denoted by suite_name 105 106 @type suite_name: string 107 @param suite_name: The name of the suite to remove from 108 109 @type session: SQLA Session 110 @param session: The database session in use 111 """"" 112 rows = queryNBS(suite_id, session) 113 arch2ids = dict((a.arch_string, a.arch_id) for a in get_suite_architectures(suite_name)) 114 115 for row in rows: 116 (pkg_list, arch_list, source, _) = row 117 message = '[auto-cruft] NBS (no longer built by %s, no reverse dependencies)' % source 118 removal_request = dict((pkg, arch_list) for pkg in pkg_list) 119 group_info = { 120 "name": "NBS:%s" % source, 121 "packages": tuple(sorted(pkg_list)), 122 "architectures": sorted(arch_list), 123 "architecture_ids": tuple(arch2ids[arch] for arch in arch_list), 124 "message": message, 125 "removal_request": removal_request, 126 } 127 yield group_info
128 129
130 -def remove_groups(groups, suite_id, suite_name, session):
131 for group in groups: 132 message = group["message"] 133 params = { 134 "architecture_ids": group["architecture_ids"], 135 "packages": group["packages"], 136 "suite_id": suite_id 137 } 138 q = session.execute(sql.text(""" 139 SELECT b.package, b.version, a.arch_string, b.id 140 FROM binaries b 141 JOIN bin_associations ba ON b.id = ba.bin 142 JOIN architecture a ON b.architecture = a.id 143 JOIN suite su ON ba.suite = su.id 144 WHERE a.id IN :architecture_ids AND b.package IN :packages AND su.id = :suite_id 145 """), params) 146 147 remove(session, message, [suite_name], list(q), partial=True, whoami="DAK's auto-decrufter")
148 149
150 -def dedup(*args):
151 seen = set() 152 for iterable in args: 153 for value in iterable: 154 if value not in seen: 155 seen.add(value) 156 yield value
157 158
159 -def merge_group(groupA, groupB):
160 """Merges two removal groups into one 161 162 Note that some values are taken entirely from groupA (e.g. name and message) 163 164 @type groupA: dict 165 @param groupA: A removal group 166 167 @type groupB: dict 168 @param groupB: Another removal group 169 170 @rtype: dict 171 @returns: A merged group 172 """ 173 pkg_list = sorted(dedup(groupA["packages"], groupB["packages"])) 174 arch_list = sorted(dedup(groupA["architectures"], groupB["architectures"])) 175 arch_list_id = dedup(groupA["architecture_ids"], groupB["architecture_ids"]) 176 removalA = groupA["removal_request"] 177 removalB = groupB["removal_request"] 178 new_removal = {} 179 for pkg in dedup(removalA, removalB): 180 listA = removalA[pkg] if pkg in removalA else [] 181 listB = removalB[pkg] if pkg in removalB else [] 182 new_removal[pkg] = sorted(dedup(listA, listB)) 183 184 merged_group = { 185 "name": groupA["name"], 186 "packages": tuple(pkg_list), 187 "architectures": arch_list, 188 "architecture_ids": tuple(arch_list_id), 189 "message": groupA["message"], 190 "removal_request": new_removal, 191 } 192 193 return merged_group
194 195
196 -def auto_decruft_suite(suite_name, suite_id, session, dryrun, debug):
197 """Run the auto-decrufter on a given suite 198 199 @type suite_name: string 200 @param suite_name: The name of the suite to remove from 201 202 @type suite_id: int 203 @param suite_id: The id of the suite denoted by suite_name 204 205 @type session: SQLA Session 206 @param session: The database session in use 207 208 @type dryrun: bool 209 @param dryrun: If True, just print the actions rather than actually doing them 210 211 @type debug: bool 212 @param debug: If True, print some extra information 213 """ 214 all_architectures = [a.arch_string for a in get_suite_architectures(suite_name)] 215 pkg_arch2groups = defaultdict(set) 216 group_order = [] 217 groups = {} 218 full_removal_request = [] 219 group_generator = chain( 220 compute_sourceless_groups(suite_id, session), 221 compute_nbs_groups(suite_id, suite_name, session) 222 ) 223 for group in group_generator: 224 group_name = group["name"] 225 pkgs = group["packages"] 226 affected_archs = group["architectures"] 227 # If we remove an arch:all package, then the breakage can occur on any 228 # of the architectures. 229 if "all" in affected_archs: 230 affected_archs = all_architectures 231 for pkg_arch in product(pkgs, affected_archs): 232 pkg_arch2groups[pkg_arch].add(group_name) 233 if group_name not in groups: 234 groups[group_name] = group 235 group_order.append(group_name) 236 else: 237 # This case usually happens when versions differ between architectures... 238 if debug: 239 print("N: Merging group %s" % (group_name)) 240 groups[group_name] = merge_group(groups[group_name], group) 241 242 for group_name in group_order: 243 removal_request = groups[group_name]["removal_request"] 244 full_removal_request.extend(removal_request.items()) 245 246 if not groups: 247 if debug: 248 print("N: Found no candidates") 249 return 250 251 if debug: 252 print("N: Considering to remove the following packages:") 253 for group_name in sorted(groups): 254 group_info = groups[group_name] 255 pkgs = group_info["packages"] 256 archs = group_info["architectures"] 257 print("N: * %s: %s [%s]" % (group_name, ", ".join(pkgs), " ".join(archs))) 258 259 if debug: 260 print("N: Compiling ReverseDependencyChecker (RDC) - please hold ...") 261 rdc = ReverseDependencyChecker(session, suite_name) 262 if debug: 263 print("N: Computing initial breakage...") 264 265 breakage = rdc.check_reverse_depends(full_removal_request) 266 while breakage: 267 by_breakers = [(len(breakage[x]), x, breakage[x]) for x in breakage] 268 by_breakers.sort(reverse=True) 269 if debug: 270 print("N: - Removal would break %s (package, architecture)-pairs" % (len(breakage))) 271 print("N: - full breakage:") 272 for _, breaker, broken in by_breakers: 273 bname = "%s/%s" % breaker 274 broken_str = ", ".join("%s/%s" % b for b in sorted(broken)) 275 print("N: * %s => %s" % (bname, broken_str)) 276 277 averted_breakage = set() 278 279 for _, package_arch, breakage in by_breakers: 280 if breakage <= averted_breakage: 281 # We already avoided this break 282 continue 283 guilty_groups = pkg_arch2groups[package_arch] 284 285 if not guilty_groups: 286 utils.fubar("Cannot figure what group provided %s" % str(package_arch)) 287 288 if debug: 289 # Only output it, if it truly a new group being discarded 290 # - a group can reach this part multiple times, if it breaks things on 291 # more than one architecture. This being rather common in fact. 292 already_discard = True 293 if any(group_name for group_name in guilty_groups if group_name in groups): 294 already_discard = False 295 296 if not already_discard: 297 avoided = sorted(breakage - averted_breakage) 298 print("N: - skipping removal of %s (breakage: %s)" % (", ".join(sorted(guilty_groups)), str(avoided))) 299 300 averted_breakage |= breakage 301 for group_name in guilty_groups: 302 if group_name in groups: 303 del groups[group_name] 304 305 if not groups: 306 if debug: 307 print("N: Nothing left to remove") 308 return 309 310 if debug: 311 print("N: Now considering to remove: %s" % str(", ".join(sorted(groups.keys())))) 312 313 # Rebuild the removal request with the remaining groups and off 314 # we go to (not) break the world once more time 315 full_removal_request = [] 316 for group_info in groups.values(): 317 full_removal_request.extend(group_info["removal_request"].items()) 318 breakage = rdc.check_reverse_depends(full_removal_request) 319 320 if debug: 321 print("N: Removal looks good") 322 323 if dryrun: 324 print("Would remove the equivalent of:") 325 for group_name in group_order: 326 if group_name not in groups: 327 continue 328 group_info = groups[group_name] 329 pkgs = group_info["packages"] 330 archs = group_info["architectures"] 331 message = group_info["message"] 332 333 # Embed the -R just in case someone wants to run it manually later 334 print(' dak rm -m "{message}" -s {suite} -a {architectures} -p -R -b {packages}'.format( 335 message=message, suite=suite_name, 336 architectures=",".join(archs), packages=" ".join(pkgs), 337 )) 338 339 print() 340 print("Note: The removals may be interdependent. A non-breaking result may require the execution of all") 341 print("of the removals") 342 else: 343 remove_groups(groups.values(), suite_id, suite_name, session)
344 345
346 -def sources2removals(source_list, suite_id, session):
347 """Compute removals items given a list of names of source packages 348 349 @type source_list: list 350 @param source_list: A list of names of source packages 351 352 @type suite_id: int 353 @param suite_id: The id of the suite from which these sources should be removed 354 355 @type session: SQLA Session 356 @param session: The database session in use 357 358 @rtype: list 359 @return: A list of items to be removed to remove all sources and their binaries from the given suite 360 """ 361 to_remove = [] 362 params = {"suite_id": suite_id, "sources": tuple(source_list)} 363 q = session.execute(sql.text(""" 364 SELECT s.source, s.version, 'source', s.id 365 FROM source s 366 JOIN src_associations sa ON sa.source = s.id 367 WHERE sa.suite = :suite_id AND s.source IN :sources"""), params) 368 to_remove.extend(q) 369 q = session.execute(sql.text(""" 370 SELECT b.package, b.version, a.arch_string, b.id 371 FROM binaries b 372 JOIN bin_associations ba ON b.id = ba.bin 373 JOIN architecture a ON b.architecture = a.id 374 JOIN source s ON b.source = s.id 375 WHERE ba.suite = :suite_id AND s.source IN :sources"""), params) 376 to_remove.extend(q) 377 return to_remove
378 379
380 -def decruft_newer_version_in(othersuite, suite_name, suite_id, rm_msg, session, dryrun, decruft_equal_versions):
381 """Compute removals items given a list of names of source packages 382 383 @type othersuite: str 384 @param othersuite: The name of the suite to compare with (e.g. "unstable" for "NVIU") 385 386 @type suite: str 387 @param suite: The name of the suite from which to do removals (e.g. "experimental" for "NVIU") 388 389 @type suite_id: int 390 @param suite_id: The id of the suite from which these sources should be removed 391 392 @type rm_msg: str 393 @param rm_msg: The removal message (or tag, e.g. "NVIU") 394 395 @type session: SQLA Session 396 @param session: The database session in use 397 398 @type dryrun: bool 399 @param dryrun: If True, just print the actions rather than actually doing them 400 401 @type decruft_equal_versions: bool 402 @param decruft_equal_versions: If True, use >= instead of > for finding decruftable packages. 403 """ 404 nvi_list = [x[0] for x in newer_version(othersuite, suite_name, session, include_equal=decruft_equal_versions)] 405 if nvi_list: 406 message = "[auto-cruft] %s" % rm_msg 407 if dryrun: 408 print(" dak rm -m \"%s\" -s %s %s" % (message, suite_name, " ".join(nvi_list))) 409 else: 410 removals = sources2removals(nvi_list, suite_id, session) 411 remove(session, message, [suite_name], removals, whoami="DAK's auto-decrufter")
412 413 ################################################################################ 414 415
416 -def main():
417 global Options 418 cnf = Config() 419 420 Arguments = [('h', "help", "Auto-Decruft::Options::Help"), 421 ('n', "dry-run", "Auto-Decruft::Options::Dry-Run"), 422 ('d', "debug", "Auto-Decruft::Options::Debug"), 423 ('s', "suite", "Auto-Decruft::Options::Suite", "HasArg"), 424 # The "\0" seems to be the only way to disable short options. 425 ("\0", 'if-newer-version-in', "Auto-Decruft::Options::OtherSuite", "HasArg"), 426 ("\0", 'if-newer-version-in-rm-msg', "Auto-Decruft::Options::OtherSuiteRMMsg", "HasArg"), 427 ("\0", 'decruft-equal-versions', "Auto-Decruft::Options::OtherSuiteDecruftEqual") 428 ] 429 for i in ["help", "Dry-Run", "Debug", "OtherSuite", "OtherSuiteRMMsg", "OtherSuiteDecruftEqual"]: 430 key = "Auto-Decruft::Options::%s" % i 431 if key not in cnf: 432 cnf[key] = "" 433 434 cnf["Auto-Decruft::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable") 435 436 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) 437 438 Options = cnf.subtree("Auto-Decruft::Options") 439 if Options["Help"]: 440 usage() 441 442 debug = False 443 dryrun = False 444 decruft_equal_versions = False 445 if Options["Dry-Run"]: 446 dryrun = True 447 if Options["Debug"]: 448 debug = True 449 if Options["OtherSuiteDecruftEqual"]: 450 decruft_equal_versions = True 451 452 if Options["OtherSuite"] and not Options["OtherSuiteRMMsg"]: 453 utils.fubar("--if-newer-version-in requires --if-newer-version-in-rm-msg") 454 455 session = DBConn().session() 456 457 suite = get_suite(Options["Suite"].lower(), session) 458 if not suite: 459 utils.fubar("Cannot find suite %s" % Options["Suite"].lower()) 460 461 suite_id = suite.suite_id 462 suite_name = suite.suite_name.lower() 463 464 auto_decruft_suite(suite_name, suite_id, session, dryrun, debug) 465 466 if Options["OtherSuite"]: 467 osuite = get_suite(Options["OtherSuite"].lower(), session).suite_name 468 decruft_newer_version_in(osuite, suite_name, suite_id, Options["OtherSuiteRMMsg"], session, dryrun, decruft_equal_versions) 469 470 if not dryrun: 471 session.commit()
472 473 ################################################################################ 474 475 476 if __name__ == '__main__': 477 main() 478