Coverage for dak/cruft_report.py: 45%

1""" 

2Check for obsolete binary packages 

3 

4@contact: Debian FTP Master <ftpmaster@debian.org> 

5@copyright: 2000-2006 James Troup <james@nocrew.org> 

6@copyright: 2009 Torsten Werner <twerner@debian.org> 

7@license: GNU General Public License version 2 or later 

8""" 

9 

10# This program is free software; you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation; either version 2 of the License, or 

13# (at your option) any later version. 

14 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19 

20# You should have received a copy of the GNU General Public License 

21# along with this program; if not, write to the Free Software 

22# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 

23 

24################################################################################ 

25 

26# ``If you're claiming that's a "problem" that needs to be "fixed", 

27# you might as well write some letters to God about how unfair entropy 

28# is while you're at it.'' -- 20020802143104.GA5628@azure.humbug.org.au 

29 

30## TODO: fix NBS looping for version, implement Dubious NBS, fix up output of 

31## duplicate source package stuff, improve experimental ?, add overrides, 

32## avoid ANAIS for duplicated packages 

33 

34################################################################################ 

35 

36import functools 

37import os 

38import re 

39import sys 

40from collections import defaultdict 

41from collections.abc import Iterable 

42from typing import TYPE_CHECKING, NoReturn, cast 

43 

44import apt_pkg 

45from sqlalchemy import sql 

46from sqlalchemy.engine import CursorResult 

47 

48from daklib import utils 

49from daklib.config import Config 

50from daklib.cruft import ( 

51 newer_version, 

52 query_without_source, 

53 queryNBS, 

54 queryNBS_metadata, 

55 report_multiple_source, 

56) 

57from daklib.dbconn import DBConn, Suite, get_suite, get_suite_architectures 

58from daklib.regexes import re_extract_src_version 

59 

60if TYPE_CHECKING: 

61 from sqlalchemy.engine import Result 

62 from sqlalchemy.orm import Session 

63 

64################################################################################ 

65 

66suite: Suite 

67suite_id: int 

68 

69no_longer_in_suite: set[str] = set() # Really should be static to add_nbs, but I'm lazy 

70 

71source_binaries: dict[str, str] = {} 

72source_versions: dict[str, str] = {} 

73 

74################################################################################ 

75 

76 

77def usage(exit_code=0) -> NoReturn: 

78 print( 

79 """Usage: dak cruft-report 

80Check for obsolete or duplicated packages. 

81 

82 -h, --help show this help and exit. 

83 -m, --mode=MODE chose the MODE to run in (full, daily, bdo). 

84 -s, --suite=SUITE check suite SUITE. 

85 -R, --rdep-check check reverse dependencies 

86 -w, --wanna-build-dump where to find the copies of https://buildd.debian.org/stats/*.txt 

87 --skip-sources=SRCS comma-separated source packages to skip in the 

88 cruft report (the reverse-dependency check 

89 still sees their binaries).""" 

90 ) 

91 sys.exit(exit_code) 

92 

93 

94################################################################################ 

95 

96 

97def print_info(s="") -> None: 

98 cnf = Config() 

99 

100 if cnf.subtree("Cruft-Report::Options")["Commands-Only"]: 100 ↛ 101line 100 didn't jump to line 101 because the condition on line 100 was never true

101 return 

102 

103 print(s) 

104 

105 

106def print_cmd(s: str, indent=4) -> None: 

107 cnf = Config() 

108 

109 # Indent if doing the human readable display 

110 if not cnf.subtree("Cruft-Report::Options")["Commands-Only"]: 110 ↛ 114line 110 didn't jump to line 114 because the condition on line 110 was always true

111 ind = " " * indent 

112 s = ind + s 

113 

114 print(s) 

115 

116 

117################################################################################ 

118 

119 

120def add_nbs( 

121 nbs_d: dict[str, dict[str, set[str]]], 

122 source: str, 

123 version: str, 

124 package: str, 

125 suite_id: int, 

126 session: "Session", 

127) -> None: 

128 # Ensure the package is still in the suite (someone may have already removed it) 

129 if package in no_longer_in_suite: 

130 return 

131 else: 

132 q = session.execute( 

133 sql.text( 

134 """SELECT b.id FROM binaries b, bin_associations ba 

135 WHERE ba.bin = b.id AND ba.suite = :suite_id 

136 AND b.package = :package LIMIT 1""" 

137 ), 

138 {"suite_id": suite_id, "package": package}, 

139 ) 

140 if not q.fetchall(): 

141 no_longer_in_suite.add(package) 

142 return 

143 

144 nbs_d[source][version].add(package) 

145 

146 

147################################################################################ 

148 

149# Check for packages built on architectures they shouldn't be. 

150 

151 

152def do_anais( 

153 architecture: str, binaries_list: Iterable[str], source: str, session: "Session" 

154) -> str: 

155 if architecture == "any" or architecture == "all": 

156 return "" 

157 

158 version_sort_key = functools.cmp_to_key(apt_pkg.version_compare) 

159 anais_output = "" 

160 architectures = {a.strip() for a in architecture.split()} 

161 for binary in binaries_list: 

162 q = session.execute( 

163 sql.text( 

164 """SELECT a.arch_string, b.version 

165 FROM binaries b, bin_associations ba, architecture a 

166 WHERE ba.suite = :suiteid AND ba.bin = b.id 

167 AND b.architecture = a.id AND b.package = :package""" 

168 ), 

169 {"suiteid": suite_id, "package": binary}, 

170 ) 

171 ql = q.fetchall() 

172 versions = [] 

173 for arch, version in ql: 

174 if arch in architectures: 

175 versions.append(version) 

176 versions.sort(key=version_sort_key) 

177 if versions: 

178 latest_version = versions.pop() 

179 else: 

180 latest_version = None 

181 # Check for 'invalid' architectures 

182 versions_d = defaultdict(list) 

183 for arch, version in ql: 

184 if arch not in architectures: 

185 versions_d[version].append(arch) 

186 

187 if versions_d: 

188 anais_output += "\n (*) %s_%s [%s]: %s\n" % ( 

189 binary, 

190 latest_version, 

191 source, 

192 architecture, 

193 ) 

194 for version in sorted(versions_d, key=version_sort_key): 

195 arches = sorted(versions_d[version]) 

196 anais_output += " o %s: %s\n" % (version, ", ".join(arches)) 

197 return anais_output 

198 

199 

200################################################################################ 

201 

202 

203# Check for out-of-date binaries on architectures that do not want to build that 

204# package any more, and have them listed as Not-For-Us 

205def do_nfu(nfu_packages: dict[str, list[tuple[str, str, str]]]) -> None: 

206 output = "" 

207 

208 a2p: dict[str, list[str]] = {} 

209 

210 for architecture, packages in nfu_packages.items(): 210 ↛ 211line 210 didn't jump to line 211 because the loop on line 210 never started

211 a2p[architecture] = [] 

212 for package, bver, sver in packages: 

213 output += " * [%s] does not want %s (binary %s, source %s)\n" % ( 

214 architecture, 

215 package, 

216 bver, 

217 sver, 

218 ) 

219 a2p[architecture].append(package) 

220 

221 if output: 221 ↛ 222line 221 didn't jump to line 222 because the condition on line 221 was never true

222 print_info("Obsolete by Not-For-Us") 

223 print_info("----------------------") 

224 print_info() 

225 print_info(output) 

226 

227 print_info("Suggested commands:") 

228 for architecture, a_packages in a2p.items(): 

229 if not a_packages: 

230 continue 

231 print_cmd( 

232 ( 

233 'dak rm -o -m "[auto-cruft] NFU" -s %s -a %s -b %s' 

234 % (suite.suite_name, architecture, " ".join(a_packages)) 

235 ), 

236 indent=1, 

237 ) 

238 print_info() 

239 

240 

241def parse_nfu(architecture: str) -> set[str]: 

242 cnf = Config() 

243 # utils/hpodder_1.1.5.0: Not-For-Us [optional:out-of-date] 

244 r = re.compile(r"^\w+/([^_]+)_.*: Not-For-Us") 

245 

246 ret = set() 

247 

248 filename = "%s/%s-all.txt" % ( 

249 cnf["Cruft-Report::Options::Wanna-Build-Dump"], 

250 architecture, 

251 ) 

252 

253 # Not all architectures may have a wanna-build dump, so we want to ignore missin 

254 # files 

255 if os.path.exists(filename): 255 ↛ 256line 255 didn't jump to line 256 because the condition on line 255 was never true

256 with open(filename) as f: 

257 for line in f: 

258 if line[0] == " ": 

259 continue 

260 

261 m = r.match(line) 

262 if m: 

263 ret.add(m.group(1)) 

264 else: 

265 utils.warn("No wanna-build dump file for architecture %s" % architecture) 

266 return ret 

267 

268 

269################################################################################ 

270 

271 

272def do_newer_version( 

273 lowersuite_name: str, 

274 highersuite_name: str, 

275 code: str, 

276 session: "Session", 

277 skip_sources: "frozenset[str]" = frozenset(), 

278) -> None: 

279 list = [ 

280 entry 

281 for entry in newer_version(lowersuite_name, highersuite_name, session) 

282 if entry[0] not in skip_sources 

283 ] 

284 if len(list) > 0: 284 ↛ 285line 284 didn't jump to line 285 because the condition on line 284 was never true

285 nv_to_remove = [] 

286 title = "Newer version in %s" % lowersuite_name 

287 print_info(title) 

288 print_info("-" * len(title)) 

289 print_info() 

290 for i in list: 

291 (source, higher_version, lower_version) = i 

292 print_info(" o %s (%s, %s)" % (source, higher_version, lower_version)) 

293 nv_to_remove.append(source) 

294 print_info() 

295 print_info("Suggested command:") 

296 print_cmd( 

297 'dak rm -m "[auto-cruft] %s" -s %s %s' 

298 % (code, highersuite_name, " ".join(nv_to_remove)), 

299 indent=1, 

300 ) 

301 print_info() 

302 

303 

304################################################################################ 

305 

306 

307def reportWithoutSource( 

308 suite_name: str, suite_id: int, session: "Session", rdeps=False 

309) -> None: 

310 rows = query_without_source(suite_id, session) 

311 title = "packages without source in suite %s" % suite_name 

312 if rows.rowcount > 0: 312 ↛ 313line 312 didn't jump to line 313 because the condition on line 312 was never true

313 print_info("%s\n%s\n" % (title, "-" * len(title))) 

314 message = '"[auto-cruft] no longer built from source"' 

315 for package, version in rows: 315 ↛ 316line 315 didn't jump to line 316 because the loop on line 315 never started

316 print_info( 

317 "* package %s in version %s is no longer built from source" 

318 % (package, version) 

319 ) 

320 print_info(" - suggested command:") 

321 print_cmd( 

322 "dak rm -m %s -s %s -a all -p -R -b %s" % (message, suite_name, package) 

323 ) 

324 if rdeps: 

325 if utils.check_reverse_depends([package], suite_name, [], session, True): 

326 print_info() 

327 else: 

328 print_info(" - No dependency problem found\n") 

329 else: 

330 print_info() 

331 

332 

333def queryNewerAll( 

334 suite_name: str, session: "Session" 

335) -> CursorResult[tuple[str, str, str, str]]: 

336 """searches for arch != all packages that have an arch == all 

337 package with a higher version in the same suite""" 

338 

339 query = """ 

340select bab1.package, bab1.version as oldver, 

341 array_to_string(array_agg(a.arch_string), ',') as oldarch, 

342 bab2.version as newver 

343 from bin_associations_binaries bab1 

344 join bin_associations_binaries bab2 

345 on bab1.package = bab2.package and bab1.version < bab2.version and 

346 bab1.suite = bab2.suite and bab1.architecture > 2 and 

347 bab2.architecture = 2 

348 join architecture a on bab1.architecture = a.id 

349 join suite s on bab1.suite = s.id 

350 where s.suite_name = :suite_name 

351 group by bab1.package, oldver, bab1.suite, newver""" 

352 return cast( 

353 CursorResult, session.execute(sql.text(query), {"suite_name": suite_name}) 

354 ) 

355 

356 

357def reportNewerAll(suite_name: str, session: "Session") -> None: 

358 rows = queryNewerAll(suite_name, session) 

359 title = "obsolete arch any packages in suite %s" % suite_name 

360 if rows.rowcount > 0: 360 ↛ 361line 360 didn't jump to line 361 because the condition on line 360 was never true

361 print_info("%s\n%s\n" % (title, "-" * len(title))) 

362 message = '"[auto-cruft] obsolete arch any package"' 

363 for row in rows: 363 ↛ 364line 363 didn't jump to line 364 because the loop on line 363 never started

364 (package, oldver, oldarch, newver) = row 

365 print_info( 

366 "* package %s is arch any in version %s but arch all in version %s" 

367 % (package, oldver, newver) 

368 ) 

369 print_info(" - suggested command:") 

370 print_cmd( 

371 "dak rm -o -m %s -s %s -a %s -p -b %s\n" 

372 % (message, suite_name, oldarch, package) 

373 ) 

374 

375 

376def reportNBS( 

377 suite_name: str, 

378 suite_id: int, 

379 rdeps=False, 

380 skip_sources: "frozenset[str]" = frozenset(), 

381) -> None: 

382 session = DBConn().session() 

383 nbsRows = queryNBS(suite_id, session) 

384 title = "NBS packages in suite %s" % suite_name 

385 if nbsRows.rowcount > 0: 385 ↛ 386line 385 didn't jump to line 386 because the condition on line 385 was never true

386 print_info("%s\n%s\n" % (title, "-" * len(title))) 

387 for pkg_list, arch_list, source, version in nbsRows: 387 ↛ 388line 387 didn't jump to line 388 because the loop on line 387 never started

388 if source in skip_sources: 

389 continue 

390 pkg_string = " ".join(pkg_list) 

391 arch_string = ",".join(arch_list) 

392 print_info( 

393 "* source package %s version %s no longer builds" % (source, version) 

394 ) 

395 print_info(" binary package(s): %s" % pkg_string) 

396 print_info(" on %s" % arch_string) 

397 print_info(" - suggested command:") 

398 message = '"[auto-cruft] NBS (no longer built by %s)"' % source 

399 print_cmd( 

400 "dak rm -o -m %s -s %s -a %s -p -R -b %s" 

401 % (message, suite_name, arch_string, pkg_string) 

402 ) 

403 if rdeps: 

404 if utils.check_reverse_depends( 

405 pkg_list, suite_name, arch_list, session, True 

406 ): 

407 print_info() 

408 else: 

409 print_info(" - No dependency problem found\n") 

410 else: 

411 print_info() 

412 session.close() 

413 

414 

415def reportNBSMetadata( 

416 suite_name: str, 

417 suite_id: int, 

418 session: "Session", 

419 rdeps=False, 

420 skip_sources: "frozenset[str]" = frozenset(), 

421) -> None: 

422 rows = queryNBS_metadata(suite_id, session) 

423 title = "NBS packages (from metadata) in suite %s" % suite_name 

424 if rows.rowcount > 0: 424 ↛ 426line 424 didn't jump to line 426 because the condition on line 424 was always true

425 print_info("%s\n%s\n" % (title, "-" * len(title))) 

426 for packages, architecture, source, version in rows: 

427 if source in skip_sources: 427 ↛ 428line 427 didn't jump to line 428 because the condition on line 427 was never true

428 continue 

429 print_info( 

430 "* source package %s version %s no longer builds" % (source, version) 

431 ) 

432 print_info(" binary package(s): %s" % packages) 

433 print_info(" on %s" % architecture) 

434 print_info(" - suggested command:") 

435 message = ( 

436 '"[auto-cruft] NBS (no longer built by %s - based on source metadata)"' 

437 % source 

438 ) 

439 print_cmd( 

440 "dak rm -o -m %s -s %s -a %s -p -R -b %s" 

441 % (message, suite_name, architecture, packages) 

442 ) 

443 if rdeps: 443 ↛ 445line 443 didn't jump to line 445 because the condition on line 443 was never true

444 # when archs is None, rdeps are checked on all archs in the suite 

445 archs = [architecture] if architecture != "all" else None 

446 if utils.check_reverse_depends( 

447 packages.split(), suite_name, archs, session, True 

448 ): 

449 print_info() 

450 else: 

451 print_info(" - No dependency problem found\n") 

452 else: 

453 print_info() 

454 

455 

456def reportAllNBS( 

457 suite_name: str, 

458 suite_id: int, 

459 session: "Session", 

460 rdeps=False, 

461 skip_sources: "frozenset[str]" = frozenset(), 

462) -> None: 

463 reportWithoutSource(suite_name, suite_id, session, rdeps) 

464 reportNewerAll(suite_name, session) 

465 reportNBS(suite_name, suite_id, rdeps, skip_sources=skip_sources) 

466 

467 

468################################################################################ 

469 

470 

471def do_dubious_nbs( 

472 dubious_nbs: dict[str, dict[str, set[str]]], 

473 skip_sources: "frozenset[str]" = frozenset(), 

474) -> None: 

475 print_info("Dubious NBS") 

476 print_info("-----------") 

477 print_info() 

478 

479 version_sort_key = functools.cmp_to_key(apt_pkg.version_compare) 

480 for source in sorted(dubious_nbs): 

481 if source in skip_sources: 

482 continue 

483 print_info( 

484 " * %s_%s builds: %s" 

485 % ( 

486 source, 

487 source_versions.get(source, "??"), 

488 source_binaries.get(source, "(source does not exist)"), 

489 ) 

490 ) 

491 print_info(" won't admit to building:") 

492 versions = sorted(dubious_nbs[source], key=version_sort_key) 

493 for version in versions: 

494 packages = sorted(dubious_nbs[source][version]) 

495 print_info(" o %s: %s" % (version, ", ".join(packages))) 

496 

497 print_info() 

498 

499 

500################################################################################ 

501 

502 

503def obsolete_source( 

504 suite_name: str, session: "Session" 

505) -> "CursorResult[tuple[int, str, str, str]]": 

506 """returns obsolete source packages for suite_name without binaries 

507 in the same suite sorted by install_date; install_date should help 

508 detecting source only (or binary throw away) uploads; duplicates in 

509 the suite are skipped 

510 

511 subquery 'source_suite_unique' returns source package names from 

512 suite without duplicates; the rationale behind is that neither 

513 cruft-report nor rm cannot handle duplicates (yet)""" 

514 

515 query = """ 

516WITH source_suite_unique AS 

517 (SELECT source, suite 

518 FROM source_suite GROUP BY source, suite HAVING count(*) = 1) 

519SELECT ss.src, ss.source, ss.version, 

520 to_char(ss.install_date, 'YYYY-MM-DD') AS install_date 

521 FROM source_suite ss 

522 JOIN source_suite_unique ssu 

523 ON ss.source = ssu.source AND ss.suite = ssu.suite 

524 JOIN suite s ON s.id = ss.suite 

525 LEFT JOIN bin_associations_binaries bab 

526 ON ss.src = bab.source AND ss.suite = bab.suite 

527 WHERE s.suite_name = :suite_name AND bab.id IS NULL 

528 AND now() - ss.install_date > '1 day'::interval 

529 ORDER BY install_date""" 

530 args = {"suite_name": suite_name} 

531 return cast(CursorResult, session.execute(sql.text(query), args)) 

532 

533 

534def source_bin(source: str, session: "Session") -> "Result[tuple[str]]": 

535 """returns binaries built by source for all or no suite grouped and 

536 ordered by package name""" 

537 

538 query = """ 

539SELECT b.package 

540 FROM binaries b 

541 JOIN src_associations_src sas ON b.source = sas.src 

542 WHERE sas.source = :source 

543 GROUP BY b.package 

544 ORDER BY b.package""" 

545 args = {"source": source} 

546 return session.execute(sql.text(query), args) 

547 

548 

549def newest_source_bab( 

550 suite_name: str, package: str, session: "Session" 

551) -> "Result[tuple[str, str]]": 

552 """returns newest source that builds binary package in suite grouped 

553 and sorted by source and package name""" 

554 

555 query = """ 

556SELECT sas.source, MAX(sas.version) AS srcver 

557 FROM src_associations_src sas 

558 JOIN bin_associations_binaries bab ON sas.src = bab.source 

559 JOIN suite s on s.id = bab.suite 

560 WHERE s.suite_name = :suite_name AND bab.package = :package 

561 GROUP BY sas.source, bab.package 

562 ORDER BY sas.source, bab.package""" 

563 args = {"suite_name": suite_name, "package": package} 

564 return session.execute(sql.text(query), args) 

565 

566 

567def report_obsolete_source( 

568 suite_name: str, 

569 session: "Session", 

570 skip_sources: "frozenset[str]" = frozenset(), 

571) -> None: 

572 rows = obsolete_source(suite_name, session) 

573 if rows.rowcount == 0: 573 ↛ 575line 573 didn't jump to line 575 because the condition on line 573 was always true

574 return 

575 print_info( 

576 """Obsolete source packages in suite %s 

577----------------------------------%s\n""" 

578 % (suite_name, "-" * len(suite_name)) 

579 ) 

580 for src, old_source, version, install_date in rows.fetchall(): 

581 if old_source in skip_sources: 

582 continue 

583 print_info( 

584 " * obsolete source %s version %s installed at %s" 

585 % (old_source, version, install_date) 

586 ) 

587 for sb_row in source_bin(old_source, session): 

588 (package,) = sb_row 

589 print_info(" - has built binary %s" % package) 

590 for nsb_row in newest_source_bab(suite_name, package, session): 

591 (new_source, srcver) = nsb_row 

592 print_info( 

593 " currently built by source %s version %s" 

594 % (new_source, srcver) 

595 ) 

596 print_info(" - suggested command:") 

597 rm_opts = '-S -p -m "[auto-cruft] obsolete source package"' 

598 print_cmd("dak rm -s %s %s %s\n" % (suite_name, rm_opts, old_source)) 

599 

600 

601def get_suite_binaries(suite: Suite, session: "Session") -> set[str]: 

602 # Initalize a large hash table of all binary packages 

603 print_info("Getting a list of binary packages in %s..." % suite.suite_name) 

604 q = session.execute( 

605 sql.text( 

606 """SELECT distinct b.package 

607 FROM binaries b, bin_associations ba 

608 WHERE ba.suite = :suiteid AND ba.bin = b.id""" 

609 ), 

610 {"suiteid": suite.suite_id}, 

611 ) 

612 return {row[0] for row in q} 

613 

614 

615################################################################################ 

616 

617 

618def report_outdated_nonfree( 

619 suite: str, 

620 session: "Session", 

621 rdeps=False, 

622 skip_sources: "frozenset[str]" = frozenset(), 

623) -> None: 

624 

625 packages: dict[str, dict[str, set[str]]] = {} 

626 query = """WITH outdated_sources AS ( 

627 SELECT s.source, s.version, s.id 

628 FROM source s 

629 JOIN src_associations sa ON sa.source = s.id 

630 WHERE sa.suite IN ( 

631 SELECT id 

632 FROM suite 

633 WHERE suite_name = :suite ) 

634 AND sa.created < (now() - interval :delay) 

635 EXCEPT SELECT s.source, max(s.version) AS version, max(s.id) 

636 FROM source s 

637 JOIN src_associations sa ON sa.source = s.id 

638 WHERE sa.suite IN ( 

639 SELECT id 

640 FROM suite 

641 WHERE suite_name = :suite ) 

642 AND sa.created < (now() - interval :delay) 

643 GROUP BY s.source ), 

644 binaries AS ( 

645 SELECT b.package, s.source, ( 

646 SELECT a.arch_string 

647 FROM architecture a 

648 WHERE a.id = b.architecture ) AS arch 

649 FROM binaries b 

650 JOIN outdated_sources s ON s.id = b.source 

651 JOIN bin_associations ba ON ba.bin = b.id 

652 JOIN override o ON o.package = b.package AND o.suite = ba.suite 

653 WHERE ba.suite IN ( 

654 SELECT id 

655 FROM suite 

656 WHERE suite_name = :suite ) 

657 AND o.component IN ( 

658 SELECT id 

659 FROM component 

660 WHERE name = 'non-free' ) ) 

661 SELECT DISTINCT package, source, arch 

662 FROM binaries 

663 ORDER BY source, package, arch""" 

664 

665 res = session.execute(sql.text(query), {"suite": suite, "delay": "'15 days'"}) 

666 for package in res: 666 ↛ 667line 666 didn't jump to line 667 because the loop on line 666 never started

667 binary = package[0] 

668 source = package[1] 

669 arch = package[2] 

670 if arch == "all": 

671 continue 

672 if source not in packages: 

673 packages[source] = {} 

674 if binary not in packages[source]: 

675 packages[source][binary] = set() 

676 packages[source][binary].add(arch) 

677 if packages: 677 ↛ 678line 677 didn't jump to line 678 because the condition on line 677 was never true

678 title = "Outdated non-free binaries in suite %s" % suite 

679 message = '"[auto-cruft] outdated non-free binaries"' 

680 print_info("%s\n%s\n" % (title, "-" * len(title))) 

681 for source in sorted(packages): 

682 if source in skip_sources: 

683 continue 

684 archs: set[str] = set() 

685 binaries: set[str] = set() 

686 print_info("* package %s has outdated non-free binaries" % source) 

687 print_info(" - suggested command:") 

688 for binary in sorted(packages[source]): 

689 binaries.add(binary) 

690 archs = archs.union(packages[source][binary]) 

691 print_cmd( 

692 "dak rm -o -m %s -s %s -a %s -p -R -b %s" 

693 % (message, suite, ",".join(archs), " ".join(binaries)) 

694 ) 

695 if rdeps: 

696 if utils.check_reverse_depends( 

697 list(binaries), suite, archs, session, True 

698 ): 

699 print_info() 

700 else: 

701 print_info(" - No dependency problem found\n") 

702 else: 

703 print_info() 

704 

705 

706################################################################################ 

707 

708 

709def main() -> None: 

710 global suite, suite_id, source_binaries, source_versions 

711 

712 cnf = Config() 

713 

714 Arguments = [ 

715 ("h", "help", "Cruft-Report::Options::Help"), 

716 ("m", "mode", "Cruft-Report::Options::Mode", "HasArg"), 

717 ("R", "rdep-check", "Cruft-Report::Options::Rdep-Check"), 

718 ("s", "suite", "Cruft-Report::Options::Suite", "HasArg"), 

719 ("w", "wanna-build-dump", "Cruft-Report::Options::Wanna-Build-Dump", "HasArg"), 

720 ("c", "commands-only", "Cruft-Report::Options::Commands-Only"), 

721 ("\0", "skip-sources", "Cruft-Report::Options::Skip-Sources", "HasArg"), 

722 ] 

723 for i in ["help", "Rdep-Check"]: 

724 key = "Cruft-Report::Options::%s" % i 

725 if key not in cnf: 725 ↛ 723line 725 didn't jump to line 723 because the condition on line 725 was always true

726 cnf[key] = "" 

727 

728 if "Cruft-Report::Options::Commands-Only" not in cnf: 728 ↛ 731line 728 didn't jump to line 731 because the condition on line 728 was always true

729 cnf["Cruft-Report::Options::Commands-Only"] = "" 

730 

731 if "Cruft-Report::Options::Skip-Sources" not in cnf: 731 ↛ 734line 731 didn't jump to line 734 because the condition on line 731 was always true

732 cnf["Cruft-Report::Options::Skip-Sources"] = "" 

733 

734 cnf["Cruft-Report::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable") 

735 

736 if "Cruft-Report::Options::Mode" not in cnf: 736 ↛ 739line 736 didn't jump to line 739 because the condition on line 736 was always true

737 cnf["Cruft-Report::Options::Mode"] = "daily" 

738 

739 if "Cruft-Report::Options::Wanna-Build-Dump" not in cnf: 739 ↛ 744line 739 didn't jump to line 744 because the condition on line 739 was always true

740 cnf["Cruft-Report::Options::Wanna-Build-Dump"] = ( 

741 "/srv/ftp-master.debian.org/scripts/nfu" 

742 ) 

743 

744 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) # type: ignore[attr-defined] 

745 

746 Options = cnf.subtree("Cruft-Report::Options") 

747 if Options["Help"]: 

748 usage() 

749 

750 if Options["Rdep-Check"]: 750 ↛ 751line 750 didn't jump to line 751 because the condition on line 750 was never true

751 rdeps = True 

752 else: 

753 rdeps = False 

754 

755 skip_sources: frozenset[str] = frozenset( 

756 s for s in Options["Skip-Sources"].split(",") if s 

757 ) 

758 

759 # Set up checks based on mode 

760 if Options["Mode"] == "daily": 760 ↛ 770line 760 didn't jump to line 770 because the condition on line 760 was always true

761 checks = [ 

762 "nbs", 

763 "nviu", 

764 "nvit", 

765 "obsolete source", 

766 "outdated non-free", 

767 "nfu", 

768 "nbs metadata", 

769 ] 

770 elif Options["Mode"] == "full": 

771 checks = [ 

772 "nbs", 

773 "nviu", 

774 "nvit", 

775 "obsolete source", 

776 "outdated non-free", 

777 "nfu", 

778 "nbs metadata", 

779 "dubious nbs", 

780 "bnb", 

781 "bms", 

782 "anais", 

783 ] 

784 elif Options["Mode"] == "bdo": 

785 checks = ["nbs", "obsolete source"] 

786 else: 

787 utils.warn( 

788 "%s is not a recognised mode - only 'full', 'daily' or 'bdo' are understood." 

789 % (Options["Mode"]) 

790 ) 

791 usage(1) 

792 

793 session = DBConn().session() 

794 

795 bin_pkgs = {} 

796 src_pkgs = {} 

797 bin2source: dict[str, dict[str, str]] = {} 

798 bins_in_suite: set[str] = set() 

799 nbs: dict[str, dict[str, set[str]]] = defaultdict(lambda: defaultdict(set)) 799 ↛ exitline 799 didn't run the lambda on line 799

800 source_versions = {} 

801 

802 anais_output = "" 

803 

804 nfu_packages: dict[str, list[tuple[str, str, str]]] = defaultdict(list) 

805 

806 suite_or_none = get_suite(Options["Suite"].lower(), session) 

807 if not suite_or_none: 807 ↛ 808line 807 didn't jump to line 808 because the condition on line 807 was never true

808 utils.fubar("Cannot find suite %s" % Options["Suite"].lower()) 

809 suite = suite_or_none 

810 

811 suite_id = suite.suite_id 

812 suite_name = suite.suite_name.lower() 

813 

814 if "obsolete source" in checks: 814 ↛ 817line 814 didn't jump to line 817 because the condition on line 814 was always true

815 report_obsolete_source(suite_name, session, skip_sources=skip_sources) 

816 

817 if "nbs" in checks: 817 ↛ 820line 817 didn't jump to line 820 because the condition on line 817 was always true

818 reportAllNBS(suite_name, suite_id, session, rdeps, skip_sources=skip_sources) 

819 

820 if "nbs metadata" in checks: 820 ↛ 825line 820 didn't jump to line 825 because the condition on line 820 was always true

821 reportNBSMetadata( 

822 suite_name, suite_id, session, rdeps, skip_sources=skip_sources 

823 ) 

824 

825 if "outdated non-free" in checks: 825 ↛ 828line 825 didn't jump to line 828 because the condition on line 825 was always true

826 report_outdated_nonfree(suite_name, session, rdeps, skip_sources=skip_sources) 

827 

828 bin_not_built = defaultdict(set) 

829 

830 if "bnb" in checks: 830 ↛ 831line 830 didn't jump to line 831 because the condition on line 830 was never true

831 bins_in_suite = get_suite_binaries(suite, session) 

832 

833 section: apt_pkg.TagSection 

834 

835 # Checks based on the Sources files 

836 components = [c.component_name for c in suite.components] 

837 for component in [c.component_name for c in suite.components]: 

838 filename = "%s/dists/%s/%s/source/Sources" % ( 

839 suite.archive.path, 

840 suite_name, 

841 component, 

842 ) 

843 filename = utils.find_possibly_compressed_file(filename) 

844 with apt_pkg.TagFile(filename) as Sources: 

845 while Sources.step(): # type: ignore[attr-defined] 

846 section = Sources.section # type: ignore[attr-defined] 

847 source = section.find("Package") 

848 source_version = section.find("Version") 

849 architecture = section.find("Architecture") 

850 binaries = section.find("Binary") 

851 binaries_list = [i.strip() for i in binaries.split(",")] 

852 

853 if "bnb" in checks and source not in skip_sources: 853 ↛ 855line 853 didn't jump to line 855 because the condition on line 853 was never true

854 # Check for binaries not built on any architecture. 

855 for binary in binaries_list: 

856 if binary not in bins_in_suite: 

857 bin_not_built[source].add(binary) 

858 

859 if "anais" in checks and source not in skip_sources: 859 ↛ 860line 859 didn't jump to line 860 because the condition on line 859 was never true

860 anais_output += do_anais( 

861 architecture, binaries_list, source, session 

862 ) 

863 

864 # build indices for checking "no source" later 

865 source_index = component + "/" + source 

866 src_pkgs[source] = source_index 

867 for binary in binaries_list: 

868 bin_pkgs[binary] = source 

869 source_binaries[source] = binaries 

870 source_versions[source] = source_version 

871 

872 # Checks based on the Packages files 

873 check_components = components[:] 

874 if suite_name != "experimental": 874 ↛ 877line 874 didn't jump to line 877 because the condition on line 874 was always true

875 check_components.append("main/debian-installer") 

876 

877 for component in check_components: 

878 architectures = [ 

879 a.arch_string 

880 for a in get_suite_architectures( 

881 suite_name, skipsrc=True, skipall=True, session=session 

882 ) 

883 ] 

884 for architecture in architectures: 

885 if component == "main/debian-installer" and re.match( 885 ↛ 888line 885 didn't jump to line 888 because the condition on line 885 was never true