Coverage for dak/cruft_report.py: 46%

1#! /usr/bin/env python3 

2 

3""" 

4Check for obsolete binary packages 

5 

6@contact: Debian FTP Master <ftpmaster@debian.org> 

7@copyright: 2000-2006 James Troup <james@nocrew.org> 

8@copyright: 2009 Torsten Werner <twerner@debian.org> 

9@license: GNU General Public License version 2 or later 

10""" 

11 

12# This program is free software; you can redistribute it and/or modify 

13# it under the terms of the GNU General Public License as published by 

14# the Free Software Foundation; either version 2 of the License, or 

15# (at your option) any later version. 

16 

17# This program is distributed in the hope that it will be useful, 

18# but WITHOUT ANY WARRANTY; without even the implied warranty of 

19# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

20# GNU General Public License for more details. 

21 

22# You should have received a copy of the GNU General Public License 

23# along with this program; if not, write to the Free Software 

24# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 

25 

26################################################################################ 

27 

28# ``If you're claiming that's a "problem" that needs to be "fixed", 

29# you might as well write some letters to God about how unfair entropy 

30# is while you're at it.'' -- 20020802143104.GA5628@azure.humbug.org.au 

31 

32## TODO: fix NBS looping for version, implement Dubious NBS, fix up output of 

33## duplicate source package stuff, improve experimental ?, add overrides, 

34## avoid ANAIS for duplicated packages 

35 

36################################################################################ 

37 

38import functools 

39import os 

40import re 

41import sys 

42from collections import defaultdict 

43from collections.abc import Iterable 

44from typing import TYPE_CHECKING, NoReturn, cast 

45 

46import apt_pkg 

47from sqlalchemy import sql 

48from sqlalchemy.engine import CursorResult 

49 

50from daklib import utils 

51from daklib.config import Config 

52from daklib.cruft import ( 

53 newer_version, 

54 query_without_source, 

55 queryNBS, 

56 queryNBS_metadata, 

57 report_multiple_source, 

58) 

59from daklib.dbconn import DBConn, Suite, get_suite, get_suite_architectures 

60from daklib.regexes import re_extract_src_version 

61 

62if TYPE_CHECKING: 

63 from sqlalchemy.engine import Result 

64 from sqlalchemy.orm import Session 

65 

66################################################################################ 

67 

68suite: Suite 

69suite_id: int 

70 

71no_longer_in_suite: set[str] = set() # Really should be static to add_nbs, but I'm lazy 

72 

73source_binaries: dict[str, str] = {} 

74source_versions: dict[str, str] = {} 

75 

76################################################################################ 

77 

78 

79def usage(exit_code=0) -> NoReturn: 

80 print( 

81 """Usage: dak cruft-report 

82Check for obsolete or duplicated packages. 

83 

84 -h, --help show this help and exit. 

85 -m, --mode=MODE chose the MODE to run in (full, daily, bdo). 

86 -s, --suite=SUITE check suite SUITE. 

87 -R, --rdep-check check reverse dependencies 

88 -w, --wanna-build-dump where to find the copies of https://buildd.debian.org/stats/*.txt""" 

89 ) 

90 sys.exit(exit_code) 

91 

92 

93################################################################################ 

94 

95 

96def print_info(s="") -> None: 

97 cnf = Config() 

98 

99 if cnf.subtree("Cruft-Report::Options")["Commands-Only"]: 99 ↛ 100line 99 didn't jump to line 100 because the condition on line 99 was never true

100 return 

101 

102 print(s) 

103 

104 

105def print_cmd(s: str, indent=4) -> None: 

106 cnf = Config() 

107 

108 # Indent if doing the human readable display 

109 if not cnf.subtree("Cruft-Report::Options")["Commands-Only"]: 109 ↛ 113line 109 didn't jump to line 113 because the condition on line 109 was always true

110 ind = " " * indent 

111 s = ind + s 

112 

113 print(s) 

114 

115 

116################################################################################ 

117 

118 

119def add_nbs( 

120 nbs_d: dict[str, dict[str, set[str]]], 

121 source: str, 

122 version: str, 

123 package: str, 

124 suite_id: int, 

125 session: "Session", 

126) -> None: 

127 # Ensure the package is still in the suite (someone may have already removed it) 

128 if package in no_longer_in_suite: 

129 return 

130 else: 

131 q = session.execute( 

132 sql.text( 

133 """SELECT b.id FROM binaries b, bin_associations ba 

134 WHERE ba.bin = b.id AND ba.suite = :suite_id 

135 AND b.package = :package LIMIT 1""" 

136 ), 

137 {"suite_id": suite_id, "package": package}, 

138 ) 

139 if not q.fetchall(): 

140 no_longer_in_suite.add(package) 

141 return 

142 

143 nbs_d[source][version].add(package) 

144 

145 

146################################################################################ 

147 

148# Check for packages built on architectures they shouldn't be. 

149 

150 

151def do_anais( 

152 architecture: str, binaries_list: Iterable[str], source: str, session: "Session" 

153) -> str: 

154 if architecture == "any" or architecture == "all": 

155 return "" 

156 

157 version_sort_key = functools.cmp_to_key(apt_pkg.version_compare) 

158 anais_output = "" 

159 architectures = {a.strip() for a in architecture.split()} 

160 for binary in binaries_list: 

161 q = session.execute( 

162 sql.text( 

163 """SELECT a.arch_string, b.version 

164 FROM binaries b, bin_associations ba, architecture a 

165 WHERE ba.suite = :suiteid AND ba.bin = b.id 

166 AND b.architecture = a.id AND b.package = :package""" 

167 ), 

168 {"suiteid": suite_id, "package": binary}, 

169 ) 

170 ql = q.fetchall() 

171 versions = [] 

172 for arch, version in ql: 

173 if arch in architectures: 

174 versions.append(version) 

175 versions.sort(key=version_sort_key) 

176 if versions: 

177 latest_version = versions.pop() 

178 else: 

179 latest_version = None 

180 # Check for 'invalid' architectures 

181 versions_d = defaultdict(list) 

182 for arch, version in ql: 

183 if arch not in architectures: 

184 versions_d[version].append(arch) 

185 

186 if versions_d: 

187 anais_output += "\n (*) %s_%s [%s]: %s\n" % ( 

188 binary, 

189 latest_version, 

190 source, 

191 architecture, 

192 ) 

193 for version in sorted(versions_d, key=version_sort_key): 

194 arches = sorted(versions_d[version]) 

195 anais_output += " o %s: %s\n" % (version, ", ".join(arches)) 

196 return anais_output 

197 

198 

199################################################################################ 

200 

201 

202# Check for out-of-date binaries on architectures that do not want to build that 

203# package any more, and have them listed as Not-For-Us 

204def do_nfu(nfu_packages: dict[str, list[tuple[str, str, str]]]) -> None: 

205 output = "" 

206 

207 a2p: dict[str, list[str]] = {} 

208 

209 for architecture in nfu_packages: 209 ↛ 210line 209 didn't jump to line 210 because the loop on line 209 never started

210 a2p[architecture] = [] 

211 for package, bver, sver in nfu_packages[architecture]: 

212 output += " * [%s] does not want %s (binary %s, source %s)\n" % ( 

213 architecture, 

214 package, 

215 bver, 

216 sver, 

217 ) 

218 a2p[architecture].append(package) 

219 

220 if output: 220 ↛ 221line 220 didn't jump to line 221 because the condition on line 220 was never true

221 print_info("Obsolete by Not-For-Us") 

222 print_info("----------------------") 

223 print_info() 

224 print_info(output) 

225 

226 print_info("Suggested commands:") 

227 for architecture in a2p: 

228 if a2p[architecture]: 

229 print_cmd( 

230 ( 

231 'dak rm -o -m "[auto-cruft] NFU" -s %s -a %s -b %s' 

232 % (suite.suite_name, architecture, " ".join(a2p[architecture])) 

233 ), 

234 indent=1, 

235 ) 

236 print_info() 

237 

238 

239def parse_nfu(architecture: str) -> set[str]: 

240 cnf = Config() 

241 # utils/hpodder_1.1.5.0: Not-For-Us [optional:out-of-date] 

242 r = re.compile(r"^\w+/([^_]+)_.*: Not-For-Us") 

243 

244 ret = set() 

245 

246 filename = "%s/%s-all.txt" % ( 

247 cnf["Cruft-Report::Options::Wanna-Build-Dump"], 

248 architecture, 

249 ) 

250 

251 # Not all architectures may have a wanna-build dump, so we want to ignore missin 

252 # files 

253 if os.path.exists(filename): 253 ↛ 254line 253 didn't jump to line 254 because the condition on line 253 was never true

254 with open(filename) as f: 

255 for line in f: 

256 if line[0] == " ": 

257 continue 

258 

259 m = r.match(line) 

260 if m: 

261 ret.add(m.group(1)) 

262 else: 

263 utils.warn("No wanna-build dump file for architecture %s" % architecture) 

264 return ret 

265 

266 

267################################################################################ 

268 

269 

270def do_newer_version( 

271 lowersuite_name: str, highersuite_name: str, code: str, session: "Session" 

272) -> None: 

273 list = newer_version(lowersuite_name, highersuite_name, session) 

274 if len(list) > 0: 274 ↛ 275line 274 didn't jump to line 275 because the condition on line 274 was never true

275 nv_to_remove = [] 

276 title = "Newer version in %s" % lowersuite_name 

277 print_info(title) 

278 print_info("-" * len(title)) 

279 print_info() 

280 for i in list: 

281 (source, higher_version, lower_version) = i 

282 print_info(" o %s (%s, %s)" % (source, higher_version, lower_version)) 

283 nv_to_remove.append(source) 

284 print_info() 

285 print_info("Suggested command:") 

286 print_cmd( 

287 'dak rm -m "[auto-cruft] %s" -s %s %s' 

288 % (code, highersuite_name, " ".join(nv_to_remove)), 

289 indent=1, 

290 ) 

291 print_info() 

292 

293 

294################################################################################ 

295 

296 

297def reportWithoutSource( 

298 suite_name: str, suite_id: int, session: "Session", rdeps=False 

299) -> None: 

300 rows = query_without_source(suite_id, session) 

301 title = "packages without source in suite %s" % suite_name 

302 if rows.rowcount > 0: 302 ↛ 303line 302 didn't jump to line 303 because the condition on line 302 was never true

303 print_info("%s\n%s\n" % (title, "-" * len(title))) 

304 message = '"[auto-cruft] no longer built from source"' 

305 for row in rows: 305 ↛ 306line 305 didn't jump to line 306 because the loop on line 305 never started

306 (package, version) = row 

307 print_info( 

308 "* package %s in version %s is no longer built from source" 

309 % (package, version) 

310 ) 

311 print_info(" - suggested command:") 

312 print_cmd( 

313 "dak rm -m %s -s %s -a all -p -R -b %s" % (message, suite_name, package) 

314 ) 

315 if rdeps: 

316 if utils.check_reverse_depends([package], suite_name, [], session, True): 

317 print_info() 

318 else: 

319 print_info(" - No dependency problem found\n") 

320 else: 

321 print_info() 

322 

323 

324def queryNewerAll( 

325 suite_name: str, session: "Session" 

326) -> CursorResult[tuple[str, str, str, str]]: 

327 """searches for arch != all packages that have an arch == all 

328 package with a higher version in the same suite""" 

329 

330 query = """ 

331select bab1.package, bab1.version as oldver, 

332 array_to_string(array_agg(a.arch_string), ',') as oldarch, 

333 bab2.version as newver 

334 from bin_associations_binaries bab1 

335 join bin_associations_binaries bab2 

336 on bab1.package = bab2.package and bab1.version < bab2.version and 

337 bab1.suite = bab2.suite and bab1.architecture > 2 and 

338 bab2.architecture = 2 

339 join architecture a on bab1.architecture = a.id 

340 join suite s on bab1.suite = s.id 

341 where s.suite_name = :suite_name 

342 group by bab1.package, oldver, bab1.suite, newver""" 

343 return cast( 

344 CursorResult, session.execute(sql.text(query), {"suite_name": suite_name}) 

345 ) 

346 

347 

348def reportNewerAll(suite_name: str, session: "Session") -> None: 

349 rows = queryNewerAll(suite_name, session) 

350 title = "obsolete arch any packages in suite %s" % suite_name 

351 if rows.rowcount > 0: 351 ↛ 352line 351 didn't jump to line 352 because the condition on line 351 was never true

352 print_info("%s\n%s\n" % (title, "-" * len(title))) 

353 message = '"[auto-cruft] obsolete arch any package"' 

354 for row in rows: 354 ↛ 355line 354 didn't jump to line 355 because the loop on line 354 never started

355 (package, oldver, oldarch, newver) = row 

356 print_info( 

357 "* package %s is arch any in version %s but arch all in version %s" 

358 % (package, oldver, newver) 

359 ) 

360 print_info(" - suggested command:") 

361 print_cmd( 

362 "dak rm -o -m %s -s %s -a %s -p -b %s\n" 

363 % (message, suite_name, oldarch, package) 

364 ) 

365 

366 

367def reportNBS(suite_name: str, suite_id: int, rdeps=False) -> None: 

368 session = DBConn().session() 

369 nbsRows = queryNBS(suite_id, session) 

370 title = "NBS packages in suite %s" % suite_name 

371 if nbsRows.rowcount > 0: 371 ↛ 372line 371 didn't jump to line 372 because the condition on line 371 was never true

372 print_info("%s\n%s\n" % (title, "-" * len(title))) 

373 for row in nbsRows: 373 ↛ 374line 373 didn't jump to line 374 because the loop on line 373 never started

374 (pkg_list, arch_list, source, version) = row 

375 pkg_string = " ".join(pkg_list) 

376 arch_string = ",".join(arch_list) 

377 print_info( 

378 "* source package %s version %s no longer builds" % (source, version) 

379 ) 

380 print_info(" binary package(s): %s" % pkg_string) 

381 print_info(" on %s" % arch_string) 

382 print_info(" - suggested command:") 

383 message = '"[auto-cruft] NBS (no longer built by %s)"' % source 

384 print_cmd( 

385 "dak rm -o -m %s -s %s -a %s -p -R -b %s" 

386 % (message, suite_name, arch_string, pkg_string) 

387 ) 

388 if rdeps: 

389 if utils.check_reverse_depends( 

390 pkg_list, suite_name, arch_list, session, True 

391 ): 

392 print_info() 

393 else: 

394 print_info(" - No dependency problem found\n") 

395 else: 

396 print_info() 

397 session.close() 

398 

399 

400def reportNBSMetadata( 

401 suite_name: str, suite_id: int, session: "Session", rdeps=False 

402) -> None: 

403 rows = queryNBS_metadata(suite_id, session) 

404 title = "NBS packages (from metadata) in suite %s" % suite_name 

405 if rows.rowcount > 0: 405 ↛ 407line 405 didn't jump to line 407 because the condition on line 405 was always true

406 print_info("%s\n%s\n" % (title, "-" * len(title))) 

407 for row in rows: 

408 (packages, architecture, source, version) = row 

409 print_info( 

410 "* source package %s version %s no longer builds" % (source, version) 

411 ) 

412 print_info(" binary package(s): %s" % packages) 

413 print_info(" on %s" % architecture) 

414 print_info(" - suggested command:") 

415 message = ( 

416 '"[auto-cruft] NBS (no longer built by %s - based on source metadata)"' 

417 % source 

418 ) 

419 print_cmd( 

420 "dak rm -o -m %s -s %s -a %s -p -R -b %s" 

421 % (message, suite_name, architecture, packages) 

422 ) 

423 if rdeps: 423 ↛ 425line 423 didn't jump to line 425 because the condition on line 423 was never true

424 # when archs is None, rdeps are checked on all archs in the suite 

425 archs = [architecture] if architecture != "all" else None 

426 if utils.check_reverse_depends( 

427 packages.split(), suite_name, archs, session, True 

428 ): 

429 print_info() 

430 else: 

431 print_info(" - No dependency problem found\n") 

432 else: 

433 print_info() 

434 

435 

436def reportAllNBS( 

437 suite_name: str, suite_id: int, session: "Session", rdeps=False 

438) -> None: 

439 reportWithoutSource(suite_name, suite_id, session, rdeps) 

440 reportNewerAll(suite_name, session) 

441 reportNBS(suite_name, suite_id, rdeps) 

442 

443 

444################################################################################ 

445 

446 

447def do_dubious_nbs(dubious_nbs: dict[str, dict[str, set[str]]]) -> None: 

448 print_info("Dubious NBS") 

449 print_info("-----------") 

450 print_info() 

451 

452 version_sort_key = functools.cmp_to_key(apt_pkg.version_compare) 

453 for source in sorted(dubious_nbs): 

454 print_info( 

455 " * %s_%s builds: %s" 

456 % ( 

457 source, 

458 source_versions.get(source, "??"), 

459 source_binaries.get(source, "(source does not exist)"), 

460 ) 

461 ) 

462 print_info(" won't admit to building:") 

463 versions = sorted(dubious_nbs[source], key=version_sort_key) 

464 for version in versions: 

465 packages = sorted(dubious_nbs[source][version]) 

466 print_info(" o %s: %s" % (version, ", ".join(packages))) 

467 

468 print_info() 

469 

470 

471################################################################################ 

472 

473 

474def obsolete_source( 

475 suite_name: str, session: "Session" 

476) -> "CursorResult[tuple[int, str, str, str]]": 

477 """returns obsolete source packages for suite_name without binaries 

478 in the same suite sorted by install_date; install_date should help 

479 detecting source only (or binary throw away) uploads; duplicates in 

480 the suite are skipped 

481 

482 subquery 'source_suite_unique' returns source package names from 

483 suite without duplicates; the rationale behind is that neither 

484 cruft-report nor rm cannot handle duplicates (yet)""" 

485 

486 query = """ 

487WITH source_suite_unique AS 

488 (SELECT source, suite 

489 FROM source_suite GROUP BY source, suite HAVING count(*) = 1) 

490SELECT ss.src, ss.source, ss.version, 

491 to_char(ss.install_date, 'YYYY-MM-DD') AS install_date 

492 FROM source_suite ss 

493 JOIN source_suite_unique ssu 

494 ON ss.source = ssu.source AND ss.suite = ssu.suite 

495 JOIN suite s ON s.id = ss.suite 

496 LEFT JOIN bin_associations_binaries bab 

497 ON ss.src = bab.source AND ss.suite = bab.suite 

498 WHERE s.suite_name = :suite_name AND bab.id IS NULL 

499 AND now() - ss.install_date > '1 day'::interval 

500 ORDER BY install_date""" 

501 args = {"suite_name": suite_name} 

502 return cast(CursorResult, session.execute(sql.text(query), args)) 

503 

504 

505def source_bin(source: str, session: "Session") -> "Result[tuple[str]]": 

506 """returns binaries built by source for all or no suite grouped and 

507 ordered by package name""" 

508 

509 query = """ 

510SELECT b.package 

511 FROM binaries b 

512 JOIN src_associations_src sas ON b.source = sas.src 

513 WHERE sas.source = :source 

514 GROUP BY b.package 

515 ORDER BY b.package""" 

516 args = {"source": source} 

517 return session.execute(sql.text(query), args) 

518 

519 

520def newest_source_bab( 

521 suite_name: str, package: str, session: "Session" 

522) -> "Result[tuple[str, str]]": 

523 """returns newest source that builds binary package in suite grouped 

524 and sorted by source and package name""" 

525 

526 query = """ 

527SELECT sas.source, MAX(sas.version) AS srcver 

528 FROM src_associations_src sas 

529 JOIN bin_associations_binaries bab ON sas.src = bab.source 

530 JOIN suite s on s.id = bab.suite 

531 WHERE s.suite_name = :suite_name AND bab.package = :package 

532 GROUP BY sas.source, bab.package 

533 ORDER BY sas.source, bab.package""" 

534 args = {"suite_name": suite_name, "package": package} 

535 return session.execute(sql.text(query), args) 

536 

537 

538def report_obsolete_source(suite_name: str, session: "Session") -> None: 

539 rows = obsolete_source(suite_name, session) 

540 if rows.rowcount == 0: 540 ↛ 542line 540 didn't jump to line 542 because the condition on line 540 was always true

541 return 

542 print_info( 

543 """Obsolete source packages in suite %s 

544----------------------------------%s\n""" 

545 % (suite_name, "-" * len(suite_name)) 

546 ) 

547 for os_row in rows.fetchall(): 

548 (src, old_source, version, install_date) = os_row 

549 print_info( 

550 " * obsolete source %s version %s installed at %s" 

551 % (old_source, version, install_date) 

552 ) 

553 for sb_row in source_bin(old_source, session): 

554 (package,) = sb_row 

555 print_info(" - has built binary %s" % package) 

556 for nsb_row in newest_source_bab(suite_name, package, session): 

557 (new_source, srcver) = nsb_row 

558 print_info( 

559 " currently built by source %s version %s" 

560 % (new_source, srcver) 

561 ) 

562 print_info(" - suggested command:") 

563 rm_opts = '-S -p -m "[auto-cruft] obsolete source package"' 

564 print_cmd("dak rm -s %s %s %s\n" % (suite_name, rm_opts, old_source)) 

565 

566 

567def get_suite_binaries(suite: Suite, session: "Session") -> set[str]: 

568 # Initalize a large hash table of all binary packages 

569 print_info("Getting a list of binary packages in %s..." % suite.suite_name) 

570 q = session.execute( 

571 sql.text( 

572 """SELECT distinct b.package 

573 FROM binaries b, bin_associations ba 

574 WHERE ba.suite = :suiteid AND ba.bin = b.id""" 

575 ), 

576 {"suiteid": suite.suite_id}, 

577 ) 

578 return {row[0] for row in q} 

579 

580 

581################################################################################ 

582 

583 

584def report_outdated_nonfree(suite: str, session: "Session", rdeps=False) -> None: 

585 

586 packages: dict[str, dict[str, set[str]]] = {} 

587 query = """WITH outdated_sources AS ( 

588 SELECT s.source, s.version, s.id 

589 FROM source s 

590 JOIN src_associations sa ON sa.source = s.id 

591 WHERE sa.suite IN ( 

592 SELECT id 

593 FROM suite 

594 WHERE suite_name = :suite ) 

595 AND sa.created < (now() - interval :delay) 

596 EXCEPT SELECT s.source, max(s.version) AS version, max(s.id) 

597 FROM source s 

598 JOIN src_associations sa ON sa.source = s.id 

599 WHERE sa.suite IN ( 

600 SELECT id 

601 FROM suite 

602 WHERE suite_name = :suite ) 

603 AND sa.created < (now() - interval :delay) 

604 GROUP BY s.source ), 

605 binaries AS ( 

606 SELECT b.package, s.source, ( 

607 SELECT a.arch_string 

608 FROM architecture a 

609 WHERE a.id = b.architecture ) AS arch 

610 FROM binaries b 

611 JOIN outdated_sources s ON s.id = b.source 

612 JOIN bin_associations ba ON ba.bin = b.id 

613 JOIN override o ON o.package = b.package AND o.suite = ba.suite 

614 WHERE ba.suite IN ( 

615 SELECT id 

616 FROM suite 

617 WHERE suite_name = :suite ) 

618 AND o.component IN ( 

619 SELECT id 

620 FROM component 

621 WHERE name = 'non-free' ) ) 

622 SELECT DISTINCT package, source, arch 

623 FROM binaries 

624 ORDER BY source, package, arch""" 

625 

626 res = session.execute(sql.text(query), {"suite": suite, "delay": "'15 days'"}) 

627 for package in res: 627 ↛ 628line 627 didn't jump to line 628 because the loop on line 627 never started

628 binary = package[0] 

629 source = package[1] 

630 arch = package[2] 

631 if arch == "all": 

632 continue 

633 if source not in packages: 

634 packages[source] = {} 

635 if binary not in packages[source]: 

636 packages[source][binary] = set() 

637 packages[source][binary].add(arch) 

638 if packages: 638 ↛ 639line 638 didn't jump to line 639 because the condition on line 638 was never true

639 title = "Outdated non-free binaries in suite %s" % suite 

640 message = '"[auto-cruft] outdated non-free binaries"' 

641 print_info("%s\n%s\n" % (title, "-" * len(title))) 

642 for source in sorted(packages): 

643 archs: set[str] = set() 

644 binaries: set[str] = set() 

645 print_info("* package %s has outdated non-free binaries" % source) 

646 print_info(" - suggested command:") 

647 for binary in sorted(packages[source]): 

648 binaries.add(binary) 

649 archs = archs.union(packages[source][binary]) 

650 print_cmd( 

651 "dak rm -o -m %s -s %s -a %s -p -R -b %s" 

652 % (message, suite, ",".join(archs), " ".join(binaries)) 

653 ) 

654 if rdeps: 

655 if utils.check_reverse_depends( 

656 list(binaries), suite, archs, session, True 

657 ): 

658 print_info() 

659 else: 

660 print_info(" - No dependency problem found\n") 

661 else: 

662 print_info() 

663 

664 

665################################################################################ 

666 

667 

668def main() -> None: 

669 global suite, suite_id, source_binaries, source_versions 

670 

671 cnf = Config() 

672 

673 Arguments = [ 

674 ("h", "help", "Cruft-Report::Options::Help"), 

675 ("m", "mode", "Cruft-Report::Options::Mode", "HasArg"), 

676 ("R", "rdep-check", "Cruft-Report::Options::Rdep-Check"), 

677 ("s", "suite", "Cruft-Report::Options::Suite", "HasArg"), 

678 ("w", "wanna-build-dump", "Cruft-Report::Options::Wanna-Build-Dump", "HasArg"), 

679 ("c", "commands-only", "Cruft-Report::Options::Commands-Only"), 

680 ] 

681 for i in ["help", "Rdep-Check"]: 

682 key = "Cruft-Report::Options::%s" % i 

683 if key not in cnf: 683 ↛ 681line 683 didn't jump to line 681 because the condition on line 683 was always true

684 cnf[key] = "" 

685 

686 if "Cruft-Report::Options::Commands-Only" not in cnf: 686 ↛ 689line 686 didn't jump to line 689 because the condition on line 686 was always true

687 cnf["Cruft-Report::Options::Commands-Only"] = "" 

688 

689 cnf["Cruft-Report::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable") 

690 

691 if "Cruft-Report::Options::Mode" not in cnf: 691 ↛ 694line 691 didn't jump to line 694 because the condition on line 691 was always true

692 cnf["Cruft-Report::Options::Mode"] = "daily" 

693 

694 if "Cruft-Report::Options::Wanna-Build-Dump" not in cnf: 694 ↛ 699line 694 didn't jump to line 699 because the condition on line 694 was always true

695 cnf["Cruft-Report::Options::Wanna-Build-Dump"] = ( 

696 "/srv/ftp-master.debian.org/scripts/nfu" 

697 ) 

698 

699 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) # type: ignore[attr-defined] 

700 

701 Options = cnf.subtree("Cruft-Report::Options") 

702 if Options["Help"]: 

703 usage() 

704 

705 if Options["Rdep-Check"]: 705 ↛ 706line 705 didn't jump to line 706 because the condition on line 705 was never true

706 rdeps = True 

707 else: 

708 rdeps = False 

709 

710 # Set up checks based on mode 

711 if Options["Mode"] == "daily": 711 ↛ 721line 711 didn't jump to line 721 because the condition on line 711 was always true

712 checks = [ 

713 "nbs", 

714 "nviu", 

715 "nvit", 

716 "obsolete source", 

717 "outdated non-free", 

718 "nfu", 

719 "nbs metadata", 

720 ] 

721 elif Options["Mode"] == "full": 

722 checks = [ 

723 "nbs", 

724 "nviu", 

725 "nvit", 

726 "obsolete source", 

727 "outdated non-free", 

728 "nfu", 

729 "nbs metadata", 

730 "dubious nbs", 

731 "bnb", 

732 "bms", 

733 "anais", 

734 ] 

735 elif Options["Mode"] == "bdo": 

736 checks = ["nbs", "obsolete source"] 

737 else: 

738 utils.warn( 

739 "%s is not a recognised mode - only 'full', 'daily' or 'bdo' are understood." 

740 % (Options["Mode"]) 

741 ) 

742 usage(1) 

743 

744 session = DBConn().session() 

745 

746 bin_pkgs = {} 

747 src_pkgs = {} 

748 bin2source: dict[str, dict[str, str]] = {} 

749 bins_in_suite: set[str] = set() 

750 nbs: dict[str, dict[str, set[str]]] = defaultdict(lambda: defaultdict(set)) 750 ↛ exitline 750 didn't run the lambda on line 750

751 source_versions = {} 

752 

753 anais_output = "" 

754 

755 nfu_packages: dict[str, list[tuple[str, str, str]]] = defaultdict(list) 

756 

757 suite_or_none = get_suite(Options["Suite"].lower(), session) 

758 if not suite_or_none: 758 ↛ 759line 758 didn't jump to line 759 because the condition on line 758 was never true

759 utils.fubar("Cannot find suite %s" % Options["Suite"].lower()) 

760 suite = suite_or_none 

761 

762 suite_id = suite.suite_id 

763 suite_name = suite.suite_name.lower() 

764 

765 if "obsolete source" in checks: 765 ↛ 768line 765 didn't jump to line 768 because the condition on line 765 was always true

766 report_obsolete_source(suite_name, session) 

767 

768 if "nbs" in checks: 768 ↛ 771line 768 didn't jump to line 771 because the condition on line 768 was always true

769 reportAllNBS(suite_name, suite_id, session, rdeps) 

770 

771 if "nbs metadata" in checks: 771 ↛ 774line 771 didn't jump to line 774 because the condition on line 771 was always true

772 reportNBSMetadata(suite_name, suite_id, session, rdeps) 

773 

774 if "outdated non-free" in checks: 774 ↛ 777line 774 didn't jump to line 777 because the condition on line 774 was always true

775 report_outdated_nonfree(suite_name, session, rdeps) 

776 

777 bin_not_built = defaultdict(set) 

778 

779 if "bnb" in checks: 779 ↛ 780line 779 didn't jump to line 780 because the condition on line 779 was never true

780 bins_in_suite = get_suite_binaries(suite, session) 

781 

782 section: apt_pkg.TagSection 

783 

784 # Checks based on the Sources files 

785 components = [c.component_name for c in suite.components] 

786 for component in [c.component_name for c in suite.components]: 

787 filename = "%s/dists/%s/%s/source/Sources" % ( 

788 suite.archive.path, 

789 suite_name, 

790 component, 

791 ) 

792 filename = utils.find_possibly_compressed_file(filename) 

793 with apt_pkg.TagFile(filename) as Sources: 

794 while Sources.step(): # type: ignore[attr-defined] 

795 section = Sources.section # type: ignore[attr-defined] 

796 source = section.find("Package") 

797 source_version = section.find("Version") 

798 architecture = section.find("Architecture") 

799 binaries = section.find("Binary") 

800 binaries_list = [i.strip() for i in binaries.split(",")] 

801 

802 if "bnb" in checks: 802 ↛ 804line 802 didn't jump to line 804 because the condition on line 802 was never true

803 # Check for binaries not built on any architecture. 

804 for binary in binaries_list: 

805 if binary not in bins_in_suite: 

806 bin_not_built[source].add(binary) 

807 

808 if "anais" in checks: 808 ↛ 809line 808 didn't jump to line 809 because the condition on line 808 was never true

809 anais_output += do_anais( 

810 architecture, binaries_list, source, session 

811 ) 

812 

813 # build indices for checking "no source" later 

814 source_index = component + "/" + source 

815 src_pkgs[source] = source_index 

816 for binary in binaries_list: 

817 bin_pkgs[binary] = source 

818 source_binaries[source] = binaries 

819 source_versions[source] = source_version 

820 

821 # Checks based on the Packages files 

822 check_components = components[:] 

823 if suite_name != "experimental": 823 ↛ 826line 823 didn't jump to line 826 because the condition on line 823 was always true

824 check_components.append("main/debian-installer") 

825 

826 for component in check_components: 

827 architectures = [ 

828 a.arch_string 

829 for a in get_suite_architectures( 

830 suite_name, skipsrc=True, skipall=True, session=session 

831 ) 

832 ] 

833 for architecture in architectures: 

834 if component == "main/debian-installer" and re.match( 834 ↛ 837line 834 didn't jump to line 837 because the condition on line 834 was never true

835 "kfreebsd", architecture 

836 ): 

837 continue 

838 

839 if "nfu" in checks: 839 ↛ 842line 839 didn't jump to line 842 because the condition on line 839 was always true

840 nfu_entries = parse_nfu(architecture) 

841 

842 filename = "%s/dists/%s/%s/binary-%s/Packages" % ( 

843 suite.archive.path, 

844 suite_name, 

845 component, 

846 architecture, 

847 ) 

848 filename = utils.find_possibly_compressed_file(filename) 

849 with apt_pkg.TagFile(filename) as Packages: 

850 while Packages.step(): # type: ignore[attr-defined] 

851 section = Packages.section # type: ignore[attr-defined] 

852 package = section.find("Package") 

853 source = section.find("Source", "") 

854 version = section.find("Version") 

855 if source == "": 

856 source = package 

857 if ( 857 ↛ 864line 857 didn't jump to line 864

858 package in bin2source 

859 and apt_pkg.version_compare( 

860 version, bin2source[package]["version"] 

861 ) 

862 > 0 

863 ): 

864 bin2source[package]["version"] = version 

865 bin2source[package]["source"] = source 

866 else: 

867 bin2source[package] = {} 

868 bin2source[package]["version"] = version 

869 bin2source[package]["source"] = source 

870 if source.find("(") != -1: 870 ↛ 871line 870 didn't jump to line 871 because the condition on line 870 was never true

871 m = re_extract_src_version.match(source) 

872 assert m is not None 

873 source = m.group(1) 

874 version = m.group(2) 

875 if package not in bin_pkgs: 875 ↛ 876line 875 didn't jump to line 876 because the condition on line 875 was never true