1#! /usr/bin/env python3 

2 

3""" 

4Check for obsolete binary packages 

5 

6@contact: Debian FTP Master <ftpmaster@debian.org> 

7@copyright: 2000-2006 James Troup <james@nocrew.org> 

8@copyright: 2009 Torsten Werner <twerner@debian.org> 

9@license: GNU General Public License version 2 or later 

10""" 

11 

12# This program is free software; you can redistribute it and/or modify 

13# it under the terms of the GNU General Public License as published by 

14# the Free Software Foundation; either version 2 of the License, or 

15# (at your option) any later version. 

16 

17# This program is distributed in the hope that it will be useful, 

18# but WITHOUT ANY WARRANTY; without even the implied warranty of 

19# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

20# GNU General Public License for more details. 

21 

22# You should have received a copy of the GNU General Public License 

23# along with this program; if not, write to the Free Software 

24# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 

25 

26################################################################################ 

27 

28# ``If you're claiming that's a "problem" that needs to be "fixed", 

29# you might as well write some letters to God about how unfair entropy 

30# is while you're at it.'' -- 20020802143104.GA5628@azure.humbug.org.au 

31 

32## TODO: fix NBS looping for version, implement Dubious NBS, fix up output of 

33## duplicate source package stuff, improve experimental ?, add overrides, 

34## avoid ANAIS for duplicated packages 

35 

36################################################################################ 

37 

38import functools 

39import os 

40import sys 

41import re 

42import apt_pkg 

43from collections import defaultdict 

44 

45from daklib.config import Config 

46from daklib.dbconn import * 

47from daklib import utils 

48from daklib.regexes import re_extract_src_version 

49from daklib.cruft import * 

50 

51################################################################################ 

52 

53no_longer_in_suite = {} # Really should be static to add_nbs, but I'm lazy 

54 

55source_binaries = {} 

56source_versions = {} 

57 

58################################################################################ 

59 

60 

61def usage(exit_code=0): 

62 print("""Usage: dak cruft-report 

63Check for obsolete or duplicated packages. 

64 

65 -h, --help show this help and exit. 

66 -m, --mode=MODE chose the MODE to run in (full, daily, bdo). 

67 -s, --suite=SUITE check suite SUITE. 

68 -R, --rdep-check check reverse dependencies 

69 -w, --wanna-build-dump where to find the copies of https://buildd.debian.org/stats/*.txt""") 

70 sys.exit(exit_code) 

71 

72################################################################################ 

73 

74 

75def print_info(s=""): 

76 cnf = Config() 

77 

78 if cnf.subtree("Cruft-Report::Options")["Commands-Only"]: 78 ↛ 79line 78 didn't jump to line 79, because the condition on line 78 was never true

79 return 

80 

81 print(s) 

82 

83 

84def print_cmd(s, indent=4): 

85 cnf = Config() 

86 

87 # Indent if doing the human readable display 

88 if not cnf.subtree("Cruft-Report::Options")["Commands-Only"]: 88 ↛ 92line 88 didn't jump to line 92, because the condition on line 88 was never false

89 ind = " " * indent 

90 s = ind + s 

91 

92 print(s) 

93 

94 

95################################################################################ 

96 

97 

98def add_nbs(nbs_d, source, version, package, suite_id, session): 

99 # Ensure the package is still in the suite (someone may have already removed it) 

100 if package in no_longer_in_suite: 

101 return 

102 else: 

103 q = session.execute("""SELECT b.id FROM binaries b, bin_associations ba 

104 WHERE ba.bin = b.id AND ba.suite = :suite_id 

105 AND b.package = :package LIMIT 1""", {'suite_id': suite_id, 

106 'package': package}) 

107 if not q.fetchall(): 

108 no_longer_in_suite[package] = "" 

109 return 

110 

111 nbs_d[source][version].add(package) 

112 

113################################################################################ 

114 

115# Check for packages built on architectures they shouldn't be. 

116 

117 

118def do_anais(architecture, binaries_list, source, session): 

119 if architecture == "any" or architecture == "all": 

120 return "" 

121 

122 version_sort_key = functools.cmp_to_key(apt_pkg.version_compare) 

123 anais_output = "" 

124 architectures = {} 

125 for arch in architecture.split(): 

126 architectures[arch.strip()] = "" 

127 for binary in binaries_list: 

128 q = session.execute("""SELECT a.arch_string, b.version 

129 FROM binaries b, bin_associations ba, architecture a 

130 WHERE ba.suite = :suiteid AND ba.bin = b.id 

131 AND b.architecture = a.id AND b.package = :package""", 

132 {'suiteid': suite_id, 'package': binary}) 

133 ql = q.fetchall() 

134 versions = [] 

135 for arch, version in ql: 

136 if arch in architectures: 

137 versions.append(version) 

138 versions.sort(key=version_sort_key) 

139 if versions: 

140 latest_version = versions.pop() 

141 else: 

142 latest_version = None 

143 # Check for 'invalid' architectures 

144 versions_d = defaultdict(list) 

145 for arch, version in ql: 

146 if arch not in architectures: 

147 versions_d[version].append(arch) 

148 

149 if versions_d: 

150 anais_output += "\n (*) %s_%s [%s]: %s\n" % (binary, latest_version, source, architecture) 

151 for version in sorted(versions_d, key=version_sort_key): 

152 arches = sorted(versions_d[version]) 

153 anais_output += " o %s: %s\n" % (version, ", ".join(arches)) 

154 return anais_output 

155 

156 

157################################################################################ 

158 

159# Check for out-of-date binaries on architectures that do not want to build that 

160# package any more, and have them listed as Not-For-Us 

161def do_nfu(nfu_packages): 

162 output = "" 

163 

164 a2p = {} 

165 

166 for architecture in nfu_packages: 166 ↛ 167line 166 didn't jump to line 167, because the loop on line 166 never started

167 a2p[architecture] = [] 

168 for (package, bver, sver) in nfu_packages[architecture]: 

169 output += " * [%s] does not want %s (binary %s, source %s)\n" % (architecture, package, bver, sver) 

170 a2p[architecture].append(package) 

171 

172 if output: 172 ↛ 173line 172 didn't jump to line 173, because the condition on line 172 was never true

173 print_info("Obsolete by Not-For-Us") 

174 print_info("----------------------") 

175 print_info() 

176 print_info(output) 

177 

178 print_info("Suggested commands:") 

179 for architecture in a2p: 

180 if a2p[architecture]: 

181 print_cmd(("dak rm -o -m \"[auto-cruft] NFU\" -s %s -a %s -b %s" % 

182 (suite.suite_name, architecture, " ".join(a2p[architecture]))), indent=1) 

183 print_info() 

184 

185 

186def parse_nfu(architecture): 

187 cnf = Config() 

188 # utils/hpodder_1.1.5.0: Not-For-Us [optional:out-of-date] 

189 r = re.compile(r"^\w+/([^_]+)_.*: Not-For-Us") 

190 

191 ret = set() 

192 

193 filename = "%s/%s-all.txt" % (cnf["Cruft-Report::Options::Wanna-Build-Dump"], architecture) 

194 

195 # Not all architectures may have a wanna-build dump, so we want to ignore missin 

196 # files 

197 if os.path.exists(filename): 197 ↛ 198line 197 didn't jump to line 198, because the condition on line 197 was never true

198 with open(filename) as f: 

199 for line in f: 

200 if line[0] == ' ': 

201 continue 

202 

203 m = r.match(line) 

204 if m: 

205 ret.add(m.group(1)) 

206 else: 

207 utils.warn("No wanna-build dump file for architecture %s" % architecture) 

208 return ret 

209 

210################################################################################ 

211 

212 

213def do_newer_version(lowersuite_name, highersuite_name, code, session): 

214 list = newer_version(lowersuite_name, highersuite_name, session) 

215 if len(list) > 0: 215 ↛ 216line 215 didn't jump to line 216, because the condition on line 215 was never true

216 nv_to_remove = [] 

217 title = "Newer version in %s" % lowersuite_name 

218 print_info(title) 

219 print_info("-" * len(title)) 

220 print_info() 

221 for i in list: 

222 (source, higher_version, lower_version) = i 

223 print_info(" o %s (%s, %s)" % (source, higher_version, lower_version)) 

224 nv_to_remove.append(source) 

225 print_info() 

226 print_info("Suggested command:") 

227 print_cmd("dak rm -m \"[auto-cruft] %s\" -s %s %s" % (code, highersuite_name, 

228 " ".join(nv_to_remove)), indent=1) 

229 print_info() 

230 

231################################################################################ 

232 

233 

234def reportWithoutSource(suite_name, suite_id, session, rdeps=False): 

235 rows = query_without_source(suite_id, session) 

236 title = 'packages without source in suite %s' % suite_name 

237 if rows.rowcount > 0: 237 ↛ 238line 237 didn't jump to line 238, because the condition on line 237 was never true

238 print_info('%s\n%s\n' % (title, '-' * len(title))) 

239 message = '"[auto-cruft] no longer built from source"' 

240 for row in rows: 240 ↛ 241line 240 didn't jump to line 241, because the loop on line 240 never started

241 (package, version) = row 

242 print_info("* package %s in version %s is no longer built from source" % 

243 (package, version)) 

244 print_info(" - suggested command:") 

245 print_cmd("dak rm -m %s -s %s -a all -p -R -b %s" % 

246 (message, suite_name, package)) 

247 if rdeps: 

248 if utils.check_reverse_depends([package], suite_name, [], session, True): 

249 print_info() 

250 else: 

251 print_info(" - No dependency problem found\n") 

252 else: 

253 print_info() 

254 

255 

256def queryNewerAll(suite_name, session): 

257 """searches for arch != all packages that have an arch == all 

258 package with a higher version in the same suite""" 

259 

260 query = """ 

261select bab1.package, bab1.version as oldver, 

262 array_to_string(array_agg(a.arch_string), ',') as oldarch, 

263 bab2.version as newver 

264 from bin_associations_binaries bab1 

265 join bin_associations_binaries bab2 

266 on bab1.package = bab2.package and bab1.version < bab2.version and 

267 bab1.suite = bab2.suite and bab1.architecture > 2 and 

268 bab2.architecture = 2 

269 join architecture a on bab1.architecture = a.id 

270 join suite s on bab1.suite = s.id 

271 where s.suite_name = :suite_name 

272 group by bab1.package, oldver, bab1.suite, newver""" 

273 return session.execute(query, {'suite_name': suite_name}) 

274 

275 

276def reportNewerAll(suite_name, session): 

277 rows = queryNewerAll(suite_name, session) 

278 title = 'obsolete arch any packages in suite %s' % suite_name 

279 if rows.rowcount > 0: 279 ↛ 280line 279 didn't jump to line 280, because the condition on line 279 was never true

280 print_info('%s\n%s\n' % (title, '-' * len(title))) 

281 message = '"[auto-cruft] obsolete arch any package"' 

282 for row in rows: 282 ↛ 283line 282 didn't jump to line 283, because the loop on line 282 never started

283 (package, oldver, oldarch, newver) = row 

284 print_info("* package %s is arch any in version %s but arch all in version %s" % 

285 (package, oldver, newver)) 

286 print_info(" - suggested command:") 

287 print_cmd("dak rm -o -m %s -s %s -a %s -p -b %s\n" % 

288 (message, suite_name, oldarch, package)) 

289 

290 

291def reportNBS(suite_name, suite_id, rdeps=False): 

292 session = DBConn().session() 

293 nbsRows = queryNBS(suite_id, session) 

294 title = 'NBS packages in suite %s' % suite_name 

295 if nbsRows.rowcount > 0: 295 ↛ 296line 295 didn't jump to line 296, because the condition on line 295 was never true

296 print_info('%s\n%s\n' % (title, '-' * len(title))) 

297 for row in nbsRows: 297 ↛ 298line 297 didn't jump to line 298, because the loop on line 297 never started

298 (pkg_list, arch_list, source, version) = row 

299 pkg_string = ' '.join(pkg_list) 

300 arch_string = ','.join(arch_list) 

301 print_info("* source package %s version %s no longer builds" % 

302 (source, version)) 

303 print_info(" binary package(s): %s" % pkg_string) 

304 print_info(" on %s" % arch_string) 

305 print_info(" - suggested command:") 

306 message = '"[auto-cruft] NBS (no longer built by %s)"' % source 

307 print_cmd("dak rm -o -m %s -s %s -a %s -p -R -b %s" % 

308 (message, suite_name, arch_string, pkg_string)) 

309 if rdeps: 

310 if utils.check_reverse_depends(pkg_list, suite_name, arch_list, session, True): 

311 print_info() 

312 else: 

313 print_info(" - No dependency problem found\n") 

314 else: 

315 print_info() 

316 session.close() 

317 

318 

319def reportNBSMetadata(suite_name, suite_id, session, rdeps=False): 

320 rows = queryNBS_metadata(suite_id, session) 

321 title = 'NBS packages (from metadata) in suite %s' % suite_name 

322 if rows.rowcount > 0: 322 ↛ 324line 322 didn't jump to line 324, because the condition on line 322 was never false

323 print_info('%s\n%s\n' % (title, '-' * len(title))) 

324 for row in rows: 

325 (packages, architecture, source, version) = row 

326 print_info("* source package %s version %s no longer builds" % 

327 (source, version)) 

328 print_info(" binary package(s): %s" % packages) 

329 print_info(" on %s" % architecture) 

330 print_info(" - suggested command:") 

331 message = '"[auto-cruft] NBS (no longer built by %s - based on source metadata)"' % source 

332 print_cmd("dak rm -o -m %s -s %s -a %s -p -R -b %s" % 

333 (message, suite_name, architecture, packages)) 

334 if rdeps: 334 ↛ 335line 334 didn't jump to line 335, because the condition on line 334 was never true

335 archs = [architecture] 

336 if architecture == "all": 

337 # when archs is None, rdeps are checked on all archs in the suite 

338 archs = None 

339 if utils.check_reverse_depends(packages.split(), suite_name, archs, session, True): 

340 print_info() 

341 else: 

342 print_info(" - No dependency problem found\n") 

343 else: 

344 print_info() 

345 

346 

347def reportAllNBS(suite_name, suite_id, session, rdeps=False): 

348 reportWithoutSource(suite_name, suite_id, session, rdeps) 

349 reportNewerAll(suite_name, session) 

350 reportNBS(suite_name, suite_id, rdeps) 

351 

352################################################################################ 

353 

354 

355def do_dubious_nbs(dubious_nbs): 

356 print_info("Dubious NBS") 

357 print_info("-----------") 

358 print_info() 

359 

360 version_sort_key = functools.cmp_to_key(apt_pkg.version_compare) 

361 for source in sorted(dubious_nbs): 

362 print_info(" * %s_%s builds: %s" % (source, 

363 source_versions.get(source, "??"), 

364 source_binaries.get(source, "(source does not exist)"))) 

365 print_info(" won't admit to building:") 

366 versions = sorted(dubious_nbs[source], key=version_sort_key) 

367 for version in versions: 

368 packages = sorted(dubious_nbs[source][version]) 

369 print_info(" o %s: %s" % (version, ", ".join(packages))) 

370 

371 print_info() 

372 

373################################################################################ 

374 

375 

376def obsolete_source(suite_name, session): 

377 """returns obsolete source packages for suite_name without binaries 

378 in the same suite sorted by install_date; install_date should help 

379 detecting source only (or binary throw away) uploads; duplicates in 

380 the suite are skipped 

381 

382 subquery 'source_suite_unique' returns source package names from 

383 suite without duplicates; the rationale behind is that neither 

384 cruft-report nor rm cannot handle duplicates (yet)""" 

385 

386 query = """ 

387WITH source_suite_unique AS 

388 (SELECT source, suite 

389 FROM source_suite GROUP BY source, suite HAVING count(*) = 1) 

390SELECT ss.src, ss.source, ss.version, 

391 to_char(ss.install_date, 'YYYY-MM-DD') AS install_date 

392 FROM source_suite ss 

393 JOIN source_suite_unique ssu 

394 ON ss.source = ssu.source AND ss.suite = ssu.suite 

395 JOIN suite s ON s.id = ss.suite 

396 LEFT JOIN bin_associations_binaries bab 

397 ON ss.src = bab.source AND ss.suite = bab.suite 

398 WHERE s.suite_name = :suite_name AND bab.id IS NULL 

399 AND now() - ss.install_date > '1 day'::interval 

400 ORDER BY install_date""" 

401 args = {'suite_name': suite_name} 

402 return session.execute(query, args) 

403 

404 

405def source_bin(source, session): 

406 """returns binaries built by source for all or no suite grouped and 

407 ordered by package name""" 

408 

409 query = """ 

410SELECT b.package 

411 FROM binaries b 

412 JOIN src_associations_src sas ON b.source = sas.src 

413 WHERE sas.source = :source 

414 GROUP BY b.package 

415 ORDER BY b.package""" 

416 args = {'source': source} 

417 return session.execute(query, args) 

418 

419 

420def newest_source_bab(suite_name, package, session): 

421 """returns newest source that builds binary package in suite grouped 

422 and sorted by source and package name""" 

423 

424 query = """ 

425SELECT sas.source, MAX(sas.version) AS srcver 

426 FROM src_associations_src sas 

427 JOIN bin_associations_binaries bab ON sas.src = bab.source 

428 JOIN suite s on s.id = bab.suite 

429 WHERE s.suite_name = :suite_name AND bab.package = :package 

430 GROUP BY sas.source, bab.package 

431 ORDER BY sas.source, bab.package""" 

432 args = {'suite_name': suite_name, 'package': package} 

433 return session.execute(query, args) 

434 

435 

436def report_obsolete_source(suite_name, session): 

437 rows = obsolete_source(suite_name, session) 

438 if rows.rowcount == 0: 438 ↛ 440line 438 didn't jump to line 440, because the condition on line 438 was never false

439 return 

440 print_info("""Obsolete source packages in suite %s 

441----------------------------------%s\n""" % 

442 (suite_name, '-' * len(suite_name))) 

443 for os_row in rows.fetchall(): 

444 (src, old_source, version, install_date) = os_row 

445 print_info(" * obsolete source %s version %s installed at %s" % 

446 (old_source, version, install_date)) 

447 for sb_row in source_bin(old_source, session): 

448 (package, ) = sb_row 

449 print_info(" - has built binary %s" % package) 

450 for nsb_row in newest_source_bab(suite_name, package, session): 

451 (new_source, srcver) = nsb_row 

452 print_info(" currently built by source %s version %s" % 

453 (new_source, srcver)) 

454 print_info(" - suggested command:") 

455 rm_opts = "-S -p -m \"[auto-cruft] obsolete source package\"" 

456 print_cmd("dak rm -s %s %s %s\n" % (suite_name, rm_opts, old_source)) 

457 

458 

459def get_suite_binaries(suite, session): 

460 # Initalize a large hash table of all binary packages 

461 binaries = {} 

462 

463 print_info("Getting a list of binary packages in %s..." % suite.suite_name) 

464 q = session.execute("""SELECT distinct b.package 

465 FROM binaries b, bin_associations ba 

466 WHERE ba.suite = :suiteid AND ba.bin = b.id""", 

467 {'suiteid': suite.suite_id}) 

468 for i in q.fetchall(): 

469 binaries[i[0]] = "" 

470 

471 return binaries 

472 

473################################################################################ 

474 

475 

476def report_outdated_nonfree(suite, session, rdeps=False): 

477 

478 packages = {} 

479 query = """WITH outdated_sources AS ( 

480 SELECT s.source, s.version, s.id 

481 FROM source s 

482 JOIN src_associations sa ON sa.source = s.id 

483 WHERE sa.suite IN ( 

484 SELECT id 

485 FROM suite 

486 WHERE suite_name = :suite ) 

487 AND sa.created < (now() - interval :delay) 

488 EXCEPT SELECT s.source, max(s.version) AS version, max(s.id) 

489 FROM source s 

490 JOIN src_associations sa ON sa.source = s.id 

491 WHERE sa.suite IN ( 

492 SELECT id 

493 FROM suite 

494 WHERE suite_name = :suite ) 

495 AND sa.created < (now() - interval :delay) 

496 GROUP BY s.source ), 

497 binaries AS ( 

498 SELECT b.package, s.source, ( 

499 SELECT a.arch_string 

500 FROM architecture a 

501 WHERE a.id = b.architecture ) AS arch 

502 FROM binaries b 

503 JOIN outdated_sources s ON s.id = b.source 

504 JOIN bin_associations ba ON ba.bin = b.id 

505 JOIN override o ON o.package = b.package AND o.suite = ba.suite 

506 WHERE ba.suite IN ( 

507 SELECT id 

508 FROM suite 

509 WHERE suite_name = :suite ) 

510 AND o.component IN ( 

511 SELECT id 

512 FROM component 

513 WHERE name = 'non-free' ) ) 

514 SELECT DISTINCT package, source, arch 

515 FROM binaries 

516 ORDER BY source, package, arch""" 

517 

518 res = session.execute(query, {'suite': suite, 'delay': "'15 days'"}) 

519 for package in res: 519 ↛ 520line 519 didn't jump to line 520, because the loop on line 519 never started

520 binary = package[0] 

521 source = package[1] 

522 arch = package[2] 

523 if arch == 'all': 

524 continue 

525 if source not in packages: 

526 packages[source] = {} 

527 if binary not in packages[source]: 

528 packages[source][binary] = set() 

529 packages[source][binary].add(arch) 

530 if packages: 530 ↛ 531line 530 didn't jump to line 531, because the condition on line 530 was never true

531 title = 'Outdated non-free binaries in suite %s' % suite 

532 message = '"[auto-cruft] outdated non-free binaries"' 

533 print_info('%s\n%s\n' % (title, '-' * len(title))) 

534 for source in sorted(packages): 

535 archs = set() 

536 binaries = set() 

537 print_info('* package %s has outdated non-free binaries' % source) 

538 print_info(' - suggested command:') 

539 for binary in sorted(packages[source]): 

540 binaries.add(binary) 

541 archs = archs.union(packages[source][binary]) 

542 print_cmd('dak rm -o -m %s -s %s -a %s -p -R -b %s' % 

543 (message, suite, ','.join(archs), ' '.join(binaries))) 

544 if rdeps: 

545 if utils.check_reverse_depends(list(binaries), suite, archs, session, True): 

546 print_info() 

547 else: 

548 print_info(" - No dependency problem found\n") 

549 else: 

550 print_info() 

551 

552################################################################################ 

553 

554 

555def main(): 

556 global suite, suite_id, source_binaries, source_versions 

557 

558 cnf = Config() 

559 

560 Arguments = [('h', "help", "Cruft-Report::Options::Help"), 

561 ('m', "mode", "Cruft-Report::Options::Mode", "HasArg"), 

562 ('R', "rdep-check", "Cruft-Report::Options::Rdep-Check"), 

563 ('s', "suite", "Cruft-Report::Options::Suite", "HasArg"), 

564 ('w', "wanna-build-dump", "Cruft-Report::Options::Wanna-Build-Dump", "HasArg"), 

565 ('c', "commands-only", "Cruft-Report::Options::Commands-Only")] 

566 for i in ["help", "Rdep-Check"]: 

567 key = "Cruft-Report::Options::%s" % i 

568 if key not in cnf: 568 ↛ 566line 568 didn't jump to line 566, because the condition on line 568 was never false

569 cnf[key] = "" 

570 

571 if "Cruft-Report::Options::Commands-Only" not in cnf: 571 ↛ 574line 571 didn't jump to line 574, because the condition on line 571 was never false

572 cnf["Cruft-Report::Options::Commands-Only"] = "" 

573 

574 cnf["Cruft-Report::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable") 

575 

576 if "Cruft-Report::Options::Mode" not in cnf: 576 ↛ 579line 576 didn't jump to line 579, because the condition on line 576 was never false

577 cnf["Cruft-Report::Options::Mode"] = "daily" 

578 

579 if "Cruft-Report::Options::Wanna-Build-Dump" not in cnf: 579 ↛ 582line 579 didn't jump to line 582, because the condition on line 579 was never false

580 cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp-master.debian.org/scripts/nfu" 

581 

582 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) 

583 

584 Options = cnf.subtree("Cruft-Report::Options") 

585 if Options["Help"]: 

586 usage() 

587 

588 if Options["Rdep-Check"]: 588 ↛ 589line 588 didn't jump to line 589, because the condition on line 588 was never true

589 rdeps = True 

590 else: 

591 rdeps = False 

592 

593 # Set up checks based on mode 

594 if Options["Mode"] == "daily": 594 ↛ 596line 594 didn't jump to line 596, because the condition on line 594 was never false

595 checks = ["nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu", "nbs metadata"] 

596 elif Options["Mode"] == "full": 

597 checks = ["nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu", "nbs metadata", "dubious nbs", "bnb", "bms", "anais"] 

598 elif Options["Mode"] == "bdo": 

599 checks = ["nbs", "obsolete source"] 

600 else: 

601 utils.warn("%s is not a recognised mode - only 'full', 'daily' or 'bdo' are understood." % (Options["Mode"])) 

602 usage(1) 

603 

604 session = DBConn().session() 

605 

606 bin_pkgs = {} 

607 src_pkgs = {} 

608 bin2source = {} 

609 bins_in_suite = {} 

610 nbs = defaultdict(lambda: defaultdict(set)) 610 ↛ exitline 610 didn't run the lambda on line 610

611 source_versions = {} 

612 

613 anais_output = "" 

614 

615 nfu_packages = defaultdict(list) 

616 

617 suite = get_suite(Options["Suite"].lower(), session) 

618 if not suite: 618 ↛ 619line 618 didn't jump to line 619, because the condition on line 618 was never true

619 utils.fubar("Cannot find suite %s" % Options["Suite"].lower()) 

620 

621 suite_id = suite.suite_id 

622 suite_name = suite.suite_name.lower() 

623 

624 if "obsolete source" in checks: 624 ↛ 627line 624 didn't jump to line 627, because the condition on line 624 was never false

625 report_obsolete_source(suite_name, session) 

626 

627 if "nbs" in checks: 627 ↛ 630line 627 didn't jump to line 630, because the condition on line 627 was never false

628 reportAllNBS(suite_name, suite_id, session, rdeps) 

629 

630 if "nbs metadata" in checks: 630 ↛ 633line 630 didn't jump to line 633, because the condition on line 630 was never false

631 reportNBSMetadata(suite_name, suite_id, session, rdeps) 

632 

633 if "outdated non-free" in checks: 633 ↛ 636line 633 didn't jump to line 636, because the condition on line 633 was never false

634 report_outdated_nonfree(suite_name, session, rdeps) 

635 

636 bin_not_built = defaultdict(set) 

637 

638 if "bnb" in checks: 638 ↛ 639line 638 didn't jump to line 639, because the condition on line 638 was never true

639 bins_in_suite = get_suite_binaries(suite, session) 

640 

641 # Checks based on the Sources files 

642 components = [c.component_name for c in suite.components] 

643 for component in [c.component_name for c in suite.components]: 

644 filename = "%s/dists/%s/%s/source/Sources" % (suite.archive.path, suite_name, component) 

645 filename = utils.find_possibly_compressed_file(filename) 

646 with apt_pkg.TagFile(filename) as Sources: 

647 while Sources.step(): 

648 source = Sources.section.find('Package') 

649 source_version = Sources.section.find('Version') 

650 architecture = Sources.section.find('Architecture') 

651 binaries = Sources.section.find('Binary') 

652 binaries_list = [i.strip() for i in binaries.split(',')] 

653 

654 if "bnb" in checks: 654 ↛ 656line 654 didn't jump to line 656, because the condition on line 654 was never true

655 # Check for binaries not built on any architecture. 

656 for binary in binaries_list: 

657 if binary not in bins_in_suite: 

658 bin_not_built[source].add(binary) 

659 

660 if "anais" in checks: 660 ↛ 661line 660 didn't jump to line 661, because the condition on line 660 was never true

661 anais_output += do_anais(architecture, binaries_list, source, session) 

662 

663 # build indices for checking "no source" later 

664 source_index = component + '/' + source 

665 src_pkgs[source] = source_index 

666 for binary in binaries_list: 

667 bin_pkgs[binary] = source 

668 source_binaries[source] = binaries 

669 source_versions[source] = source_version 

670 

671 # Checks based on the Packages files 

672 check_components = components[:] 

673 if suite_name != "experimental": 673 ↛ 676line 673 didn't jump to line 676, because the condition on line 673 was never false

674 check_components.append('main/debian-installer') 

675 

676 for component in check_components: 

677 architectures = [a.arch_string for a in get_suite_architectures(suite_name, 

678 skipsrc=True, skipall=True, 

679 session=session)] 

680 for architecture in architectures: 

681 if component == 'main/debian-installer' and re.match("kfreebsd", architecture): 681 ↛ 682line 681 didn't jump to line 682, because the condition on line 681 was never true

682 continue 

683 

684 if "nfu" in checks: 684 ↛ 687line 684 didn't jump to line 687, because the condition on line 684 was never false

685 nfu_entries = parse_nfu(architecture) 

686 

687 filename = "%s/dists/%s/%s/binary-%s/Packages" % (suite.archive.path, suite_name, component, architecture) 

688 filename = utils.find_possibly_compressed_file(filename) 

689 with apt_pkg.TagFile(filename) as Packages: 

690 while Packages.step(): 

691 package = Packages.section.find('Package') 

692 source = Packages.section.find('Source', "") 

693 version = Packages.section.find('Version') 

694 if source == "": 

695 source = package 

696 if package in bin2source and \ 696 ↛ 698line 696 didn't jump to line 698, because the condition on line 696 was never true

697 apt_pkg.version_compare(version, bin2source[package]["version"]) > 0: 

698 bin2source[package]["version"] = version 

699 bin2source[package]["source"] = source 

700 else: 

701 bin2source[package] = {} 

702 bin2source[package]["version"] = version 

703 bin2source[package]["source"] = source 

704 if source.find("(") != -1: 704 ↛ 705line 704 didn't jump to line 705, because the condition on line 704 was never true

705 m = re_extract_src_version.match(source) 

706 source = m.group(1) 

707 version = m.group(2) 

708 if package not in bin_pkgs: 708 ↛ 709line 708 didn't jump to line 709, because the condition on line 708 was never true

709 nbs[source][package].add(version) 

710 else: 

711 if "nfu" in checks: 711 ↛ 690line 711 didn't jump to line 690, because the condition on line 711 was never false

712 if package in nfu_entries and \ 712 ↛ 714line 712 didn't jump to line 714, because the condition on line 712 was never true

713 version != source_versions[source]: # only suggest to remove out-of-date packages 

714 nfu_packages[architecture].append((package, version, source_versions[source])) 

715 

716 # Distinguish dubious (version numbers match) and 'real' NBS (they don't) 

717 dubious_nbs = defaultdict(lambda: defaultdict(set)) 717 ↛ exitline 717 didn't run the lambda on line 717

718 version_sort_key = functools.cmp_to_key(apt_pkg.version_compare) 

719 for source in nbs: 719 ↛ 720line 719 didn't jump to line 720, because the loop on line 719 never started

720 for package in nbs[source]: 

721 latest_version = max(nbs[source][package], key=version_sort_key) 

722 source_version = source_versions.get(source, "0") 

723 if apt_pkg.version_compare(latest_version, source_version) == 0: 

724 add_nbs(dubious_nbs, source, latest_version, package, suite_id, session) 

725 

726 if "nviu" in checks: 726 ↛ 729line 726 didn't jump to line 729, because the condition on line 726 was never false

727 do_newer_version('unstable', 'experimental', 'NVIU', session) 

728 

729 if "nvit" in checks: 729 ↛ 734line 729 didn't jump to line 734, because the condition on line 729 was never false

730 do_newer_version('testing', 'testing-proposed-updates', 'NVIT', session) 

731 

732 ### 

733 

734 if Options["Mode"] == "full": 734 ↛ 735line 734 didn't jump to line 735, because the condition on line 734 was never true

735 print_info("=" * 75) 

736 print_info() 

737 

738 if "nfu" in checks: 738 ↛ 741line 738 didn't jump to line 741, because the condition on line 738 was never false

739 do_nfu(nfu_packages) 

740 

741 if "bnb" in checks: 741 ↛ 742line 741 didn't jump to line 742, because the condition on line 741 was never true

742 print_info("Unbuilt binary packages") 

743 print_info("-----------------------") 

744 print_info() 

745 for source in sorted(bin_not_built): 

746 binaries = sorted(bin_not_built[source]) 

747 print_info(" o %s: %s" % (source, ", ".join(binaries))) 

748 print_info() 

749 

750 if "bms" in checks: 750 ↛ 751line 750 didn't jump to line 751, because the condition on line 750 was never true

751 report_multiple_source(suite) 

752 

753 if "anais" in checks: 753 ↛ 754line 753 didn't jump to line 754, because the condition on line 753 was never true

754 print_info("Architecture Not Allowed In Source") 

755 print_info("----------------------------------") 

756 print_info(anais_output) 

757 print_info() 

758 

759 if "dubious nbs" in checks: 759 ↛ 760line 759 didn't jump to line 760, because the condition on line 759 was never true

760 do_dubious_nbs(dubious_nbs) 

761 

762 

763################################################################################ 

764 

765if __name__ == '__main__': 

766 main()