Coverage for dak/cruft_report.py: 45%

1#! /usr/bin/env python3

3"""

4Check for obsolete binary packages

6@contact: Debian FTP Master <ftpmaster@debian.org>

7@copyright: 2000-2006 James Troup <james@nocrew.org>

8@copyright: 2009 Torsten Werner <twerner@debian.org>

9@license: GNU General Public License version 2 or later

10"""

12# This program is free software; you can redistribute it and/or modify

13# it under the terms of the GNU General Public License as published by

14# the Free Software Foundation; either version 2 of the License, or

15# (at your option) any later version.

17# This program is distributed in the hope that it will be useful,

18# but WITHOUT ANY WARRANTY; without even the implied warranty of

19# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

20# GNU General Public License for more details.

22# You should have received a copy of the GNU General Public License

23# along with this program; if not, write to the Free Software

24# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

26################################################################################

28# ``If you're claiming that's a "problem" that needs to be "fixed",

29# you might as well write some letters to God about how unfair entropy

30# is while you're at it.'' -- 20020802143104.GA5628@azure.humbug.org.au

32## TODO: fix NBS looping for version, implement Dubious NBS, fix up output of

33## duplicate source package stuff, improve experimental ?, add overrides,

34## avoid ANAIS for duplicated packages

36################################################################################

38import functools

39import os

40import re

41import sys

42from collections import defaultdict

44import apt_pkg

46from daklib import utils

47from daklib.config import Config

48from daklib.cruft import (

49 newer_version,

50 query_without_source,

51 queryNBS,

52 queryNBS_metadata,

53 report_multiple_source,

54)

55from daklib.dbconn import DBConn, get_suite, get_suite_architectures

56from daklib.regexes import re_extract_src_version

58################################################################################

60no_longer_in_suite = {} # Really should be static to add_nbs, but I'm lazy

62source_binaries = {}

63source_versions = {}

65################################################################################

68def usage(exit_code=0):

69 print(

70 """Usage: dak cruft-report

71Check for obsolete or duplicated packages.

73 -h, --help show this help and exit.

74 -m, --mode=MODE chose the MODE to run in (full, daily, bdo).

75 -s, --suite=SUITE check suite SUITE.

76 -R, --rdep-check check reverse dependencies

77 -w, --wanna-build-dump where to find the copies of https://buildd.debian.org/stats/*.txt"""

78 )

79 sys.exit(exit_code)

82################################################################################

85def print_info(s=""):

86 cnf = Config()

88 if cnf.subtree("Cruft-Report::Options")["Commands-Only"]: 88 ↛ 89line 88 didn't jump to line 89, because the condition on line 88 was never true

89 return

91 print(s)

94def print_cmd(s, indent=4):

95 cnf = Config()

97 # Indent if doing the human readable display

98 if not cnf.subtree("Cruft-Report::Options")["Commands-Only"]: 98 ↛ 102line 98 didn't jump to line 102, because the condition on line 98 was never false

99 ind = " " * indent

100 s = ind + s

101

102 print(s)

103

104

105################################################################################

106

107

108def add_nbs(nbs_d, source, version, package, suite_id, session):

109 # Ensure the package is still in the suite (someone may have already removed it)

110 if package in no_longer_in_suite:

111 return

112 else:

113 q = session.execute(

114 """SELECT b.id FROM binaries b, bin_associations ba

115 WHERE ba.bin = b.id AND ba.suite = :suite_id

116 AND b.package = :package LIMIT 1""",

117 {"suite_id": suite_id, "package": package},

118 )

119 if not q.fetchall():

120 no_longer_in_suite[package] = ""

121 return

122

123 nbs_d[source][version].add(package)

124

125

126################################################################################

127

128# Check for packages built on architectures they shouldn't be.

129

130

131def do_anais(architecture, binaries_list, source, session):

132 if architecture == "any" or architecture == "all":

133 return ""

134

135 version_sort_key = functools.cmp_to_key(apt_pkg.version_compare)

136 anais_output = ""

137 architectures = {}

138 for arch in architecture.split():

139 architectures[arch.strip()] = ""

140 for binary in binaries_list:

141 q = session.execute(

142 """SELECT a.arch_string, b.version

143 FROM binaries b, bin_associations ba, architecture a

144 WHERE ba.suite = :suiteid AND ba.bin = b.id

145 AND b.architecture = a.id AND b.package = :package""",

146 {"suiteid": suite_id, "package": binary},

147 )

148 ql = q.fetchall()

149 versions = []

150 for arch, version in ql:

151 if arch in architectures:

152 versions.append(version)

153 versions.sort(key=version_sort_key)

154 if versions:

155 latest_version = versions.pop()

156 else:

157 latest_version = None

158 # Check for 'invalid' architectures

159 versions_d = defaultdict(list)

160 for arch, version in ql:

161 if arch not in architectures:

162 versions_d[version].append(arch)

163

164 if versions_d:

165 anais_output += "\n (*) %s_%s [%s]: %s\n" % (

166 binary,

167 latest_version,

168 source,

169 architecture,

170 )

171 for version in sorted(versions_d, key=version_sort_key):

172 arches = sorted(versions_d[version])

173 anais_output += " o %s: %s\n" % (version, ", ".join(arches))

174 return anais_output

175

176

177################################################################################

178

179

180# Check for out-of-date binaries on architectures that do not want to build that

181# package any more, and have them listed as Not-For-Us

182def do_nfu(nfu_packages):

183 output = ""

184

185 a2p = {}

186

187 for architecture in nfu_packages: 187 ↛ 188line 187 didn't jump to line 188, because the loop on line 187 never started

188 a2p[architecture] = []

189 for package, bver, sver in nfu_packages[architecture]:

190 output += " * [%s] does not want %s (binary %s, source %s)\n" % (

191 architecture,

192 package,

193 bver,

194 sver,

195 )

196 a2p[architecture].append(package)

197

198 if output: 198 ↛ 199line 198 didn't jump to line 199, because the condition on line 198 was never true

199 print_info("Obsolete by Not-For-Us")

200 print_info("----------------------")

201 print_info()

202 print_info(output)

203

204 print_info("Suggested commands:")

205 for architecture in a2p:

206 if a2p[architecture]:

207 print_cmd(

208 (

209 'dak rm -o -m "[auto-cruft] NFU" -s %s -a %s -b %s'

210 % (suite.suite_name, architecture, " ".join(a2p[architecture]))

211 ),

212 indent=1,

213 )

214 print_info()

215

216

217def parse_nfu(architecture):

218 cnf = Config()

219 # utils/hpodder_1.1.5.0: Not-For-Us [optional:out-of-date]

220 r = re.compile(r"^\w+/([^_]+)_.*: Not-For-Us")

221

222 ret = set()

223

224 filename = "%s/%s-all.txt" % (

225 cnf["Cruft-Report::Options::Wanna-Build-Dump"],

226 architecture,

227 )

228

229 # Not all architectures may have a wanna-build dump, so we want to ignore missin

230 # files

231 if os.path.exists(filename): 231 ↛ 232line 231 didn't jump to line 232, because the condition on line 231 was never true

232 with open(filename) as f:

233 for line in f:

234 if line[0] == " ":

235 continue

236

237 m = r.match(line)

238 if m:

239 ret.add(m.group(1))

240 else:

241 utils.warn("No wanna-build dump file for architecture %s" % architecture)

242 return ret

243

244

245################################################################################

246

247

248def do_newer_version(lowersuite_name, highersuite_name, code, session):

249 list = newer_version(lowersuite_name, highersuite_name, session)

250 if len(list) > 0: 250 ↛ 251line 250 didn't jump to line 251, because the condition on line 250 was never true

251 nv_to_remove = []

252 title = "Newer version in %s" % lowersuite_name

253 print_info(title)

254 print_info("-" * len(title))

255 print_info()

256 for i in list:

257 (source, higher_version, lower_version) = i

258 print_info(" o %s (%s, %s)" % (source, higher_version, lower_version))

259 nv_to_remove.append(source)

260 print_info()

261 print_info("Suggested command:")

262 print_cmd(

263 'dak rm -m "[auto-cruft] %s" -s %s %s'

264 % (code, highersuite_name, " ".join(nv_to_remove)),

265 indent=1,

266 )

267 print_info()

268

269

270################################################################################

271

272

273def reportWithoutSource(suite_name, suite_id, session, rdeps=False):

274 rows = query_without_source(suite_id, session)

275 title = "packages without source in suite %s" % suite_name

276 if rows.rowcount > 0: 276 ↛ 277line 276 didn't jump to line 277, because the condition on line 276 was never true

277 print_info("%s\n%s\n" % (title, "-" * len(title)))

278 message = '"[auto-cruft] no longer built from source"'

279 for row in rows: 279 ↛ 280line 279 didn't jump to line 280, because the loop on line 279 never started

280 (package, version) = row

281 print_info(

282 "* package %s in version %s is no longer built from source"

283 % (package, version)

284 )

285 print_info(" - suggested command:")

286 print_cmd(

287 "dak rm -m %s -s %s -a all -p -R -b %s" % (message, suite_name, package)

288 )

289 if rdeps:

290 if utils.check_reverse_depends([package], suite_name, [], session, True):

291 print_info()

292 else:

293 print_info(" - No dependency problem found\n")

294 else:

295 print_info()

296

297

298def queryNewerAll(suite_name, session):

299 """searches for arch != all packages that have an arch == all

300 package with a higher version in the same suite"""

301

302 query = """

303select bab1.package, bab1.version as oldver,

304 array_to_string(array_agg(a.arch_string), ',') as oldarch,

305 bab2.version as newver

306 from bin_associations_binaries bab1

307 join bin_associations_binaries bab2

308 on bab1.package = bab2.package and bab1.version < bab2.version and

309 bab1.suite = bab2.suite and bab1.architecture > 2 and

310 bab2.architecture = 2

311 join architecture a on bab1.architecture = a.id

312 join suite s on bab1.suite = s.id

313 where s.suite_name = :suite_name

314 group by bab1.package, oldver, bab1.suite, newver"""

315 return session.execute(query, {"suite_name": suite_name})

316

317

318def reportNewerAll(suite_name, session):

319 rows = queryNewerAll(suite_name, session)

320 title = "obsolete arch any packages in suite %s" % suite_name

321 if rows.rowcount > 0: 321 ↛ 322line 321 didn't jump to line 322, because the condition on line 321 was never true

322 print_info("%s\n%s\n" % (title, "-" * len(title)))

323 message = '"[auto-cruft] obsolete arch any package"'

324 for row in rows: 324 ↛ 325line 324 didn't jump to line 325, because the loop on line 324 never started

325 (package, oldver, oldarch, newver) = row

326 print_info(

327 "* package %s is arch any in version %s but arch all in version %s"

328 % (package, oldver, newver)

329 )

330 print_info(" - suggested command:")

331 print_cmd(

332 "dak rm -o -m %s -s %s -a %s -p -b %s\n"

333 % (message, suite_name, oldarch, package)

334 )

335

336

337def reportNBS(suite_name, suite_id, rdeps=False):

338 session = DBConn().session()

339 nbsRows = queryNBS(suite_id, session)

340 title = "NBS packages in suite %s" % suite_name

341 if nbsRows.rowcount > 0: 341 ↛ 342line 341 didn't jump to line 342, because the condition on line 341 was never true

342 print_info("%s\n%s\n" % (title, "-" * len(title)))

343 for row in nbsRows: 343 ↛ 344line 343 didn't jump to line 344, because the loop on line 343 never started

344 (pkg_list, arch_list, source, version) = row

345 pkg_string = " ".join(pkg_list)

346 arch_string = ",".join(arch_list)

347 print_info(

348 "* source package %s version %s no longer builds" % (source, version)

349 )

350 print_info(" binary package(s): %s" % pkg_string)

351 print_info(" on %s" % arch_string)

352 print_info(" - suggested command:")

353 message = '"[auto-cruft] NBS (no longer built by %s)"' % source

354 print_cmd(

355 "dak rm -o -m %s -s %s -a %s -p -R -b %s"

356 % (message, suite_name, arch_string, pkg_string)

357 )

358 if rdeps:

359 if utils.check_reverse_depends(

360 pkg_list, suite_name, arch_list, session, True

361 ):

362 print_info()

363 else:

364 print_info(" - No dependency problem found\n")

365 else:

366 print_info()

367 session.close()

368

369

370def reportNBSMetadata(suite_name, suite_id, session, rdeps=False):

371 rows = queryNBS_metadata(suite_id, session)

372 title = "NBS packages (from metadata) in suite %s" % suite_name

373 if rows.rowcount > 0: 373 ↛ 375line 373 didn't jump to line 375, because the condition on line 373 was never false

374 print_info("%s\n%s\n" % (title, "-" * len(title)))

375 for row in rows:

376 (packages, architecture, source, version) = row

377 print_info(

378 "* source package %s version %s no longer builds" % (source, version)

379 )

380 print_info(" binary package(s): %s" % packages)

381 print_info(" on %s" % architecture)

382 print_info(" - suggested command:")

383 message = (

384 '"[auto-cruft] NBS (no longer built by %s - based on source metadata)"'

385 % source

386 )

387 print_cmd(

388 "dak rm -o -m %s -s %s -a %s -p -R -b %s"

389 % (message, suite_name, architecture, packages)

390 )

391 if rdeps: 391 ↛ 392line 391 didn't jump to line 392, because the condition on line 391 was never true

392 archs = [architecture]

393 if architecture == "all":

394 # when archs is None, rdeps are checked on all archs in the suite

395 archs = None

396 if utils.check_reverse_depends(

397 packages.split(), suite_name, archs, session, True

398 ):

399 print_info()

400 else:

401 print_info(" - No dependency problem found\n")

402 else:

403 print_info()

404

405

406def reportAllNBS(suite_name, suite_id, session, rdeps=False):

407 reportWithoutSource(suite_name, suite_id, session, rdeps)

408 reportNewerAll(suite_name, session)

409 reportNBS(suite_name, suite_id, rdeps)

410

411

412################################################################################

413

414

415def do_dubious_nbs(dubious_nbs):

416 print_info("Dubious NBS")

417 print_info("-----------")

418 print_info()

419

420 version_sort_key = functools.cmp_to_key(apt_pkg.version_compare)

421 for source in sorted(dubious_nbs):

422 print_info(

423 " * %s_%s builds: %s"

424 % (

425 source,

426 source_versions.get(source, "??"),

427 source_binaries.get(source, "(source does not exist)"),

428 )

429 )

430 print_info(" won't admit to building:")

431 versions = sorted(dubious_nbs[source], key=version_sort_key)

432 for version in versions:

433 packages = sorted(dubious_nbs[source][version])

434 print_info(" o %s: %s" % (version, ", ".join(packages)))

435

436 print_info()

437

438

439################################################################################

440

441

442def obsolete_source(suite_name, session):

443 """returns obsolete source packages for suite_name without binaries

444 in the same suite sorted by install_date; install_date should help

445 detecting source only (or binary throw away) uploads; duplicates in

446 the suite are skipped

447

448 subquery 'source_suite_unique' returns source package names from

449 suite without duplicates; the rationale behind is that neither

450 cruft-report nor rm cannot handle duplicates (yet)"""

451

452 query = """

453WITH source_suite_unique AS

454 (SELECT source, suite

455 FROM source_suite GROUP BY source, suite HAVING count(*) = 1)

456SELECT ss.src, ss.source, ss.version,

457 to_char(ss.install_date, 'YYYY-MM-DD') AS install_date

458 FROM source_suite ss

459 JOIN source_suite_unique ssu

460 ON ss.source = ssu.source AND ss.suite = ssu.suite

461 JOIN suite s ON s.id = ss.suite

462 LEFT JOIN bin_associations_binaries bab

463 ON ss.src = bab.source AND ss.suite = bab.suite

464 WHERE s.suite_name = :suite_name AND bab.id IS NULL

465 AND now() - ss.install_date > '1 day'::interval

466 ORDER BY install_date"""

467 args = {"suite_name": suite_name}

468 return session.execute(query, args)

469

470

471def source_bin(source, session):

472 """returns binaries built by source for all or no suite grouped and

473 ordered by package name"""

474

475 query = """

476SELECT b.package

477 FROM binaries b

478 JOIN src_associations_src sas ON b.source = sas.src

479 WHERE sas.source = :source

480 GROUP BY b.package

481 ORDER BY b.package"""

482 args = {"source": source}

483 return session.execute(query, args)

484

485

486def newest_source_bab(suite_name, package, session):

487 """returns newest source that builds binary package in suite grouped

488 and sorted by source and package name"""

489

490 query = """

491SELECT sas.source, MAX(sas.version) AS srcver

492 FROM src_associations_src sas

493 JOIN bin_associations_binaries bab ON sas.src = bab.source

494 JOIN suite s on s.id = bab.suite

495 WHERE s.suite_name = :suite_name AND bab.package = :package

496 GROUP BY sas.source, bab.package

497 ORDER BY sas.source, bab.package"""

498 args = {"suite_name": suite_name, "package": package}

499 return session.execute(query, args)

500

501

502def report_obsolete_source(suite_name, session):

503 rows = obsolete_source(suite_name, session)

504 if rows.rowcount == 0: 504 ↛ 506line 504 didn't jump to line 506, because the condition on line 504 was never false

505 return

506 print_info(

507 """Obsolete source packages in suite %s

508----------------------------------%s\n"""

509 % (suite_name, "-" * len(suite_name))

510 )

511 for os_row in rows.fetchall():

512 (src, old_source, version, install_date) = os_row

513 print_info(

514 " * obsolete source %s version %s installed at %s"

515 % (old_source, version, install_date)

516 )

517 for sb_row in source_bin(old_source, session):

518 (package,) = sb_row

519 print_info(" - has built binary %s" % package)

520 for nsb_row in newest_source_bab(suite_name, package, session):

521 (new_source, srcver) = nsb_row

522 print_info(

523 " currently built by source %s version %s"

524 % (new_source, srcver)

525 )

526 print_info(" - suggested command:")

527 rm_opts = '-S -p -m "[auto-cruft] obsolete source package"'

528 print_cmd("dak rm -s %s %s %s\n" % (suite_name, rm_opts, old_source))

529

530

531def get_suite_binaries(suite, session):

532 # Initalize a large hash table of all binary packages

533 binaries = {}

534

535 print_info("Getting a list of binary packages in %s..." % suite.suite_name)

536 q = session.execute(

537 """SELECT distinct b.package

538 FROM binaries b, bin_associations ba

539 WHERE ba.suite = :suiteid AND ba.bin = b.id""",

540 {"suiteid": suite.suite_id},

541 )

542 for i in q.fetchall():

543 binaries[i[0]] = ""

544

545 return binaries

546

547

548################################################################################

549

550

551def report_outdated_nonfree(suite, session, rdeps=False):

552

553 packages = {}

554 query = """WITH outdated_sources AS (

555 SELECT s.source, s.version, s.id

556 FROM source s

557 JOIN src_associations sa ON sa.source = s.id

558 WHERE sa.suite IN (

559 SELECT id

560 FROM suite

561 WHERE suite_name = :suite )

562 AND sa.created < (now() - interval :delay)

563 EXCEPT SELECT s.source, max(s.version) AS version, max(s.id)

564 FROM source s

565 JOIN src_associations sa ON sa.source = s.id

566 WHERE sa.suite IN (

567 SELECT id

568 FROM suite

569 WHERE suite_name = :suite )

570 AND sa.created < (now() - interval :delay)

571 GROUP BY s.source ),

572 binaries AS (

573 SELECT b.package, s.source, (

574 SELECT a.arch_string

575 FROM architecture a

576 WHERE a.id = b.architecture ) AS arch

577 FROM binaries b

578 JOIN outdated_sources s ON s.id = b.source

579 JOIN bin_associations ba ON ba.bin = b.id

580 JOIN override o ON o.package = b.package AND o.suite = ba.suite

581 WHERE ba.suite IN (

582 SELECT id

583 FROM suite

584 WHERE suite_name = :suite )

585 AND o.component IN (

586 SELECT id

587 FROM component

588 WHERE name = 'non-free' ) )

589 SELECT DISTINCT package, source, arch

590 FROM binaries

591 ORDER BY source, package, arch"""

592

593 res = session.execute(query, {"suite": suite, "delay": "'15 days'"})

594 for package in res: 594 ↛ 595line 594 didn't jump to line 595, because the loop on line 594 never started

595 binary = package[0]

596 source = package[1]

597 arch = package[2]

598 if arch == "all":

599 continue

600 if source not in packages:

601 packages[source] = {}

602 if binary not in packages[source]:

603 packages[source][binary] = set()

604 packages[source][binary].add(arch)

605 if packages: 605 ↛ 606line 605 didn't jump to line 606, because the condition on line 605 was never true

606 title = "Outdated non-free binaries in suite %s" % suite

607 message = '"[auto-cruft] outdated non-free binaries"'

608 print_info("%s\n%s\n" % (title, "-" * len(title)))

609 for source in sorted(packages):

610 archs = set()

611 binaries = set()

612 print_info("* package %s has outdated non-free binaries" % source)

613 print_info(" - suggested command:")

614 for binary in sorted(packages[source]):

615 binaries.add(binary)

616 archs = archs.union(packages[source][binary])

617 print_cmd(

618 "dak rm -o -m %s -s %s -a %s -p -R -b %s"

619 % (message, suite, ",".join(archs), " ".join(binaries))

620 )

621 if rdeps:

622 if utils.check_reverse_depends(

623 list(binaries), suite, archs, session, True

624 ):

625 print_info()

626 else:

627 print_info(" - No dependency problem found\n")

628 else:

629 print_info()

630

631

632################################################################################

633

634

635def main():

636 global suite, suite_id, source_binaries, source_versions

637

638 cnf = Config()

639

640 Arguments = [

641 ("h", "help", "Cruft-Report::Options::Help"),

642 ("m", "mode", "Cruft-Report::Options::Mode", "HasArg"),

643 ("R", "rdep-check", "Cruft-Report::Options::Rdep-Check"),

644 ("s", "suite", "Cruft-Report::Options::Suite", "HasArg"),

645 ("w", "wanna-build-dump", "Cruft-Report::Options::Wanna-Build-Dump", "HasArg"),

646 ("c", "commands-only", "Cruft-Report::Options::Commands-Only"),

647 ]

648 for i in ["help", "Rdep-Check"]:

649 key = "Cruft-Report::Options::%s" % i

650 if key not in cnf: 650 ↛ 648line 650 didn't jump to line 648, because the condition on line 650 was never false

651 cnf[key] = ""

652

653 if "Cruft-Report::Options::Commands-Only" not in cnf: 653 ↛ 656line 653 didn't jump to line 656, because the condition on line 653 was never false

654 cnf["Cruft-Report::Options::Commands-Only"] = ""

655

656 cnf["Cruft-Report::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable")

657

658 if "Cruft-Report::Options::Mode" not in cnf: 658 ↛ 661line 658 didn't jump to line 661, because the condition on line 658 was never false

659 cnf["Cruft-Report::Options::Mode"] = "daily"

660

661 if "Cruft-Report::Options::Wanna-Build-Dump" not in cnf: 661 ↛ 666line 661 didn't jump to line 666, because the condition on line 661 was never false

662 cnf["Cruft-Report::Options::Wanna-Build-Dump"] = (

663 "/srv/ftp-master.debian.org/scripts/nfu"

664 )

665

666 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)