1#! /usr/bin/env python3
3"""
4Check for obsolete binary packages
6@contact: Debian FTP Master <ftpmaster@debian.org>
7@copyright: 2000-2006 James Troup <james@nocrew.org>
8@copyright: 2009 Torsten Werner <twerner@debian.org>
9@license: GNU General Public License version 2 or later
10"""
12# This program is free software; you can redistribute it and/or modify
13# it under the terms of the GNU General Public License as published by
14# the Free Software Foundation; either version 2 of the License, or
15# (at your option) any later version.
17# This program is distributed in the hope that it will be useful,
18# but WITHOUT ANY WARRANTY; without even the implied warranty of
19# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20# GNU General Public License for more details.
22# You should have received a copy of the GNU General Public License
23# along with this program; if not, write to the Free Software
24# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26################################################################################
28# ``If you're claiming that's a "problem" that needs to be "fixed",
29# you might as well write some letters to God about how unfair entropy
30# is while you're at it.'' -- 20020802143104.GA5628@azure.humbug.org.au
32## TODO: fix NBS looping for version, implement Dubious NBS, fix up output of
33## duplicate source package stuff, improve experimental ?, add overrides,
34## avoid ANAIS for duplicated packages
36################################################################################
38import functools
39import os
40import sys
41import re
42import apt_pkg
43from collections import defaultdict
45from daklib.config import Config
46from daklib.dbconn import *
47from daklib import utils
48from daklib.regexes import re_extract_src_version
49from daklib.cruft import *
51################################################################################
53no_longer_in_suite = {} # Really should be static to add_nbs, but I'm lazy
55source_binaries = {}
56source_versions = {}
58################################################################################
61def usage(exit_code=0):
62 print("""Usage: dak cruft-report
63Check for obsolete or duplicated packages.
65 -h, --help show this help and exit.
66 -m, --mode=MODE chose the MODE to run in (full, daily, bdo).
67 -s, --suite=SUITE check suite SUITE.
68 -R, --rdep-check check reverse dependencies
69 -w, --wanna-build-dump where to find the copies of https://buildd.debian.org/stats/*.txt""")
70 sys.exit(exit_code)
72################################################################################
75def print_info(s=""):
76 cnf = Config()
78 if cnf.subtree("Cruft-Report::Options")["Commands-Only"]: 78 ↛ 79line 78 didn't jump to line 79, because the condition on line 78 was never true
79 return
81 print(s)
84def print_cmd(s, indent=4):
85 cnf = Config()
87 # Indent if doing the human readable display
88 if not cnf.subtree("Cruft-Report::Options")["Commands-Only"]: 88 ↛ 92line 88 didn't jump to line 92, because the condition on line 88 was never false
89 ind = " " * indent
90 s = ind + s
92 print(s)
95################################################################################
98def add_nbs(nbs_d, source, version, package, suite_id, session):
99 # Ensure the package is still in the suite (someone may have already removed it)
100 if package in no_longer_in_suite:
101 return
102 else:
103 q = session.execute("""SELECT b.id FROM binaries b, bin_associations ba
104 WHERE ba.bin = b.id AND ba.suite = :suite_id
105 AND b.package = :package LIMIT 1""", {'suite_id': suite_id,
106 'package': package})
107 if not q.fetchall():
108 no_longer_in_suite[package] = ""
109 return
111 nbs_d[source][version].add(package)
113################################################################################
115# Check for packages built on architectures they shouldn't be.
118def do_anais(architecture, binaries_list, source, session):
119 if architecture == "any" or architecture == "all":
120 return ""
122 version_sort_key = functools.cmp_to_key(apt_pkg.version_compare)
123 anais_output = ""
124 architectures = {}
125 for arch in architecture.split():
126 architectures[arch.strip()] = ""
127 for binary in binaries_list:
128 q = session.execute("""SELECT a.arch_string, b.version
129 FROM binaries b, bin_associations ba, architecture a
130 WHERE ba.suite = :suiteid AND ba.bin = b.id
131 AND b.architecture = a.id AND b.package = :package""",
132 {'suiteid': suite_id, 'package': binary})
133 ql = q.fetchall()
134 versions = []
135 for arch, version in ql:
136 if arch in architectures:
137 versions.append(version)
138 versions.sort(key=version_sort_key)
139 if versions:
140 latest_version = versions.pop()
141 else:
142 latest_version = None
143 # Check for 'invalid' architectures
144 versions_d = defaultdict(list)
145 for arch, version in ql:
146 if arch not in architectures:
147 versions_d[version].append(arch)
149 if versions_d:
150 anais_output += "\n (*) %s_%s [%s]: %s\n" % (binary, latest_version, source, architecture)
151 for version in sorted(versions_d, key=version_sort_key):
152 arches = sorted(versions_d[version])
153 anais_output += " o %s: %s\n" % (version, ", ".join(arches))
154 return anais_output
157################################################################################
159# Check for out-of-date binaries on architectures that do not want to build that
160# package any more, and have them listed as Not-For-Us
161def do_nfu(nfu_packages):
162 output = ""
164 a2p = {}
166 for architecture in nfu_packages: 166 ↛ 167line 166 didn't jump to line 167, because the loop on line 166 never started
167 a2p[architecture] = []
168 for (package, bver, sver) in nfu_packages[architecture]:
169 output += " * [%s] does not want %s (binary %s, source %s)\n" % (architecture, package, bver, sver)
170 a2p[architecture].append(package)
172 if output: 172 ↛ 173line 172 didn't jump to line 173, because the condition on line 172 was never true
173 print_info("Obsolete by Not-For-Us")
174 print_info("----------------------")
175 print_info()
176 print_info(output)
178 print_info("Suggested commands:")
179 for architecture in a2p:
180 if a2p[architecture]:
181 print_cmd(("dak rm -o -m \"[auto-cruft] NFU\" -s %s -a %s -b %s" %
182 (suite.suite_name, architecture, " ".join(a2p[architecture]))), indent=1)
183 print_info()
186def parse_nfu(architecture):
187 cnf = Config()
188 # utils/hpodder_1.1.5.0: Not-For-Us [optional:out-of-date]
189 r = re.compile(r"^\w+/([^_]+)_.*: Not-For-Us")
191 ret = set()
193 filename = "%s/%s-all.txt" % (cnf["Cruft-Report::Options::Wanna-Build-Dump"], architecture)
195 # Not all architectures may have a wanna-build dump, so we want to ignore missin
196 # files
197 if os.path.exists(filename): 197 ↛ 198line 197 didn't jump to line 198, because the condition on line 197 was never true
198 with open(filename) as f:
199 for line in f:
200 if line[0] == ' ':
201 continue
203 m = r.match(line)
204 if m:
205 ret.add(m.group(1))
206 else:
207 utils.warn("No wanna-build dump file for architecture %s" % architecture)
208 return ret
210################################################################################
213def do_newer_version(lowersuite_name, highersuite_name, code, session):
214 list = newer_version(lowersuite_name, highersuite_name, session)
215 if len(list) > 0: 215 ↛ 216line 215 didn't jump to line 216, because the condition on line 215 was never true
216 nv_to_remove = []
217 title = "Newer version in %s" % lowersuite_name
218 print_info(title)
219 print_info("-" * len(title))
220 print_info()
221 for i in list:
222 (source, higher_version, lower_version) = i
223 print_info(" o %s (%s, %s)" % (source, higher_version, lower_version))
224 nv_to_remove.append(source)
225 print_info()
226 print_info("Suggested command:")
227 print_cmd("dak rm -m \"[auto-cruft] %s\" -s %s %s" % (code, highersuite_name,
228 " ".join(nv_to_remove)), indent=1)
229 print_info()
231################################################################################
234def reportWithoutSource(suite_name, suite_id, session, rdeps=False):
235 rows = query_without_source(suite_id, session)
236 title = 'packages without source in suite %s' % suite_name
237 if rows.rowcount > 0: 237 ↛ 238line 237 didn't jump to line 238, because the condition on line 237 was never true
238 print_info('%s\n%s\n' % (title, '-' * len(title)))
239 message = '"[auto-cruft] no longer built from source"'
240 for row in rows: 240 ↛ 241line 240 didn't jump to line 241, because the loop on line 240 never started
241 (package, version) = row
242 print_info("* package %s in version %s is no longer built from source" %
243 (package, version))
244 print_info(" - suggested command:")
245 print_cmd("dak rm -m %s -s %s -a all -p -R -b %s" %
246 (message, suite_name, package))
247 if rdeps:
248 if utils.check_reverse_depends([package], suite_name, [], session, True):
249 print_info()
250 else:
251 print_info(" - No dependency problem found\n")
252 else:
253 print_info()
256def queryNewerAll(suite_name, session):
257 """searches for arch != all packages that have an arch == all
258 package with a higher version in the same suite"""
260 query = """
261select bab1.package, bab1.version as oldver,
262 array_to_string(array_agg(a.arch_string), ',') as oldarch,
263 bab2.version as newver
264 from bin_associations_binaries bab1
265 join bin_associations_binaries bab2
266 on bab1.package = bab2.package and bab1.version < bab2.version and
267 bab1.suite = bab2.suite and bab1.architecture > 2 and
268 bab2.architecture = 2
269 join architecture a on bab1.architecture = a.id
270 join suite s on bab1.suite = s.id
271 where s.suite_name = :suite_name
272 group by bab1.package, oldver, bab1.suite, newver"""
273 return session.execute(query, {'suite_name': suite_name})
276def reportNewerAll(suite_name, session):
277 rows = queryNewerAll(suite_name, session)
278 title = 'obsolete arch any packages in suite %s' % suite_name
279 if rows.rowcount > 0: 279 ↛ 280line 279 didn't jump to line 280, because the condition on line 279 was never true
280 print_info('%s\n%s\n' % (title, '-' * len(title)))
281 message = '"[auto-cruft] obsolete arch any package"'
282 for row in rows: 282 ↛ 283line 282 didn't jump to line 283, because the loop on line 282 never started
283 (package, oldver, oldarch, newver) = row
284 print_info("* package %s is arch any in version %s but arch all in version %s" %
285 (package, oldver, newver))
286 print_info(" - suggested command:")
287 print_cmd("dak rm -o -m %s -s %s -a %s -p -b %s\n" %
288 (message, suite_name, oldarch, package))
291def reportNBS(suite_name, suite_id, rdeps=False):
292 session = DBConn().session()
293 nbsRows = queryNBS(suite_id, session)
294 title = 'NBS packages in suite %s' % suite_name
295 if nbsRows.rowcount > 0: 295 ↛ 296line 295 didn't jump to line 296, because the condition on line 295 was never true
296 print_info('%s\n%s\n' % (title, '-' * len(title)))
297 for row in nbsRows: 297 ↛ 298line 297 didn't jump to line 298, because the loop on line 297 never started
298 (pkg_list, arch_list, source, version) = row
299 pkg_string = ' '.join(pkg_list)
300 arch_string = ','.join(arch_list)
301 print_info("* source package %s version %s no longer builds" %
302 (source, version))
303 print_info(" binary package(s): %s" % pkg_string)
304 print_info(" on %s" % arch_string)
305 print_info(" - suggested command:")
306 message = '"[auto-cruft] NBS (no longer built by %s)"' % source
307 print_cmd("dak rm -o -m %s -s %s -a %s -p -R -b %s" %
308 (message, suite_name, arch_string, pkg_string))
309 if rdeps:
310 if utils.check_reverse_depends(pkg_list, suite_name, arch_list, session, True):
311 print_info()
312 else:
313 print_info(" - No dependency problem found\n")
314 else:
315 print_info()
316 session.close()
319def reportNBSMetadata(suite_name, suite_id, session, rdeps=False):
320 rows = queryNBS_metadata(suite_id, session)
321 title = 'NBS packages (from metadata) in suite %s' % suite_name
322 if rows.rowcount > 0: 322 ↛ 324line 322 didn't jump to line 324, because the condition on line 322 was never false
323 print_info('%s\n%s\n' % (title, '-' * len(title)))
324 for row in rows:
325 (packages, architecture, source, version) = row
326 print_info("* source package %s version %s no longer builds" %
327 (source, version))
328 print_info(" binary package(s): %s" % packages)
329 print_info(" on %s" % architecture)
330 print_info(" - suggested command:")
331 message = '"[auto-cruft] NBS (no longer built by %s - based on source metadata)"' % source
332 print_cmd("dak rm -o -m %s -s %s -a %s -p -R -b %s" %
333 (message, suite_name, architecture, packages))
334 if rdeps: 334 ↛ 335line 334 didn't jump to line 335, because the condition on line 334 was never true
335 archs = [architecture]
336 if architecture == "all":
337 # when archs is None, rdeps are checked on all archs in the suite
338 archs = None
339 if utils.check_reverse_depends(packages.split(), suite_name, archs, session, True):
340 print_info()
341 else:
342 print_info(" - No dependency problem found\n")
343 else:
344 print_info()
347def reportAllNBS(suite_name, suite_id, session, rdeps=False):
348 reportWithoutSource(suite_name, suite_id, session, rdeps)
349 reportNewerAll(suite_name, session)
350 reportNBS(suite_name, suite_id, rdeps)
352################################################################################
355def do_dubious_nbs(dubious_nbs):
356 print_info("Dubious NBS")
357 print_info("-----------")
358 print_info()
360 version_sort_key = functools.cmp_to_key(apt_pkg.version_compare)
361 for source in sorted(dubious_nbs):
362 print_info(" * %s_%s builds: %s" % (source,
363 source_versions.get(source, "??"),
364 source_binaries.get(source, "(source does not exist)")))
365 print_info(" won't admit to building:")
366 versions = sorted(dubious_nbs[source], key=version_sort_key)
367 for version in versions:
368 packages = sorted(dubious_nbs[source][version])
369 print_info(" o %s: %s" % (version, ", ".join(packages)))
371 print_info()
373################################################################################
376def obsolete_source(suite_name, session):
377 """returns obsolete source packages for suite_name without binaries
378 in the same suite sorted by install_date; install_date should help
379 detecting source only (or binary throw away) uploads; duplicates in
380 the suite are skipped
382 subquery 'source_suite_unique' returns source package names from
383 suite without duplicates; the rationale behind is that neither
384 cruft-report nor rm cannot handle duplicates (yet)"""
386 query = """
387WITH source_suite_unique AS
388 (SELECT source, suite
389 FROM source_suite GROUP BY source, suite HAVING count(*) = 1)
390SELECT ss.src, ss.source, ss.version,
391 to_char(ss.install_date, 'YYYY-MM-DD') AS install_date
392 FROM source_suite ss
393 JOIN source_suite_unique ssu
394 ON ss.source = ssu.source AND ss.suite = ssu.suite
395 JOIN suite s ON s.id = ss.suite
396 LEFT JOIN bin_associations_binaries bab
397 ON ss.src = bab.source AND ss.suite = bab.suite
398 WHERE s.suite_name = :suite_name AND bab.id IS NULL
399 AND now() - ss.install_date > '1 day'::interval
400 ORDER BY install_date"""
401 args = {'suite_name': suite_name}
402 return session.execute(query, args)
405def source_bin(source, session):
406 """returns binaries built by source for all or no suite grouped and
407 ordered by package name"""
409 query = """
410SELECT b.package
411 FROM binaries b
412 JOIN src_associations_src sas ON b.source = sas.src
413 WHERE sas.source = :source
414 GROUP BY b.package
415 ORDER BY b.package"""
416 args = {'source': source}
417 return session.execute(query, args)
420def newest_source_bab(suite_name, package, session):
421 """returns newest source that builds binary package in suite grouped
422 and sorted by source and package name"""
424 query = """
425SELECT sas.source, MAX(sas.version) AS srcver
426 FROM src_associations_src sas
427 JOIN bin_associations_binaries bab ON sas.src = bab.source
428 JOIN suite s on s.id = bab.suite
429 WHERE s.suite_name = :suite_name AND bab.package = :package
430 GROUP BY sas.source, bab.package
431 ORDER BY sas.source, bab.package"""
432 args = {'suite_name': suite_name, 'package': package}
433 return session.execute(query, args)
436def report_obsolete_source(suite_name, session):
437 rows = obsolete_source(suite_name, session)
438 if rows.rowcount == 0: 438 ↛ 440line 438 didn't jump to line 440, because the condition on line 438 was never false
439 return
440 print_info("""Obsolete source packages in suite %s
441----------------------------------%s\n""" %
442 (suite_name, '-' * len(suite_name)))
443 for os_row in rows.fetchall():
444 (src, old_source, version, install_date) = os_row
445 print_info(" * obsolete source %s version %s installed at %s" %
446 (old_source, version, install_date))
447 for sb_row in source_bin(old_source, session):
448 (package, ) = sb_row
449 print_info(" - has built binary %s" % package)
450 for nsb_row in newest_source_bab(suite_name, package, session):
451 (new_source, srcver) = nsb_row
452 print_info(" currently built by source %s version %s" %
453 (new_source, srcver))
454 print_info(" - suggested command:")
455 rm_opts = "-S -p -m \"[auto-cruft] obsolete source package\""
456 print_cmd("dak rm -s %s %s %s\n" % (suite_name, rm_opts, old_source))
459def get_suite_binaries(suite, session):
460 # Initalize a large hash table of all binary packages
461 binaries = {}
463 print_info("Getting a list of binary packages in %s..." % suite.suite_name)
464 q = session.execute("""SELECT distinct b.package
465 FROM binaries b, bin_associations ba
466 WHERE ba.suite = :suiteid AND ba.bin = b.id""",
467 {'suiteid': suite.suite_id})
468 for i in q.fetchall():
469 binaries[i[0]] = ""
471 return binaries
473################################################################################
476def report_outdated_nonfree(suite, session, rdeps=False):
478 packages = {}
479 query = """WITH outdated_sources AS (
480 SELECT s.source, s.version, s.id
481 FROM source s
482 JOIN src_associations sa ON sa.source = s.id
483 WHERE sa.suite IN (
484 SELECT id
485 FROM suite
486 WHERE suite_name = :suite )
487 AND sa.created < (now() - interval :delay)
488 EXCEPT SELECT s.source, max(s.version) AS version, max(s.id)
489 FROM source s
490 JOIN src_associations sa ON sa.source = s.id
491 WHERE sa.suite IN (
492 SELECT id
493 FROM suite
494 WHERE suite_name = :suite )
495 AND sa.created < (now() - interval :delay)
496 GROUP BY s.source ),
497 binaries AS (
498 SELECT b.package, s.source, (
499 SELECT a.arch_string
500 FROM architecture a
501 WHERE a.id = b.architecture ) AS arch
502 FROM binaries b
503 JOIN outdated_sources s ON s.id = b.source
504 JOIN bin_associations ba ON ba.bin = b.id
505 JOIN override o ON o.package = b.package AND o.suite = ba.suite
506 WHERE ba.suite IN (
507 SELECT id
508 FROM suite
509 WHERE suite_name = :suite )
510 AND o.component IN (
511 SELECT id
512 FROM component
513 WHERE name = 'non-free' ) )
514 SELECT DISTINCT package, source, arch
515 FROM binaries
516 ORDER BY source, package, arch"""
518 res = session.execute(query, {'suite': suite, 'delay': "'15 days'"})
519 for package in res: 519 ↛ 520line 519 didn't jump to line 520, because the loop on line 519 never started
520 binary = package[0]
521 source = package[1]
522 arch = package[2]
523 if arch == 'all':
524 continue
525 if source not in packages:
526 packages[source] = {}
527 if binary not in packages[source]:
528 packages[source][binary] = set()
529 packages[source][binary].add(arch)
530 if packages: 530 ↛ 531line 530 didn't jump to line 531, because the condition on line 530 was never true
531 title = 'Outdated non-free binaries in suite %s' % suite
532 message = '"[auto-cruft] outdated non-free binaries"'
533 print_info('%s\n%s\n' % (title, '-' * len(title)))
534 for source in sorted(packages):
535 archs = set()
536 binaries = set()
537 print_info('* package %s has outdated non-free binaries' % source)
538 print_info(' - suggested command:')
539 for binary in sorted(packages[source]):
540 binaries.add(binary)
541 archs = archs.union(packages[source][binary])
542 print_cmd('dak rm -o -m %s -s %s -a %s -p -R -b %s' %
543 (message, suite, ','.join(archs), ' '.join(binaries)))
544 if rdeps:
545 if utils.check_reverse_depends(list(binaries), suite, archs, session, True):
546 print_info()
547 else:
548 print_info(" - No dependency problem found\n")
549 else:
550 print_info()
552################################################################################
555def main():
556 global suite, suite_id, source_binaries, source_versions
558 cnf = Config()
560 Arguments = [('h', "help", "Cruft-Report::Options::Help"),
561 ('m', "mode", "Cruft-Report::Options::Mode", "HasArg"),
562 ('R', "rdep-check", "Cruft-Report::Options::Rdep-Check"),
563 ('s', "suite", "Cruft-Report::Options::Suite", "HasArg"),
564 ('w', "wanna-build-dump", "Cruft-Report::Options::Wanna-Build-Dump", "HasArg"),
565 ('c', "commands-only", "Cruft-Report::Options::Commands-Only")]
566 for i in ["help", "Rdep-Check"]:
567 key = "Cruft-Report::Options::%s" % i
568 if key not in cnf: 568 ↛ 566line 568 didn't jump to line 566, because the condition on line 568 was never false
569 cnf[key] = ""
571 if "Cruft-Report::Options::Commands-Only" not in cnf: 571 ↛ 574line 571 didn't jump to line 574, because the condition on line 571 was never false
572 cnf["Cruft-Report::Options::Commands-Only"] = ""
574 cnf["Cruft-Report::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable")
576 if "Cruft-Report::Options::Mode" not in cnf: 576 ↛ 579line 576 didn't jump to line 579, because the condition on line 576 was never false
577 cnf["Cruft-Report::Options::Mode"] = "daily"
579 if "Cruft-Report::Options::Wanna-Build-Dump" not in cnf: 579 ↛ 582line 579 didn't jump to line 582, because the condition on line 579 was never false
580 cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp-master.debian.org/scripts/nfu"
582 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
584 Options = cnf.subtree("Cruft-Report::Options")
585 if Options["Help"]:
586 usage()
588 if Options["Rdep-Check"]: 588 ↛ 589line 588 didn't jump to line 589, because the condition on line 588 was never true
589 rdeps = True
590 else:
591 rdeps = False
593 # Set up checks based on mode
594 if Options["Mode"] == "daily": 594 ↛ 596line 594 didn't jump to line 596, because the condition on line 594 was never false
595 checks = ["nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu", "nbs metadata"]
596 elif Options["Mode"] == "full":
597 checks = ["nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu", "nbs metadata", "dubious nbs", "bnb", "bms", "anais"]
598 elif Options["Mode"] == "bdo":
599 checks = ["nbs", "obsolete source"]
600 else:
601 utils.warn("%s is not a recognised mode - only 'full', 'daily' or 'bdo' are understood." % (Options["Mode"]))
602 usage(1)
604 session = DBConn().session()
606 bin_pkgs = {}
607 src_pkgs = {}
608 bin2source = {}
609 bins_in_suite = {}
610 nbs = defaultdict(lambda: defaultdict(set)) 610 ↛ exitline 610 didn't run the lambda on line 610
611 source_versions = {}
613 anais_output = ""
615 nfu_packages = defaultdict(list)
617 suite = get_suite(Options["Suite"].lower(), session)
618 if not suite: 618 ↛ 619line 618 didn't jump to line 619, because the condition on line 618 was never true
619 utils.fubar("Cannot find suite %s" % Options["Suite"].lower())
621 suite_id = suite.suite_id
622 suite_name = suite.suite_name.lower()
624 if "obsolete source" in checks: 624 ↛ 627line 624 didn't jump to line 627, because the condition on line 624 was never false
625 report_obsolete_source(suite_name, session)
627 if "nbs" in checks: 627 ↛ 630line 627 didn't jump to line 630, because the condition on line 627 was never false
628 reportAllNBS(suite_name, suite_id, session, rdeps)
630 if "nbs metadata" in checks: 630 ↛ 633line 630 didn't jump to line 633, because the condition on line 630 was never false
631 reportNBSMetadata(suite_name, suite_id, session, rdeps)
633 if "outdated non-free" in checks: 633 ↛ 636line 633 didn't jump to line 636, because the condition on line 633 was never false
634 report_outdated_nonfree(suite_name, session, rdeps)
636 bin_not_built = defaultdict(set)
638 if "bnb" in checks: 638 ↛ 639line 638 didn't jump to line 639, because the condition on line 638 was never true
639 bins_in_suite = get_suite_binaries(suite, session)
641 # Checks based on the Sources files
642 components = [c.component_name for c in suite.components]
643 for component in [c.component_name for c in suite.components]:
644 filename = "%s/dists/%s/%s/source/Sources" % (suite.archive.path, suite_name, component)
645 filename = utils.find_possibly_compressed_file(filename)
646 with apt_pkg.TagFile(filename) as Sources:
647 while Sources.step():
648 source = Sources.section.find('Package')
649 source_version = Sources.section.find('Version')
650 architecture = Sources.section.find('Architecture')
651 binaries = Sources.section.find('Binary')
652 binaries_list = [i.strip() for i in binaries.split(',')]
654 if "bnb" in checks: 654 ↛ 656line 654 didn't jump to line 656, because the condition on line 654 was never true
655 # Check for binaries not built on any architecture.
656 for binary in binaries_list:
657 if binary not in bins_in_suite:
658 bin_not_built[source].add(binary)
660 if "anais" in checks: 660 ↛ 661line 660 didn't jump to line 661, because the condition on line 660 was never true
661 anais_output += do_anais(architecture, binaries_list, source, session)
663 # build indices for checking "no source" later
664 source_index = component + '/' + source
665 src_pkgs[source] = source_index
666 for binary in binaries_list:
667 bin_pkgs[binary] = source
668 source_binaries[source] = binaries
669 source_versions[source] = source_version
671 # Checks based on the Packages files
672 check_components = components[:]
673 if suite_name != "experimental": 673 ↛ 676line 673 didn't jump to line 676, because the condition on line 673 was never false
674 check_components.append('main/debian-installer')
676 for component in check_components:
677 architectures = [a.arch_string for a in get_suite_architectures(suite_name,
678 skipsrc=True, skipall=True,
679 session=session)]
680 for architecture in architectures:
681 if component == 'main/debian-installer' and re.match("kfreebsd", architecture): 681 ↛ 682line 681 didn't jump to line 682, because the condition on line 681 was never true
682 continue
684 if "nfu" in checks: 684 ↛ 687line 684 didn't jump to line 687, because the condition on line 684 was never false
685 nfu_entries = parse_nfu(architecture)
687 filename = "%s/dists/%s/%s/binary-%s/Packages" % (suite.archive.path, suite_name, component, architecture)
688 filename = utils.find_possibly_compressed_file(filename)
689 with apt_pkg.TagFile(filename) as Packages:
690 while Packages.step():
691 package = Packages.section.find('Package')
692 source = Packages.section.find('Source', "")
693 version = Packages.section.find('Version')
694 if source == "":
695 source = package
696 if package in bin2source and \ 696 ↛ 698line 696 didn't jump to line 698, because the condition on line 696 was never true
697 apt_pkg.version_compare(version, bin2source[package]["version"]) > 0:
698 bin2source[package]["version"] = version
699 bin2source[package]["source"] = source
700 else:
701 bin2source[package] = {}
702 bin2source[package]["version"] = version
703 bin2source[package]["source"] = source
704 if source.find("(") != -1: 704 ↛ 705line 704 didn't jump to line 705, because the condition on line 704 was never true
705 m = re_extract_src_version.match(source)
706 source = m.group(1)
707 version = m.group(2)
708 if package not in bin_pkgs: 708 ↛ 709line 708 didn't jump to line 709, because the condition on line 708 was never true
709 nbs[source][package].add(version)
710 else:
711 if "nfu" in checks: 711 ↛ 690line 711 didn't jump to line 690, because the condition on line 711 was never false
712 if package in nfu_entries and \ 712 ↛ 714line 712 didn't jump to line 714, because the condition on line 712 was never true
713 version != source_versions[source]: # only suggest to remove out-of-date packages
714 nfu_packages[architecture].append((package, version, source_versions[source]))
716 # Distinguish dubious (version numbers match) and 'real' NBS (they don't)
717 dubious_nbs = defaultdict(lambda: defaultdict(set)) 717 ↛ exitline 717 didn't run the lambda on line 717
718 version_sort_key = functools.cmp_to_key(apt_pkg.version_compare)
719 for source in nbs: 719 ↛ 720line 719 didn't jump to line 720, because the loop on line 719 never started
720 for package in nbs[source]:
721 latest_version = max(nbs[source][package], key=version_sort_key)
722 source_version = source_versions.get(source, "0")
723 if apt_pkg.version_compare(latest_version, source_version) == 0:
724 add_nbs(dubious_nbs, source, latest_version, package, suite_id, session)
726 if "nviu" in checks: 726 ↛ 729line 726 didn't jump to line 729, because the condition on line 726 was never false
727 do_newer_version('unstable', 'experimental', 'NVIU', session)
729 if "nvit" in checks: 729 ↛ 734line 729 didn't jump to line 734, because the condition on line 729 was never false
730 do_newer_version('testing', 'testing-proposed-updates', 'NVIT', session)
732 ###
734 if Options["Mode"] == "full": 734 ↛ 735line 734 didn't jump to line 735, because the condition on line 734 was never true
735 print_info("=" * 75)
736 print_info()
738 if "nfu" in checks: 738 ↛ 741line 738 didn't jump to line 741, because the condition on line 738 was never false
739 do_nfu(nfu_packages)
741 if "bnb" in checks: 741 ↛ 742line 741 didn't jump to line 742, because the condition on line 741 was never true
742 print_info("Unbuilt binary packages")
743 print_info("-----------------------")
744 print_info()
745 for source in sorted(bin_not_built):
746 binaries = sorted(bin_not_built[source])
747 print_info(" o %s: %s" % (source, ", ".join(binaries)))
748 print_info()
750 if "bms" in checks: 750 ↛ 751line 750 didn't jump to line 751, because the condition on line 750 was never true
751 report_multiple_source(suite)
753 if "anais" in checks: 753 ↛ 754line 753 didn't jump to line 754, because the condition on line 753 was never true
754 print_info("Architecture Not Allowed In Source")
755 print_info("----------------------------------")
756 print_info(anais_output)
757 print_info()
759 if "dubious nbs" in checks: 759 ↛ 760line 759 didn't jump to line 760, because the condition on line 759 was never true
760 do_dubious_nbs(dubious_nbs)
763################################################################################
765if __name__ == '__main__':
766 main()