1
2
3 """
4 Check for obsolete binary packages
5
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2000-2006 James Troup <james@nocrew.org>
8 @copyright: 2009 Torsten Werner <twerner@debian.org>
9 @license: GNU General Public License version 2 or later
10 """
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38 import functools
39 import os
40 import re
41 import sys
42 from collections import defaultdict
43
44 import apt_pkg
45
46 from daklib import utils
47 from daklib.config import Config
48 from daklib.cruft import (
49 newer_version,
50 query_without_source,
51 queryNBS,
52 queryNBS_metadata,
53 report_multiple_source,
54 )
55 from daklib.dbconn import DBConn, get_suite, get_suite_architectures
56 from daklib.regexes import re_extract_src_version
57
58
59
60 no_longer_in_suite = {}
61
62 source_binaries = {}
63 source_versions = {}
64
65
66
67
69 print(
70 """Usage: dak cruft-report
71 Check for obsolete or duplicated packages.
72
73 -h, --help show this help and exit.
74 -m, --mode=MODE chose the MODE to run in (full, daily, bdo).
75 -s, --suite=SUITE check suite SUITE.
76 -R, --rdep-check check reverse dependencies
77 -w, --wanna-build-dump where to find the copies of https://buildd.debian.org/stats/*.txt"""
78 )
79 sys.exit(exit_code)
80
81
82
83
84
86 cnf = Config()
87
88 if cnf.subtree("Cruft-Report::Options")["Commands-Only"]:
89 return
90
91 print(s)
92
93
95 cnf = Config()
96
97
98 if not cnf.subtree("Cruft-Report::Options")["Commands-Only"]:
99 ind = " " * indent
100 s = ind + s
101
102 print(s)
103
104
105
106
107
108 -def add_nbs(nbs_d, source, version, package, suite_id, session):
109
110 if package in no_longer_in_suite:
111 return
112 else:
113 q = session.execute(
114 """SELECT b.id FROM binaries b, bin_associations ba
115 WHERE ba.bin = b.id AND ba.suite = :suite_id
116 AND b.package = :package LIMIT 1""",
117 {"suite_id": suite_id, "package": package},
118 )
119 if not q.fetchall():
120 no_longer_in_suite[package] = ""
121 return
122
123 nbs_d[source][version].add(package)
124
125
126
127
128
129
130
131 -def do_anais(architecture, binaries_list, source, session):
132 if architecture == "any" or architecture == "all":
133 return ""
134
135 version_sort_key = functools.cmp_to_key(apt_pkg.version_compare)
136 anais_output = ""
137 architectures = {}
138 for arch in architecture.split():
139 architectures[arch.strip()] = ""
140 for binary in binaries_list:
141 q = session.execute(
142 """SELECT a.arch_string, b.version
143 FROM binaries b, bin_associations ba, architecture a
144 WHERE ba.suite = :suiteid AND ba.bin = b.id
145 AND b.architecture = a.id AND b.package = :package""",
146 {"suiteid": suite_id, "package": binary},
147 )
148 ql = q.fetchall()
149 versions = []
150 for arch, version in ql:
151 if arch in architectures:
152 versions.append(version)
153 versions.sort(key=version_sort_key)
154 if versions:
155 latest_version = versions.pop()
156 else:
157 latest_version = None
158
159 versions_d = defaultdict(list)
160 for arch, version in ql:
161 if arch not in architectures:
162 versions_d[version].append(arch)
163
164 if versions_d:
165 anais_output += "\n (*) %s_%s [%s]: %s\n" % (
166 binary,
167 latest_version,
168 source,
169 architecture,
170 )
171 for version in sorted(versions_d, key=version_sort_key):
172 arches = sorted(versions_d[version])
173 anais_output += " o %s: %s\n" % (version, ", ".join(arches))
174 return anais_output
175
176
177
178
179
180
181
183 output = ""
184
185 a2p = {}
186
187 for architecture in nfu_packages:
188 a2p[architecture] = []
189 for package, bver, sver in nfu_packages[architecture]:
190 output += " * [%s] does not want %s (binary %s, source %s)\n" % (
191 architecture,
192 package,
193 bver,
194 sver,
195 )
196 a2p[architecture].append(package)
197
198 if output:
199 print_info("Obsolete by Not-For-Us")
200 print_info("----------------------")
201 print_info()
202 print_info(output)
203
204 print_info("Suggested commands:")
205 for architecture in a2p:
206 if a2p[architecture]:
207 print_cmd(
208 (
209 'dak rm -o -m "[auto-cruft] NFU" -s %s -a %s -b %s'
210 % (suite.suite_name, architecture, " ".join(a2p[architecture]))
211 ),
212 indent=1,
213 )
214 print_info()
215
216
218 cnf = Config()
219
220 r = re.compile(r"^\w+/([^_]+)_.*: Not-For-Us")
221
222 ret = set()
223
224 filename = "%s/%s-all.txt" % (
225 cnf["Cruft-Report::Options::Wanna-Build-Dump"],
226 architecture,
227 )
228
229
230
231 if os.path.exists(filename):
232 with open(filename) as f:
233 for line in f:
234 if line[0] == " ":
235 continue
236
237 m = r.match(line)
238 if m:
239 ret.add(m.group(1))
240 else:
241 utils.warn("No wanna-build dump file for architecture %s" % architecture)
242 return ret
243
244
245
246
247
249 list = newer_version(lowersuite_name, highersuite_name, session)
250 if len(list) > 0:
251 nv_to_remove = []
252 title = "Newer version in %s" % lowersuite_name
253 print_info(title)
254 print_info("-" * len(title))
255 print_info()
256 for i in list:
257 (source, higher_version, lower_version) = i
258 print_info(" o %s (%s, %s)" % (source, higher_version, lower_version))
259 nv_to_remove.append(source)
260 print_info()
261 print_info("Suggested command:")
262 print_cmd(
263 'dak rm -m "[auto-cruft] %s" -s %s %s'
264 % (code, highersuite_name, " ".join(nv_to_remove)),
265 indent=1,
266 )
267 print_info()
268
269
270
271
272
274 rows = query_without_source(suite_id, session)
275 title = "packages without source in suite %s" % suite_name
276 if rows.rowcount > 0:
277 print_info("%s\n%s\n" % (title, "-" * len(title)))
278 message = '"[auto-cruft] no longer built from source"'
279 for row in rows:
280 (package, version) = row
281 print_info(
282 "* package %s in version %s is no longer built from source"
283 % (package, version)
284 )
285 print_info(" - suggested command:")
286 print_cmd(
287 "dak rm -m %s -s %s -a all -p -R -b %s" % (message, suite_name, package)
288 )
289 if rdeps:
290 if utils.check_reverse_depends([package], suite_name, [], session, True):
291 print_info()
292 else:
293 print_info(" - No dependency problem found\n")
294 else:
295 print_info()
296
297
299 """searches for arch != all packages that have an arch == all
300 package with a higher version in the same suite"""
301
302 query = """
303 select bab1.package, bab1.version as oldver,
304 array_to_string(array_agg(a.arch_string), ',') as oldarch,
305 bab2.version as newver
306 from bin_associations_binaries bab1
307 join bin_associations_binaries bab2
308 on bab1.package = bab2.package and bab1.version < bab2.version and
309 bab1.suite = bab2.suite and bab1.architecture > 2 and
310 bab2.architecture = 2
311 join architecture a on bab1.architecture = a.id
312 join suite s on bab1.suite = s.id
313 where s.suite_name = :suite_name
314 group by bab1.package, oldver, bab1.suite, newver"""
315 return session.execute(query, {"suite_name": suite_name})
316
317
319 rows = queryNewerAll(suite_name, session)
320 title = "obsolete arch any packages in suite %s" % suite_name
321 if rows.rowcount > 0:
322 print_info("%s\n%s\n" % (title, "-" * len(title)))
323 message = '"[auto-cruft] obsolete arch any package"'
324 for row in rows:
325 (package, oldver, oldarch, newver) = row
326 print_info(
327 "* package %s is arch any in version %s but arch all in version %s"
328 % (package, oldver, newver)
329 )
330 print_info(" - suggested command:")
331 print_cmd(
332 "dak rm -o -m %s -s %s -a %s -p -b %s\n"
333 % (message, suite_name, oldarch, package)
334 )
335
336
337 -def reportNBS(suite_name, suite_id, rdeps=False):
338 session = DBConn().session()
339 nbsRows = queryNBS(suite_id, session)
340 title = "NBS packages in suite %s" % suite_name
341 if nbsRows.rowcount > 0:
342 print_info("%s\n%s\n" % (title, "-" * len(title)))
343 for row in nbsRows:
344 (pkg_list, arch_list, source, version) = row
345 pkg_string = " ".join(pkg_list)
346 arch_string = ",".join(arch_list)
347 print_info(
348 "* source package %s version %s no longer builds" % (source, version)
349 )
350 print_info(" binary package(s): %s" % pkg_string)
351 print_info(" on %s" % arch_string)
352 print_info(" - suggested command:")
353 message = '"[auto-cruft] NBS (no longer built by %s)"' % source
354 print_cmd(
355 "dak rm -o -m %s -s %s -a %s -p -R -b %s"
356 % (message, suite_name, arch_string, pkg_string)
357 )
358 if rdeps:
359 if utils.check_reverse_depends(
360 pkg_list, suite_name, arch_list, session, True
361 ):
362 print_info()
363 else:
364 print_info(" - No dependency problem found\n")
365 else:
366 print_info()
367 session.close()
368
369
404
405
406 -def reportAllNBS(suite_name, suite_id, session, rdeps=False):
410
411
412
413
414
416 print_info("Dubious NBS")
417 print_info("-----------")
418 print_info()
419
420 version_sort_key = functools.cmp_to_key(apt_pkg.version_compare)
421 for source in sorted(dubious_nbs):
422 print_info(
423 " * %s_%s builds: %s"
424 % (
425 source,
426 source_versions.get(source, "??"),
427 source_binaries.get(source, "(source does not exist)"),
428 )
429 )
430 print_info(" won't admit to building:")
431 versions = sorted(dubious_nbs[source], key=version_sort_key)
432 for version in versions:
433 packages = sorted(dubious_nbs[source][version])
434 print_info(" o %s: %s" % (version, ", ".join(packages)))
435
436 print_info()
437
438
439
440
441
443 """returns obsolete source packages for suite_name without binaries
444 in the same suite sorted by install_date; install_date should help
445 detecting source only (or binary throw away) uploads; duplicates in
446 the suite are skipped
447
448 subquery 'source_suite_unique' returns source package names from
449 suite without duplicates; the rationale behind is that neither
450 cruft-report nor rm cannot handle duplicates (yet)"""
451
452 query = """
453 WITH source_suite_unique AS
454 (SELECT source, suite
455 FROM source_suite GROUP BY source, suite HAVING count(*) = 1)
456 SELECT ss.src, ss.source, ss.version,
457 to_char(ss.install_date, 'YYYY-MM-DD') AS install_date
458 FROM source_suite ss
459 JOIN source_suite_unique ssu
460 ON ss.source = ssu.source AND ss.suite = ssu.suite
461 JOIN suite s ON s.id = ss.suite
462 LEFT JOIN bin_associations_binaries bab
463 ON ss.src = bab.source AND ss.suite = bab.suite
464 WHERE s.suite_name = :suite_name AND bab.id IS NULL
465 AND now() - ss.install_date > '1 day'::interval
466 ORDER BY install_date"""
467 args = {"suite_name": suite_name}
468 return session.execute(query, args)
469
470
472 """returns binaries built by source for all or no suite grouped and
473 ordered by package name"""
474
475 query = """
476 SELECT b.package
477 FROM binaries b
478 JOIN src_associations_src sas ON b.source = sas.src
479 WHERE sas.source = :source
480 GROUP BY b.package
481 ORDER BY b.package"""
482 args = {"source": source}
483 return session.execute(query, args)
484
485
487 """returns newest source that builds binary package in suite grouped
488 and sorted by source and package name"""
489
490 query = """
491 SELECT sas.source, MAX(sas.version) AS srcver
492 FROM src_associations_src sas
493 JOIN bin_associations_binaries bab ON sas.src = bab.source
494 JOIN suite s on s.id = bab.suite
495 WHERE s.suite_name = :suite_name AND bab.package = :package
496 GROUP BY sas.source, bab.package
497 ORDER BY sas.source, bab.package"""
498 args = {"suite_name": suite_name, "package": package}
499 return session.execute(query, args)
500
501
503 rows = obsolete_source(suite_name, session)
504 if rows.rowcount == 0:
505 return
506 print_info(
507 """Obsolete source packages in suite %s
508 ----------------------------------%s\n"""
509 % (suite_name, "-" * len(suite_name))
510 )
511 for os_row in rows.fetchall():
512 (src, old_source, version, install_date) = os_row
513 print_info(
514 " * obsolete source %s version %s installed at %s"
515 % (old_source, version, install_date)
516 )
517 for sb_row in source_bin(old_source, session):
518 (package,) = sb_row
519 print_info(" - has built binary %s" % package)
520 for nsb_row in newest_source_bab(suite_name, package, session):
521 (new_source, srcver) = nsb_row
522 print_info(
523 " currently built by source %s version %s"
524 % (new_source, srcver)
525 )
526 print_info(" - suggested command:")
527 rm_opts = '-S -p -m "[auto-cruft] obsolete source package"'
528 print_cmd("dak rm -s %s %s %s\n" % (suite_name, rm_opts, old_source))
529
530
532
533 binaries = {}
534
535 print_info("Getting a list of binary packages in %s..." % suite.suite_name)
536 q = session.execute(
537 """SELECT distinct b.package
538 FROM binaries b, bin_associations ba
539 WHERE ba.suite = :suiteid AND ba.bin = b.id""",
540 {"suiteid": suite.suite_id},
541 )
542 for i in q.fetchall():
543 binaries[i[0]] = ""
544
545 return binaries
546
547
548
549
550
552
553 packages = {}
554 query = """WITH outdated_sources AS (
555 SELECT s.source, s.version, s.id
556 FROM source s
557 JOIN src_associations sa ON sa.source = s.id
558 WHERE sa.suite IN (
559 SELECT id
560 FROM suite
561 WHERE suite_name = :suite )
562 AND sa.created < (now() - interval :delay)
563 EXCEPT SELECT s.source, max(s.version) AS version, max(s.id)
564 FROM source s
565 JOIN src_associations sa ON sa.source = s.id
566 WHERE sa.suite IN (
567 SELECT id
568 FROM suite
569 WHERE suite_name = :suite )
570 AND sa.created < (now() - interval :delay)
571 GROUP BY s.source ),
572 binaries AS (
573 SELECT b.package, s.source, (
574 SELECT a.arch_string
575 FROM architecture a
576 WHERE a.id = b.architecture ) AS arch
577 FROM binaries b
578 JOIN outdated_sources s ON s.id = b.source
579 JOIN bin_associations ba ON ba.bin = b.id
580 JOIN override o ON o.package = b.package AND o.suite = ba.suite
581 WHERE ba.suite IN (
582 SELECT id
583 FROM suite
584 WHERE suite_name = :suite )
585 AND o.component IN (
586 SELECT id
587 FROM component
588 WHERE name = 'non-free' ) )
589 SELECT DISTINCT package, source, arch
590 FROM binaries
591 ORDER BY source, package, arch"""
592
593 res = session.execute(query, {"suite": suite, "delay": "'15 days'"})
594 for package in res:
595 binary = package[0]
596 source = package[1]
597 arch = package[2]
598 if arch == "all":
599 continue
600 if source not in packages:
601 packages[source] = {}
602 if binary not in packages[source]:
603 packages[source][binary] = set()
604 packages[source][binary].add(arch)
605 if packages:
606 title = "Outdated non-free binaries in suite %s" % suite
607 message = '"[auto-cruft] outdated non-free binaries"'
608 print_info("%s\n%s\n" % (title, "-" * len(title)))
609 for source in sorted(packages):
610 archs = set()
611 binaries = set()
612 print_info("* package %s has outdated non-free binaries" % source)
613 print_info(" - suggested command:")
614 for binary in sorted(packages[source]):
615 binaries.add(binary)
616 archs = archs.union(packages[source][binary])
617 print_cmd(
618 "dak rm -o -m %s -s %s -a %s -p -R -b %s"
619 % (message, suite, ",".join(archs), " ".join(binaries))
620 )
621 if rdeps:
622 if utils.check_reverse_depends(
623 list(binaries), suite, archs, session, True
624 ):
625 print_info()
626 else:
627 print_info(" - No dependency problem found\n")
628 else:
629 print_info()
630
631
632
633
634
636 global suite, suite_id, source_binaries, source_versions
637
638 cnf = Config()
639
640 Arguments = [
641 ("h", "help", "Cruft-Report::Options::Help"),
642 ("m", "mode", "Cruft-Report::Options::Mode", "HasArg"),
643 ("R", "rdep-check", "Cruft-Report::Options::Rdep-Check"),
644 ("s", "suite", "Cruft-Report::Options::Suite", "HasArg"),
645 ("w", "wanna-build-dump", "Cruft-Report::Options::Wanna-Build-Dump", "HasArg"),
646 ("c", "commands-only", "Cruft-Report::Options::Commands-Only"),
647 ]
648 for i in ["help", "Rdep-Check"]:
649 key = "Cruft-Report::Options::%s" % i
650 if key not in cnf:
651 cnf[key] = ""
652
653 if "Cruft-Report::Options::Commands-Only" not in cnf:
654 cnf["Cruft-Report::Options::Commands-Only"] = ""
655
656 cnf["Cruft-Report::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable")
657
658 if "Cruft-Report::Options::Mode" not in cnf:
659 cnf["Cruft-Report::Options::Mode"] = "daily"
660
661 if "Cruft-Report::Options::Wanna-Build-Dump" not in cnf:
662 cnf["Cruft-Report::Options::Wanna-Build-Dump"] = (
663 "/srv/ftp-master.debian.org/scripts/nfu"
664 )
665
666 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
667
668 Options = cnf.subtree("Cruft-Report::Options")
669 if Options["Help"]:
670 usage()
671
672 if Options["Rdep-Check"]:
673 rdeps = True
674 else:
675 rdeps = False
676
677
678 if Options["Mode"] == "daily":
679 checks = [
680 "nbs",
681 "nviu",
682 "nvit",
683 "obsolete source",
684 "outdated non-free",
685 "nfu",
686 "nbs metadata",
687 ]
688 elif Options["Mode"] == "full":
689 checks = [
690 "nbs",
691 "nviu",
692 "nvit",
693 "obsolete source",
694 "outdated non-free",
695 "nfu",
696 "nbs metadata",
697 "dubious nbs",
698 "bnb",
699 "bms",
700 "anais",
701 ]
702 elif Options["Mode"] == "bdo":
703 checks = ["nbs", "obsolete source"]
704 else:
705 utils.warn(
706 "%s is not a recognised mode - only 'full', 'daily' or 'bdo' are understood."
707 % (Options["Mode"])
708 )
709 usage(1)
710
711 session = DBConn().session()
712
713 bin_pkgs = {}
714 src_pkgs = {}
715 bin2source = {}
716 bins_in_suite = {}
717 nbs = defaultdict(lambda: defaultdict(set))
718 source_versions = {}
719
720 anais_output = ""
721
722 nfu_packages = defaultdict(list)
723
724 suite = get_suite(Options["Suite"].lower(), session)
725 if not suite:
726 utils.fubar("Cannot find suite %s" % Options["Suite"].lower())
727
728 suite_id = suite.suite_id
729 suite_name = suite.suite_name.lower()
730
731 if "obsolete source" in checks:
732 report_obsolete_source(suite_name, session)
733
734 if "nbs" in checks:
735 reportAllNBS(suite_name, suite_id, session, rdeps)
736
737 if "nbs metadata" in checks:
738 reportNBSMetadata(suite_name, suite_id, session, rdeps)
739
740 if "outdated non-free" in checks:
741 report_outdated_nonfree(suite_name, session, rdeps)
742
743 bin_not_built = defaultdict(set)
744
745 if "bnb" in checks:
746 bins_in_suite = get_suite_binaries(suite, session)
747
748
749 components = [c.component_name for c in suite.components]
750 for component in [c.component_name for c in suite.components]:
751 filename = "%s/dists/%s/%s/source/Sources" % (
752 suite.archive.path,
753 suite_name,
754 component,
755 )
756 filename = utils.find_possibly_compressed_file(filename)
757 with apt_pkg.TagFile(filename) as Sources:
758 while Sources.step():
759 source = Sources.section.find("Package")
760 source_version = Sources.section.find("Version")
761 architecture = Sources.section.find("Architecture")
762 binaries = Sources.section.find("Binary")
763 binaries_list = [i.strip() for i in binaries.split(",")]
764
765 if "bnb" in checks:
766
767 for binary in binaries_list:
768 if binary not in bins_in_suite:
769 bin_not_built[source].add(binary)
770
771 if "anais" in checks:
772 anais_output += do_anais(
773 architecture, binaries_list, source, session
774 )
775
776
777 source_index = component + "/" + source
778 src_pkgs[source] = source_index
779 for binary in binaries_list:
780 bin_pkgs[binary] = source
781 source_binaries[source] = binaries
782 source_versions[source] = source_version
783
784
785 check_components = components[:]
786 if suite_name != "experimental":
787 check_components.append("main/debian-installer")
788
789 for component in check_components:
790 architectures = [
791 a.arch_string
792 for a in get_suite_architectures(
793 suite_name, skipsrc=True, skipall=True, session=session
794 )
795 ]
796 for architecture in architectures:
797 if component == "main/debian-installer" and re.match(
798 "kfreebsd", architecture
799 ):
800 continue
801
802 if "nfu" in checks:
803 nfu_entries = parse_nfu(architecture)
804
805 filename = "%s/dists/%s/%s/binary-%s/Packages" % (
806 suite.archive.path,
807 suite_name,
808 component,
809 architecture,
810 )
811 filename = utils.find_possibly_compressed_file(filename)
812 with apt_pkg.TagFile(filename) as Packages:
813 while Packages.step():
814 package = Packages.section.find("Package")
815 source = Packages.section.find("Source", "")
816 version = Packages.section.find("Version")
817 if source == "":
818 source = package
819 if (
820 package in bin2source
821 and apt_pkg.version_compare(
822 version, bin2source[package]["version"]
823 )
824 > 0
825 ):
826 bin2source[package]["version"] = version
827 bin2source[package]["source"] = source
828 else:
829 bin2source[package] = {}
830 bin2source[package]["version"] = version
831 bin2source[package]["source"] = source
832 if source.find("(") != -1:
833 m = re_extract_src_version.match(source)
834 source = m.group(1)
835 version = m.group(2)
836 if package not in bin_pkgs:
837 nbs[source][package].add(version)
838 else:
839 if "nfu" in checks:
840 if (
841 package in nfu_entries
842 and version != source_versions[source]
843 ):
844 nfu_packages[architecture].append(
845 (package, version, source_versions[source])
846 )
847
848
849 dubious_nbs = defaultdict(lambda: defaultdict(set))
850 version_sort_key = functools.cmp_to_key(apt_pkg.version_compare)
851 for source in nbs:
852 for package in nbs[source]:
853 latest_version = max(nbs[source][package], key=version_sort_key)
854 source_version = source_versions.get(source, "0")
855 if apt_pkg.version_compare(latest_version, source_version) == 0:
856 add_nbs(dubious_nbs, source, latest_version, package, suite_id, session)
857
858 if "nviu" in checks:
859 do_newer_version("unstable", "experimental", "NVIU", session)
860
861 if "nvit" in checks:
862 do_newer_version("testing", "testing-proposed-updates", "NVIT", session)
863
864
865
866 if Options["Mode"] == "full":
867 print_info("=" * 75)
868 print_info()
869
870 if "nfu" in checks:
871 do_nfu(nfu_packages)
872
873 if "bnb" in checks:
874 print_info("Unbuilt binary packages")
875 print_info("-----------------------")
876 print_info()
877 for source in sorted(bin_not_built):
878 binaries = sorted(bin_not_built[source])
879 print_info(" o %s: %s" % (source, ", ".join(binaries)))
880 print_info()
881
882 if "bms" in checks:
883 report_multiple_source(suite)
884
885 if "anais" in checks:
886 print_info("Architecture Not Allowed In Source")
887 print_info("----------------------------------")
888 print_info(anais_output)
889 print_info()
890
891 if "dubious nbs" in checks:
892 do_dubious_nbs(dubious_nbs)
893
894
895
896
897 if __name__ == "__main__":
898 main()
899