Coverage for dak/check_overrides.py: 78%
161 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-01-04 16:18 +0000
1#! /usr/bin/env python3
3"""Cruft checker and hole filler for overrides
5@contact: Debian FTPMaster <ftpmaster@debian.org>
6@copyright: 2000, 2001, 2002, 2004, 2006 James Troup <james@nocrew.org>
7@opyright: 2005 Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
8@copyright: 2011 Joerg Jaspert <joerg@debian.org>
9@license: GNU General Public License version 2 or later
11"""
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27################################################################################
29######################################################################
30# NB: dak check-overrides is not a good idea with New Incoming as it #
31# doesn't take into account accepted. You can minimize the impact #
32# of this by running it immediately after dak process-accepted but #
33# that's still racy because 'dak process-new' doesn't lock with 'dak #
34# process-accepted'. A better long term fix is the evil plan for #
35# accepted to be in the DB. #
36######################################################################
38# dak check-overrides should now work fine being done during
39# cron.daily, for example just before 'dak make-overrides' (after 'dak
40# process-accepted' and 'dak make-suite-file-list'). At that point,
41# queue/accepted should be empty and installed, so... dak
42# check-overrides does now take into account suites sharing overrides
44# TODO:
45# * Only update out-of-sync overrides when corresponding versions are equal to
46# some degree
47# * consistency checks like:
48# - section=debian-installer only for udeb and # dsc
49# - priority=optional if dsc
50# - (suite, package, 'dsc') is unique,
51# - just as (suite, package, (u)deb) (yes, across components!)
52# - sections match their component (each component has an own set of sections,
53# could probably be reduced...)
55################################################################################
57import sys
58from typing import TYPE_CHECKING, NoReturn
60import apt_pkg
61from sqlalchemy import sql
63from daklib import daklog, utils
64from daklib.config import Config
65from daklib.dbconn import (
66 Component,
67 DBConn,
68 OverrideType,
69 Suite,
70 get_component,
71 get_override_type,
72 get_priorities,
73 get_priority,
74 get_sections,
75 get_suite,
76)
78if TYPE_CHECKING:
79 from sqlalchemy.orm import Session
81################################################################################
83Options: apt_pkg.Configuration #: Commandline arguments parsed into this
84Logger: daklog.Logger #: Our logging object
85sections: dict[int, str] = {}
86priorities: dict[int, str] = {}
87blacklist: set[str] = set()
89################################################################################
92def usage(exit_code=0) -> NoReturn:
93 print(
94 """Usage: dak check-overrides
95Check for cruft in overrides.
97 -n, --no-action don't do anything
98 -h, --help show this help and exit"""
99 )
101 sys.exit(exit_code)
104################################################################################
107def process(
108 osuite: str,
109 affected_suites: list[int],
110 originosuite: str | None,
111 component: str,
112 otype: str,
113 session: "Session",
114) -> None:
115 global Logger, Options, sections, priorities
117 o = get_suite(osuite, session)
118 if o is None: 118 ↛ 119line 118 didn't jump to line 119 because the condition on line 118 was never true
119 utils.fubar("Suite '%s' not recognised." % (osuite))
120 osuite_id = o.suite_id
122 originosuite_id = None
123 if originosuite:
124 oo = get_suite(originosuite, session)
125 if oo is None: 125 ↛ 126line 125 didn't jump to line 126 because the condition on line 125 was never true
126 utils.fubar("Suite '%s' not recognised." % (originosuite))
127 originosuite_id = oo.suite_id
129 c = get_component(component, session)
130 if c is None: 130 ↛ 131line 130 didn't jump to line 131 because the condition on line 130 was never true
131 utils.fubar("Component '%s' not recognised." % (component))
132 component_id = c.component_id
134 ot = get_override_type(otype, session)
135 if ot is None: 135 ↛ 136line 135 didn't jump to line 136 because the condition on line 135 was never true
136 utils.fubar(
137 "Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype)
138 )
139 type_id = ot.overridetype_id
140 dsc_type = get_override_type("dsc", session)
141 assert dsc_type is not None
142 dsc_type_id = dsc_type.overridetype_id
144 source_priority = get_priority("optional", session)
145 assert source_priority is not None
146 source_priority_id = source_priority.priority_id
148 if otype == "deb" or otype == "udeb":
149 packages = {}
150 # TODO: Fix to use placeholders (check how to with arrays)
151 q = session.execute(
152 sql.text(
153 """
154SELECT b.package
155 FROM binaries b
156 JOIN bin_associations ba ON b.id = ba.bin
157 JOIN suite ON ba.suite = suite.id
158 JOIN files_archive_map af ON b.file = af.file_id AND suite.archive_id = af.archive_id
159 WHERE b.type = :otype AND ba.suite IN :affected_suites AND af.component_id = :component_id
160"""
161 ),
162 {
163 "otype": otype,
164 "affected_suites": tuple(affected_suites),
165 "component_id": component_id,
166 },
167 )
168 for i in q.fetchall():
169 packages[i[0]] = 0
171 src_packages = {}
172 q = session.execute(
173 sql.text(
174 """
175SELECT s.source FROM source s
176 JOIN src_associations sa ON s.id = sa.source
177 JOIN suite ON sa.suite = suite.id
178 JOIN files_archive_map af ON s.file = af.file_id AND suite.archive_id = af.archive_id
179 WHERE sa.suite IN :affected_suites AND af.component_id = :component_id
180"""
181 ),
182 {"affected_suites": tuple(affected_suites), "component_id": component_id},
183 )
184 for i in q.fetchall():
185 src_packages[i[0]] = 0
187 # -----------
188 # Drop unused overrides
190 q = session.execute(
191 sql.text(
192 """SELECT package, priority, section, maintainer
193 FROM override WHERE suite = :suite_id
194 AND component = :component_id AND type = :type_id"""
195 ),
196 {"suite_id": osuite_id, "component_id": component_id, "type_id": type_id},
197 )
198 # We're already within a transaction
199 if otype == "dsc":
200 for i in q.fetchall():
201 package = i[0]
202 if package in src_packages: 202 ↛ 205line 202 didn't jump to line 205 because the condition on line 202 was always true
203 src_packages[package] = 1
204 else:
205 if package in blacklist:
206 utils.warn("%s in incoming, not touching" % package)
207 continue
208 Logger.log(
209 [
210 "removing unused override",
211 osuite,
212 component,
213 otype,
214 package,
215 priorities[i[1]],
216 sections[i[2]],
217 i[3],
218 ]
219 )
220 if not Options["No-Action"]:
221 session.execute(
222 sql.text(
223 """DELETE FROM override WHERE package = :package
224 AND suite = :suite_id AND component = :component_id
225 AND type = :type_id
226 AND created < now() - interval '14 days'"""
227 ),
228 {
229 "package": package,
230 "suite_id": osuite_id,
231 "component_id": component_id,
232 "type_id": type_id,
233 },
234 )
235 # create source overrides based on binary overrides, as source
236 # overrides not always get created
237 q = session.execute(
238 sql.text(
239 """SELECT package, priority, section, maintainer
240 FROM override WHERE suite = :suite_id AND component = :component_id"""
241 ),
242 {"suite_id": osuite_id, "component_id": component_id},
243 )
244 for i in q.fetchall():
245 package = i[0]
246 if package not in src_packages or src_packages[package]: 246 ↛ 248line 246 didn't jump to line 248 because the condition on line 246 was always true
247 continue
248 src_packages[package] = 1
250 Logger.log(
251 [
252 "add missing override",
253 osuite,
254 component,
255 otype,
256 package,
257 "source",
258 sections[i[2]],
259 i[3],
260 ]
261 )
262 if not Options["No-Action"]:
263 session.execute(
264 sql.text(
265 """INSERT INTO override (package, suite, component,
266 priority, section, type, maintainer)
267 VALUES (:package, :suite_id, :component_id,
268 :priority_id, :section_id, :type_id, :maintainer)"""
269 ),
270 {
271 "package": package,
272 "suite_id": osuite_id,
273 "component_id": component_id,
274 "priority_id": source_priority_id,
275 "section_id": i[2],
276 "type_id": dsc_type_id,
277 "maintainer": i[3],
278 },
279 )
280 # Check whether originosuite has an override for us we can
281 # copy
282 if originosuite:
283 q = session.execute(
284 sql.text(
285 """SELECT origin.package, origin.priority, origin.section,
286 origin.maintainer, target.priority, target.section,
287 target.maintainer
288 FROM override origin
289 LEFT JOIN override target ON (origin.package = target.package
290 AND target.suite = :suite_id
291 AND origin.component = target.component
292 AND origin.type = target.type)
293 WHERE origin.suite = :originsuite_id
294 AND origin.component = :component_id
295 AND origin.type = :type_id"""
296 ),
297 {
298 "suite_id": osuite_id,
299 "originsuite_id": originosuite_id,
300 "component_id": component_id,
301 "type_id": type_id,
302 },
303 )
304 for i in q.fetchall():
305 package = i[0]
306 if package not in src_packages or src_packages[package]:
307 if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]): 307 ↛ 308line 307 didn't jump to line 308 because the condition on line 307 was never true
308 Logger.log(
309 [
310 "syncing override",
311 osuite,
312 component,
313 otype,
314 package,
315 "source",
316 sections[i[5]],
317 i[6],
318 "source",
319 sections[i[2]],
320 i[3],
321 ]
322 )
323 if not Options["No-Action"]:
324 session.execute(
325 sql.text(
326 """UPDATE override
327 SET priority = :priority,
328 section = :section,
329 maintainer = :maintainer
330 WHERE package = :package AND suite = :suite_id
331 AND component = :component_id AND type = :type_id"""
332 ),
333 {
334 "priority": i[1],
335 "section": i[2],
336 "maintainer": i[3],
337 "package": package,
338 "suite_id": osuite_id,
339 "component_id": component_id,
340 "type_id": dsc_type_id,
341 },
342 )
343 continue
345 # we can copy
346 src_packages[package] = 1
347 Logger.log(
348 [
349 "copying missing override",
350 osuite,
351 component,
352 otype,
353 package,
354 "source",
355 sections[i[2]],
356 i[3],
357 ]
358 )
359 if not Options["No-Action"]: 359 ↛ 304line 359 didn't jump to line 304 because the condition on line 359 was always true
360 session.execute(
361 sql.text(
362 """INSERT INTO override (package, suite, component,
363 priority, section, type, maintainer)
364 VALUES (:package, :suite_id, :component_id,
365 :priority_id, :section_id, :type_id,
366 :maintainer)"""
367 ),
368 {
369 "package": package,
370 "suite_id": osuite_id,
371 "component_id": component_id,
372 "priority_id": source_priority_id,
373 "section_id": i[2],
374 "type_id": dsc_type_id,
375 "maintainer": i[3],
376 },
377 )
379 for package, hasoverride in list(src_packages.items()):
380 if not hasoverride: 380 ↛ 381line 380 didn't jump to line 381 because the condition on line 380 was never true
381 utils.warn("%s has no override!" % package)
383 else: # binary override
384 for i in q.fetchall():
385 package = i[0]
386 if package in packages:
387 packages[package] = 1
388 else:
389 if package in blacklist: 389 ↛ 390line 389 didn't jump to line 390 because the condition on line 389 was never true
390 utils.warn("%s in incoming, not touching" % package)
391 continue
392 Logger.log(
393 [
394 "removing unused override",
395 osuite,
396 component,
397 otype,
398 package,
399 priorities[i[1]],
400 sections[i[2]],
401 i[3],
402 ]
403 )
404 if not Options["No-Action"]: 404 ↛ 384line 404 didn't jump to line 384 because the condition on line 404 was always true
405 session.execute(
406 sql.text(
407 """DELETE FROM override
408 WHERE package = :package AND suite = :suite_id
409 AND component = :component_id AND type = :type_id
410 AND created < now() - interval '14 days'"""
411 ),
412 {
413 "package": package,
414 "suite_id": osuite_id,
415 "component_id": component_id,
416 "type_id": type_id,
417 },
418 )
420 # Check whether originosuite has an override for us we can
421 # copy
422 if originosuite:
423 q = session.execute(
424 sql.text(
425 """SELECT origin.package, origin.priority, origin.section,
426 origin.maintainer, target.priority, target.section,
427 target.maintainer
428 FROM override origin LEFT JOIN override target
429 ON (origin.package = target.package
430 AND target.suite = :suite_id
431 AND origin.component = target.component
432 AND origin.type = target.type)
433 WHERE origin.suite = :originsuite_id
434 AND origin.component = :component_id
435 AND origin.type = :type_id"""
436 ),
437 {
438 "suite_id": osuite_id,
439 "originsuite_id": originosuite_id,
440 "component_id": component_id,
441 "type_id": type_id,
442 },
443 )
444 for i in q.fetchall():
445 package = i[0]
446 if package not in packages or packages[package]:
447 if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]): 447 ↛ 448line 447 didn't jump to line 448 because the condition on line 447 was never true
448 Logger.log(
449 [
450 "syncing override",
451 osuite,
452 component,
453 otype,
454 package,
455 priorities[i[4]],
456 sections[i[5]],
457 i[6],
458 priorities[i[1]],
459 sections[i[2]],
460 i[3],
461 ]
462 )
463 if not Options["No-Action"]:
464 session.execute(
465 sql.text(
466 """UPDATE override
467 SET priority = :priority_id,
468 section = :section_id,
469 maintainer = :maintainer
470 WHERE package = :package
471 AND suite = :suite_id
472 AND component = :component_id
473 AND type = :type_id"""
474 ),
475 {
476 "priority_id": i[1],
477 "section_id": i[2],
478 "maintainer": i[3],
479 "package": package,
480 "suite_id": osuite_id,
481 "component_id": component_id,
482 "type_id": type_id,
483 },
484 )
485 continue
486 # we can copy
487 packages[package] = 1
488 Logger.log(
489 [
490 "copying missing override",
491 osuite,
492 component,
493 otype,
494 package,
495 priorities[i[1]],
496 sections[i[2]],
497 i[3],
498 ]
499 )
500 if not Options["No-Action"]: 500 ↛ 444line 500 didn't jump to line 444 because the condition on line 500 was always true
501 session.execute(
502 sql.text(
503 """INSERT INTO override (package, suite, component,
504 priority, section, type, maintainer)
505 VALUES (:package, :suite_id, :component_id,
506 :priority_id, :section_id, :type_id, :maintainer)"""
507 ),
508 {
509 "package": package,
510 "suite_id": osuite_id,
511 "component_id": component_id,
512 "priority_id": i[1],
513 "section_id": i[2],
514 "type_id": type_id,
515 "maintainer": i[3],
516 },
517 )
519 for package, hasoverride in list(packages.items()):
520 if not hasoverride: 520 ↛ 521line 520 didn't jump to line 521 because the condition on line 520 was never true
521 utils.warn("%s has no override!" % package)
523 session.commit()
524 sys.stdout.flush()
527################################################################################
530def main() -> None:
531 global Logger, Options, sections, priorities
533 cnf = Config()
535 Arguments = [
536 ("h", "help", "Check-Overrides::Options::Help"),
537 ("n", "no-action", "Check-Overrides::Options::No-Action"),
538 ]
539 for i in ["help", "no-action"]:
540 key = "Check-Overrides::Options::%s" % i
541 if key not in cnf: 541 ↛ 539line 541 didn't jump to line 539 because the condition on line 541 was always true
542 cnf[key] = ""
543 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) # type: ignore[attr-defined]
544 Options = cnf.subtree("Check-Overrides::Options")
546 if Options["Help"]:
547 usage()
549 session = DBConn().session()
551 # init sections, priorities:
553 # We need forward and reverse
554 sections = {entry: name for name, entry in get_sections(session).items()}
555 priorities = {entry: name for name, entry in get_priorities(session).items()}
557 if not Options["No-Action"]: 557 ↛ 560line 557 didn't jump to line 560 because the condition on line 557 was always true
558 Logger = daklog.Logger("check-overrides")
559 else:
560 Logger = daklog.Logger("check-overrides", 1)
562 for suite in session.query(Suite).filter(
563 Suite.overrideprocess == True # noqa:E712
564 ):
565 originosuite_name: str | None = None
566 originremark = ""
568 if suite.overrideorigin is not None:
569 originosuite = get_suite(suite.overrideorigin, session)
570 if originosuite is None: 570 ↛ 571line 570 didn't jump to line 571 because the condition on line 570 was never true
571 utils.fubar(
572 "%s has an override origin suite of %s but it doesn't exist!"
573 % (suite.suite_name, suite.overrideorigin)
574 )
575 originosuite_name = originosuite.suite_name
576 originremark = " taking missing from %s" % originosuite_name
578 print("Processing %s%s..." % (suite.suite_name, originremark))
580 # Get a list of all suites that use the override file of 'suite.suite_name' as
581 # well as the suite
582 ocodename = suite.codename
583 suiteids = [
584 x.suite_id
585 for x in session.query(Suite)
586 .filter(Suite.overridecodename == ocodename)
587 .all()
588 ]
589 if suite.suite_id not in suiteids: 589 ↛ 592line 589 didn't jump to line 592 because the condition on line 589 was always true
590 suiteids.append(suite.suite_id)
592 if len(suiteids) < 1: 592 ↛ 593line 592 didn't jump to line 593 because the condition on line 592 was never true
593 utils.fubar("Couldn't find id's of all suites: %s" % suiteids)
595 for component in session.query(Component).all():
596 # It is crucial for the dsc override creation based on binary
597 # overrides that 'dsc' goes first
598 component_name = component.component_name
599 otypes = ["dsc"]
600 for ot in session.query(OverrideType):
601 if ot.overridetype == "dsc":
602 continue
603 otypes.append(ot.overridetype)
605 for otype in otypes:
606 print(
607 "Processing %s [%s - %s]"
608 % (suite.suite_name, component_name, otype)
609 )
610 sys.stdout.flush()
611 process(
612 suite.suite_name,
613 suiteids,
614 originosuite_name,
615 component_name,
616 otype,
617 session,
618 )
620 Logger.close()
623################################################################################
626if __name__ == "__main__":
627 main()