Coverage for dak/check_overrides.py: 78%
160 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-08-03 16:46 +0000
1"""Cruft checker and hole filler for overrides
3@contact: Debian FTPMaster <ftpmaster@debian.org>
4@copyright: 2000, 2001, 2002, 2004, 2006 James Troup <james@nocrew.org>
5@opyright: 2005 Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
6@copyright: 2011 Joerg Jaspert <joerg@debian.org>
7@license: GNU General Public License version 2 or later
9"""
11# This program is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation; either version 2 of the License, or
14# (at your option) any later version.
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
21# You should have received a copy of the GNU General Public License
22# along with this program; if not, write to the Free Software
23# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25################################################################################
27######################################################################
28# NB: dak check-overrides is not a good idea with New Incoming as it #
29# doesn't take into account accepted. You can minimize the impact #
30# of this by running it immediately after dak process-accepted but #
31# that's still racy because 'dak process-new' doesn't lock with 'dak #
32# process-accepted'. A better long term fix is the evil plan for #
33# accepted to be in the DB. #
34######################################################################
36# dak check-overrides should now work fine being done during
37# cron.daily, for example just before 'dak make-overrides' (after 'dak
38# process-accepted' and 'dak make-suite-file-list'). At that point,
39# queue/accepted should be empty and installed, so... dak
40# check-overrides does now take into account suites sharing overrides
42# TODO:
43# * Only update out-of-sync overrides when corresponding versions are equal to
44# some degree
45# * consistency checks like:
46# - section=debian-installer only for udeb and # dsc
47# - priority=optional if dsc
48# - (suite, package, 'dsc') is unique,
49# - just as (suite, package, (u)deb) (yes, across components!)
50# - sections match their component (each component has an own set of sections,
51# could probably be reduced...)
53################################################################################
55import sys
56from typing import TYPE_CHECKING, NoReturn
58import apt_pkg
59from sqlalchemy import select, sql
61from daklib import daklog, utils
62from daklib.config import Config
63from daklib.dbconn import (
64 Component,
65 DBConn,
66 OverrideType,
67 Suite,
68 get_component,
69 get_override_type,
70 get_priorities,
71 get_priority,
72 get_sections,
73 get_suite,
74)
76if TYPE_CHECKING:
77 from sqlalchemy.orm import Session
79################################################################################
81Options: apt_pkg.Configuration #: Commandline arguments parsed into this
82Logger: daklog.Logger #: Our logging object
83sections: dict[int, str] = {}
84priorities: dict[int, str] = {}
85blacklist: set[str] = set()
87################################################################################
90def usage(exit_code=0) -> NoReturn:
91 print(
92 """Usage: dak check-overrides
93Check for cruft in overrides.
95 -n, --no-action don't do anything
96 -h, --help show this help and exit"""
97 )
99 sys.exit(exit_code)
102################################################################################
105def process(
106 osuite: str,
107 affected_suites: list[int],
108 originosuite: str | None,
109 component: str,
110 otype: str,
111 session: "Session",
112) -> None:
113 global Logger, Options, sections, priorities
115 o = get_suite(osuite, session)
116 if o is None: 116 ↛ 117line 116 didn't jump to line 117 because the condition on line 116 was never true
117 utils.fubar("Suite '%s' not recognised." % (osuite))
118 osuite_id = o.suite_id
120 originosuite_id = None
121 if originosuite:
122 oo = get_suite(originosuite, session)
123 if oo is None: 123 ↛ 124line 123 didn't jump to line 124 because the condition on line 123 was never true
124 utils.fubar("Suite '%s' not recognised." % (originosuite))
125 originosuite_id = oo.suite_id
127 c = get_component(component, session)
128 if c is None: 128 ↛ 129line 128 didn't jump to line 129 because the condition on line 128 was never true
129 utils.fubar("Component '%s' not recognised." % (component))
130 component_id = c.component_id
132 ot = get_override_type(otype, session)
133 if ot is None: 133 ↛ 134line 133 didn't jump to line 134 because the condition on line 133 was never true
134 utils.fubar(
135 "Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype)
136 )
137 type_id = ot.overridetype_id
138 dsc_type = get_override_type("dsc", session)
139 assert dsc_type is not None
140 dsc_type_id = dsc_type.overridetype_id
142 source_priority = get_priority("optional", session)
143 assert source_priority is not None
144 source_priority_id = source_priority.priority_id
146 if otype == "deb" or otype == "udeb":
147 packages = {}
148 # TODO: Fix to use placeholders (check how to with arrays)
149 q = session.execute(
150 sql.text(
151 """
152SELECT b.package
153 FROM binaries b
154 JOIN bin_associations ba ON b.id = ba.bin
155 JOIN suite ON ba.suite = suite.id
156 JOIN files_archive_map af ON b.file = af.file_id AND suite.archive_id = af.archive_id
157 WHERE b.type = :otype AND ba.suite IN :affected_suites AND af.component_id = :component_id
158"""
159 ),
160 {
161 "otype": otype,
162 "affected_suites": tuple(affected_suites),
163 "component_id": component_id,
164 },
165 )
166 for i in q.fetchall():
167 packages[i[0]] = 0
169 src_packages = {}
170 q = session.execute(
171 sql.text(
172 """
173SELECT s.source FROM source s
174 JOIN src_associations sa ON s.id = sa.source
175 JOIN suite ON sa.suite = suite.id
176 JOIN files_archive_map af ON s.file = af.file_id AND suite.archive_id = af.archive_id
177 WHERE sa.suite IN :affected_suites AND af.component_id = :component_id
178"""
179 ),
180 {"affected_suites": tuple(affected_suites), "component_id": component_id},
181 )
182 for i in q.fetchall():
183 src_packages[i[0]] = 0
185 # -----------
186 # Drop unused overrides
188 q = session.execute(
189 sql.text(
190 """SELECT package, priority, section, maintainer
191 FROM override WHERE suite = :suite_id
192 AND component = :component_id AND type = :type_id"""
193 ),
194 {"suite_id": osuite_id, "component_id": component_id, "type_id": type_id},
195 )
196 # We're already within a transaction
197 if otype == "dsc":
198 for i in q.fetchall():
199 package = i[0]
200 if package in src_packages: 200 ↛ 203line 200 didn't jump to line 203 because the condition on line 200 was always true
201 src_packages[package] = 1
202 else:
203 if package in blacklist:
204 utils.warn("%s in incoming, not touching" % package)
205 continue
206 Logger.log(
207 [
208 "removing unused override",
209 osuite,
210 component,
211 otype,
212 package,
213 priorities[i[1]],
214 sections[i[2]],
215 i[3],
216 ]
217 )
218 if not Options["No-Action"]:
219 session.execute(
220 sql.text(
221 """DELETE FROM override WHERE package = :package
222 AND suite = :suite_id AND component = :component_id
223 AND type = :type_id
224 AND created < now() - interval '14 days'"""
225 ),
226 {
227 "package": package,
228 "suite_id": osuite_id,
229 "component_id": component_id,
230 "type_id": type_id,
231 },
232 )
233 # create source overrides based on binary overrides, as source
234 # overrides not always get created
235 q = session.execute(
236 sql.text(
237 """SELECT package, priority, section, maintainer
238 FROM override WHERE suite = :suite_id AND component = :component_id"""
239 ),
240 {"suite_id": osuite_id, "component_id": component_id},
241 )
242 for i in q.fetchall():
243 package = i[0]
244 if package not in src_packages or src_packages[package]: 244 ↛ 246line 244 didn't jump to line 246 because the condition on line 244 was always true
245 continue
246 src_packages[package] = 1
248 Logger.log(
249 [
250 "add missing override",
251 osuite,
252 component,
253 otype,
254 package,
255 "source",
256 sections[i[2]],
257 i[3],
258 ]
259 )
260 if not Options["No-Action"]:
261 session.execute(
262 sql.text(
263 """INSERT INTO override (package, suite, component,
264 priority, section, type, maintainer)
265 VALUES (:package, :suite_id, :component_id,
266 :priority_id, :section_id, :type_id, :maintainer)"""
267 ),
268 {
269 "package": package,
270 "suite_id": osuite_id,
271 "component_id": component_id,
272 "priority_id": source_priority_id,
273 "section_id": i[2],
274 "type_id": dsc_type_id,
275 "maintainer": i[3],
276 },
277 )
278 # Check whether originosuite has an override for us we can
279 # copy
280 if originosuite:
281 q = session.execute(
282 sql.text(
283 """SELECT origin.package, origin.priority, origin.section,
284 origin.maintainer, target.priority, target.section,
285 target.maintainer
286 FROM override origin
287 LEFT JOIN override target ON (origin.package = target.package
288 AND target.suite = :suite_id
289 AND origin.component = target.component
290 AND origin.type = target.type)
291 WHERE origin.suite = :originsuite_id
292 AND origin.component = :component_id
293 AND origin.type = :type_id"""
294 ),
295 {
296 "suite_id": osuite_id,
297 "originsuite_id": originosuite_id,
298 "component_id": component_id,
299 "type_id": type_id,
300 },
301 )
302 for i in q.fetchall():
303 package = i[0]
304 if package not in src_packages or src_packages[package]:
305 if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]): 305 ↛ 306line 305 didn't jump to line 306 because the condition on line 305 was never true
306 Logger.log(
307 [
308 "syncing override",
309 osuite,
310 component,
311 otype,
312 package,
313 "source",
314 sections[i[5]],
315 i[6],
316 "source",
317 sections[i[2]],
318 i[3],
319 ]
320 )
321 if not Options["No-Action"]:
322 session.execute(
323 sql.text(
324 """UPDATE override
325 SET priority = :priority,
326 section = :section,
327 maintainer = :maintainer
328 WHERE package = :package AND suite = :suite_id
329 AND component = :component_id AND type = :type_id"""
330 ),
331 {
332 "priority": i[1],
333 "section": i[2],
334 "maintainer": i[3],
335 "package": package,
336 "suite_id": osuite_id,
337 "component_id": component_id,
338 "type_id": dsc_type_id,
339 },
340 )
341 continue
343 # we can copy
344 src_packages[package] = 1
345 Logger.log(
346 [
347 "copying missing override",
348 osuite,
349 component,
350 otype,
351 package,
352 "source",
353 sections[i[2]],
354 i[3],
355 ]
356 )
357 if not Options["No-Action"]: 357 ↛ 302line 357 didn't jump to line 302 because the condition on line 357 was always true
358 session.execute(
359 sql.text(
360 """INSERT INTO override (package, suite, component,
361 priority, section, type, maintainer)
362 VALUES (:package, :suite_id, :component_id,
363 :priority_id, :section_id, :type_id,
364 :maintainer)"""
365 ),
366 {
367 "package": package,
368 "suite_id": osuite_id,
369 "component_id": component_id,
370 "priority_id": source_priority_id,
371 "section_id": i[2],
372 "type_id": dsc_type_id,
373 "maintainer": i[3],
374 },
375 )
377 for package, hasoverride in list(src_packages.items()):
378 if not hasoverride: 378 ↛ 379line 378 didn't jump to line 379 because the condition on line 378 was never true
379 utils.warn("%s has no override!" % package)
381 else: # binary override
382 for i in q.fetchall():
383 package = i[0]
384 if package in packages:
385 packages[package] = 1
386 else:
387 if package in blacklist: 387 ↛ 388line 387 didn't jump to line 388 because the condition on line 387 was never true
388 utils.warn("%s in incoming, not touching" % package)
389 continue
390 Logger.log(
391 [
392 "removing unused override",
393 osuite,
394 component,
395 otype,
396 package,
397 priorities[i[1]],
398 sections[i[2]],
399 i[3],
400 ]
401 )
402 if not Options["No-Action"]: 402 ↛ 382line 402 didn't jump to line 382 because the condition on line 402 was always true
403 session.execute(
404 sql.text(
405 """DELETE FROM override
406 WHERE package = :package AND suite = :suite_id
407 AND component = :component_id AND type = :type_id
408 AND created < now() - interval '14 days'"""
409 ),
410 {
411 "package": package,
412 "suite_id": osuite_id,
413 "component_id": component_id,
414 "type_id": type_id,
415 },
416 )
418 # Check whether originosuite has an override for us we can
419 # copy
420 if originosuite:
421 q = session.execute(
422 sql.text(
423 """SELECT origin.package, origin.priority, origin.section,
424 origin.maintainer, target.priority, target.section,
425 target.maintainer
426 FROM override origin LEFT JOIN override target
427 ON (origin.package = target.package
428 AND target.suite = :suite_id
429 AND origin.component = target.component
430 AND origin.type = target.type)
431 WHERE origin.suite = :originsuite_id
432 AND origin.component = :component_id
433 AND origin.type = :type_id"""
434 ),
435 {
436 "suite_id": osuite_id,
437 "originsuite_id": originosuite_id,
438 "component_id": component_id,
439 "type_id": type_id,
440 },
441 )
442 for i in q.fetchall():
443 package = i[0]
444 if package not in packages or packages[package]:
445 if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]): 445 ↛ 446line 445 didn't jump to line 446 because the condition on line 445 was never true
446 Logger.log(
447 [
448 "syncing override",
449 osuite,
450 component,
451 otype,
452 package,
453 priorities[i[4]],
454 sections[i[5]],
455 i[6],
456 priorities[i[1]],
457 sections[i[2]],
458 i[3],
459 ]
460 )
461 if not Options["No-Action"]:
462 session.execute(
463 sql.text(
464 """UPDATE override
465 SET priority = :priority_id,
466 section = :section_id,
467 maintainer = :maintainer
468 WHERE package = :package
469 AND suite = :suite_id
470 AND component = :component_id
471 AND type = :type_id"""
472 ),
473 {
474 "priority_id": i[1],
475 "section_id": i[2],
476 "maintainer": i[3],
477 "package": package,
478 "suite_id": osuite_id,
479 "component_id": component_id,
480 "type_id": type_id,
481 },
482 )
483 continue
484 # we can copy
485 packages[package] = 1
486 Logger.log(
487 [
488 "copying missing override",
489 osuite,
490 component,
491 otype,
492 package,
493 priorities[i[1]],
494 sections[i[2]],
495 i[3],
496 ]
497 )
498 if not Options["No-Action"]: 498 ↛ 442line 498 didn't jump to line 442 because the condition on line 498 was always true
499 session.execute(
500 sql.text(
501 """INSERT INTO override (package, suite, component,
502 priority, section, type, maintainer)
503 VALUES (:package, :suite_id, :component_id,
504 :priority_id, :section_id, :type_id, :maintainer)"""
505 ),
506 {
507 "package": package,
508 "suite_id": osuite_id,
509 "component_id": component_id,
510 "priority_id": i[1],
511 "section_id": i[2],
512 "type_id": type_id,
513 "maintainer": i[3],
514 },
515 )
517 for package, hasoverride in list(packages.items()):
518 if not hasoverride: 518 ↛ 519line 518 didn't jump to line 519 because the condition on line 518 was never true
519 utils.warn("%s has no override!" % package)
521 session.commit()
522 sys.stdout.flush()
525################################################################################
528def main() -> None:
529 global Logger, Options, sections, priorities
531 cnf = Config()
533 Arguments = [
534 ("h", "help", "Check-Overrides::Options::Help"),
535 ("n", "no-action", "Check-Overrides::Options::No-Action"),
536 ]
537 for i in ["help", "no-action"]:
538 key = "Check-Overrides::Options::%s" % i
539 if key not in cnf: 539 ↛ 537line 539 didn't jump to line 537 because the condition on line 539 was always true
540 cnf[key] = ""
541 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) # type: ignore[attr-defined]
542 Options = cnf.subtree("Check-Overrides::Options")
544 if Options["Help"]:
545 usage()
547 session = DBConn().session()
549 # init sections, priorities:
551 # We need forward and reverse
552 sections = {entry: name for name, entry in get_sections(session).items()}
553 priorities = {entry: name for name, entry in get_priorities(session).items()}
555 if not Options["No-Action"]: 555 ↛ 558line 555 didn't jump to line 558 because the condition on line 555 was always true
556 Logger = daklog.Logger("check-overrides")
557 else:
558 Logger = daklog.Logger("check-overrides", 1)
560 for suite in session.scalars(select(Suite).where(Suite.overrideprocess)):
561 originosuite_name: str | None = None
562 originremark = ""
564 if suite.overrideorigin is not None:
565 originosuite = get_suite(suite.overrideorigin, session)
566 if originosuite is None: 566 ↛ 567line 566 didn't jump to line 567 because the condition on line 566 was never true
567 utils.fubar(
568 "%s has an override origin suite of %s but it doesn't exist!"
569 % (suite.suite_name, suite.overrideorigin)
570 )
571 originosuite_name = originosuite.suite_name
572 originremark = " taking missing from %s" % originosuite_name
574 print("Processing %s%s..." % (suite.suite_name, originremark))
576 # Get a list of all suites that use the override file of 'suite.suite_name' as
577 # well as the suite
578 ocodename = suite.codename
579 suiteids = list(
580 session.scalars(
581 select(Suite.suite_id).where(Suite.overridecodename == ocodename)
582 )
583 )
584 if suite.suite_id not in suiteids: 584 ↛ 587line 584 didn't jump to line 587 because the condition on line 584 was always true
585 suiteids.append(suite.suite_id)
587 if len(suiteids) < 1: 587 ↛ 588line 587 didn't jump to line 588 because the condition on line 587 was never true
588 utils.fubar("Couldn't find id's of all suites: %s" % suiteids)
590 for component_name in session.scalars(select(Component.component_name)):
591 # It is crucial for the dsc override creation based on binary
592 # overrides that 'dsc' goes first
593 otypes = ["dsc"]
594 for otype_name in session.scalars(select(OverrideType.overridetype)):
595 if otype_name == "dsc":
596 continue
597 otypes.append(otype_name)
599 for otype in otypes:
600 print(
601 "Processing %s [%s - %s]"
602 % (suite.suite_name, component_name, otype)
603 )
604 sys.stdout.flush()
605 process(
606 suite.suite_name,
607 suiteids,
608 originosuite_name,
609 component_name,
610 otype,
611 session,
612 )
614 Logger.close()