1#! /usr/bin/env python3
3"""Cruft checker and hole filler for overrides
5@contact: Debian FTPMaster <ftpmaster@debian.org>
6@copyright: 2000, 2001, 2002, 2004, 2006 James Troup <james@nocrew.org>
7@opyright: 2005 Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
8@copyright: 2011 Joerg Jaspert <joerg@debian.org>
9@license: GNU General Public License version 2 or later
11"""
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27################################################################################
29######################################################################
30# NB: dak check-overrides is not a good idea with New Incoming as it #
31# doesn't take into account accepted. You can minimize the impact #
32# of this by running it immediately after dak process-accepted but #
33# that's still racy because 'dak process-new' doesn't lock with 'dak #
34# process-accepted'. A better long term fix is the evil plan for #
35# accepted to be in the DB. #
36######################################################################
38# dak check-overrides should now work fine being done during
39# cron.daily, for example just before 'dak make-overrides' (after 'dak
40# process-accepted' and 'dak make-suite-file-list'). At that point,
41# queue/accepted should be empty and installed, so... dak
42# check-overrides does now take into account suites sharing overrides
44# TODO:
45# * Only update out-of-sync overrides when corresponding versions are equal to
46# some degree
47# * consistency checks like:
48# - section=debian-installer only for udeb and # dsc
49# - priority=optional if dsc
50# - (suite, package, 'dsc') is unique,
51# - just as (suite, package, (u)deb) (yes, across components!)
52# - sections match their component (each component has an own set of sections,
53# could probably be reduced...)
55################################################################################
57import sys
59import apt_pkg
61from daklib import daklog, utils
62from daklib.config import Config
63from daklib.dbconn import (
64 Component,
65 DBConn,
66 OverrideType,
67 Suite,
68 get_component,
69 get_override_type,
70 get_priorities,
71 get_priority,
72 get_sections,
73 get_suite,
74)
76################################################################################
78Options = None #: Commandline arguments parsed into this
79Logger = None #: Our logging object
80sections = {}
81priorities = {}
82blacklist = {}
84################################################################################
87def usage(exit_code=0):
88 print(
89 """Usage: dak check-overrides
90Check for cruft in overrides.
92 -n, --no-action don't do anything
93 -h, --help show this help and exit"""
94 )
96 sys.exit(exit_code)
99################################################################################
102def process(osuite, affected_suites, originosuite, component, otype, session):
103 global Logger, Options, sections, priorities
105 o = get_suite(osuite, session)
106 if o is None: 106 ↛ 107line 106 didn't jump to line 107, because the condition on line 106 was never true
107 utils.fubar("Suite '%s' not recognised." % (osuite))
108 osuite_id = o.suite_id
110 originosuite_id = None
111 if originosuite:
112 oo = get_suite(originosuite, session)
113 if oo is None: 113 ↛ 114line 113 didn't jump to line 114, because the condition on line 113 was never true
114 utils.fubar("Suite '%s' not recognised." % (originosuite))
115 originosuite_id = oo.suite_id
117 c = get_component(component, session)
118 if c is None: 118 ↛ 119line 118 didn't jump to line 119, because the condition on line 118 was never true
119 utils.fubar("Component '%s' not recognised." % (component))
120 component_id = c.component_id
122 ot = get_override_type(otype, session)
123 if ot is None: 123 ↛ 124line 123 didn't jump to line 124, because the condition on line 123 was never true
124 utils.fubar(
125 "Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype)
126 )
127 type_id = ot.overridetype_id
128 dsc_type_id = get_override_type("dsc", session).overridetype_id
130 source_priority_id = get_priority("optional", session).priority_id
132 if otype == "deb" or otype == "udeb":
133 packages = {}
134 # TODO: Fix to use placeholders (check how to with arrays)
135 q = session.execute(
136 """
137SELECT b.package
138 FROM binaries b
139 JOIN bin_associations ba ON b.id = ba.bin
140 JOIN suite ON ba.suite = suite.id
141 JOIN files_archive_map af ON b.file = af.file_id AND suite.archive_id = af.archive_id
142 WHERE b.type = :otype AND ba.suite IN :affected_suites AND af.component_id = :component_id
143""",
144 {
145 "otype": otype,
146 "affected_suites": tuple(affected_suites),
147 "component_id": component_id,
148 },
149 )
150 for i in q.fetchall():
151 packages[i[0]] = 0
153 src_packages = {}
154 q = session.execute(
155 """
156SELECT s.source FROM source s
157 JOIN src_associations sa ON s.id = sa.source
158 JOIN suite ON sa.suite = suite.id
159 JOIN files_archive_map af ON s.file = af.file_id AND suite.archive_id = af.archive_id
160 WHERE sa.suite IN :affected_suites AND af.component_id = :component_id
161""",
162 {"affected_suites": tuple(affected_suites), "component_id": component_id},
163 )
164 for i in q.fetchall():
165 src_packages[i[0]] = 0
167 # -----------
168 # Drop unused overrides
170 q = session.execute(
171 """SELECT package, priority, section, maintainer
172 FROM override WHERE suite = :suite_id
173 AND component = :component_id AND type = :type_id""",
174 {"suite_id": osuite_id, "component_id": component_id, "type_id": type_id},
175 )
176 # We're already within a transaction
177 if otype == "dsc":
178 for i in q.fetchall():
179 package = i[0]
180 if package in src_packages: 180 ↛ 183line 180 didn't jump to line 183, because the condition on line 180 was never false
181 src_packages[package] = 1
182 else:
183 if package in blacklist:
184 utils.warn("%s in incoming, not touching" % package)
185 continue
186 Logger.log(
187 [
188 "removing unused override",
189 osuite,
190 component,
191 otype,
192 package,
193 priorities[i[1]],
194 sections[i[2]],
195 i[3],
196 ]
197 )
198 if not Options["No-Action"]:
199 session.execute(
200 """DELETE FROM override WHERE package = :package
201 AND suite = :suite_id AND component = :component_id
202 AND type = :type_id
203 AND created < now() - interval '14 days'""",
204 {
205 "package": package,
206 "suite_id": osuite_id,
207 "component_id": component_id,
208 "type_id": type_id,
209 },
210 )
211 # create source overrides based on binary overrides, as source
212 # overrides not always get created
213 q = session.execute(
214 """SELECT package, priority, section, maintainer
215 FROM override WHERE suite = :suite_id AND component = :component_id""",
216 {"suite_id": osuite_id, "component_id": component_id},
217 )
218 for i in q.fetchall():
219 package = i[0]
220 if package not in src_packages or src_packages[package]: 220 ↛ 222line 220 didn't jump to line 222, because the condition on line 220 was never false
221 continue
222 src_packages[package] = 1
224 Logger.log(
225 [
226 "add missing override",
227 osuite,
228 component,
229 otype,
230 package,
231 "source",
232 sections[i[2]],
233 i[3],
234 ]
235 )
236 if not Options["No-Action"]:
237 session.execute(
238 """INSERT INTO override (package, suite, component,
239 priority, section, type, maintainer)
240 VALUES (:package, :suite_id, :component_id,
241 :priority_id, :section_id, :type_id, :maintainer)""",
242 {
243 "package": package,
244 "suite_id": osuite_id,
245 "component_id": component_id,
246 "priority_id": source_priority_id,
247 "section_id": i[2],
248 "type_id": dsc_type_id,
249 "maintainer": i[3],
250 },
251 )
252 # Check whether originosuite has an override for us we can
253 # copy
254 if originosuite:
255 q = session.execute(
256 """SELECT origin.package, origin.priority, origin.section,
257 origin.maintainer, target.priority, target.section,
258 target.maintainer
259 FROM override origin
260 LEFT JOIN override target ON (origin.package = target.package
261 AND target.suite = :suite_id
262 AND origin.component = target.component
263 AND origin.type = target.type)
264 WHERE origin.suite = :originsuite_id
265 AND origin.component = :component_id
266 AND origin.type = :type_id""",
267 {
268 "suite_id": osuite_id,
269 "originsuite_id": originosuite_id,
270 "component_id": component_id,
271 "type_id": type_id,
272 },
273 )
274 for i in q.fetchall():
275 package = i[0]
276 if package not in src_packages or src_packages[package]:
277 if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]): 277 ↛ 278, 277 ↛ 3112 missed branches: 1) line 277 didn't jump to line 278, because the condition on line 277 was never true, 2) line 277 didn't jump to line 311, because the condition on line 277 was never false
278 Logger.log(
279 [
280 "syncing override",
281 osuite,
282 component,
283 otype,
284 package,
285 "source",
286 sections[i[5]],
287 i[6],
288 "source",
289 sections[i[2]],
290 i[3],
291 ]
292 )
293 if not Options["No-Action"]:
294 session.execute(
295 """UPDATE override
296 SET priority = :priority,
297 section = :section,
298 maintainer = :maintainer
299 WHERE package = :package AND suite = :suite_id
300 AND component = :component_id AND type = :type_id""",
301 {
302 "priority": i[1],
303 "section": i[2],
304 "maintainer": i[3],
305 "package": package,
306 "suite_id": osuite_id,
307 "component_id": component_id,
308 "type_id": dsc_type_id,
309 },
310 )
311 continue
313 # we can copy
314 src_packages[package] = 1
315 Logger.log(
316 [
317 "copying missing override",
318 osuite,
319 component,
320 otype,
321 package,
322 "source",
323 sections[i[2]],
324 i[3],
325 ]
326 )
327 if not Options["No-Action"]: 327 ↛ 274line 327 didn't jump to line 274, because the condition on line 327 was never false
328 session.execute(
329 """INSERT INTO override (package, suite, component,
330 priority, section, type, maintainer)
331 VALUES (:package, :suite_id, :component_id,
332 :priority_id, :section_id, :type_id,
333 :maintainer)""",
334 {
335 "package": package,
336 "suite_id": osuite_id,
337 "component_id": component_id,
338 "priority_id": source_priority_id,
339 "section_id": i[2],
340 "type_id": dsc_type_id,
341 "maintainer": i[3],
342 },
343 )
345 for package, hasoverride in list(src_packages.items()):
346 if not hasoverride: 346 ↛ 347line 346 didn't jump to line 347, because the condition on line 346 was never true
347 utils.warn("%s has no override!" % package)
349 else: # binary override
350 for i in q.fetchall():
351 package = i[0]
352 if package in packages:
353 packages[package] = 1
354 else:
355 if package in blacklist: 355 ↛ 356line 355 didn't jump to line 356, because the condition on line 355 was never true
356 utils.warn("%s in incoming, not touching" % package)
357 continue
358 Logger.log(
359 [
360 "removing unused override",
361 osuite,
362 component,
363 otype,
364 package,
365 priorities[i[1]],
366 sections[i[2]],
367 i[3],
368 ]
369 )
370 if not Options["No-Action"]: 370 ↛ 350line 370 didn't jump to line 350, because the condition on line 370 was never false
371 session.execute(
372 """DELETE FROM override
373 WHERE package = :package AND suite = :suite_id
374 AND component = :component_id AND type = :type_id
375 AND created < now() - interval '14 days'""",
376 {
377 "package": package,
378 "suite_id": osuite_id,
379 "component_id": component_id,
380 "type_id": type_id,
381 },
382 )
384 # Check whether originosuite has an override for us we can
385 # copy
386 if originosuite:
387 q = session.execute(
388 """SELECT origin.package, origin.priority, origin.section,
389 origin.maintainer, target.priority, target.section,
390 target.maintainer
391 FROM override origin LEFT JOIN override target
392 ON (origin.package = target.package
393 AND target.suite = :suite_id
394 AND origin.component = target.component
395 AND origin.type = target.type)
396 WHERE origin.suite = :originsuite_id
397 AND origin.component = :component_id
398 AND origin.type = :type_id""",
399 {
400 "suite_id": osuite_id,
401 "originsuite_id": originosuite_id,
402 "component_id": component_id,
403 "type_id": type_id,
404 },
405 )
406 for i in q.fetchall():
407 package = i[0]
408 if package not in packages or packages[package]:
409 if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]): 409 ↛ 410, 409 ↛ 4452 missed branches: 1) line 409 didn't jump to line 410, because the condition on line 409 was never true, 2) line 409 didn't jump to line 445, because the condition on line 409 was never false
410 Logger.log(
411 [
412 "syncing override",
413 osuite,
414 component,
415 otype,
416 package,
417 priorities[i[4]],
418 sections[i[5]],
419 i[6],
420 priorities[i[1]],
421 sections[i[2]],
422 i[3],
423 ]
424 )
425 if not Options["No-Action"]:
426 session.execute(
427 """UPDATE override
428 SET priority = :priority_id,
429 section = :section_id,
430 maintainer = :maintainer
431 WHERE package = :package
432 AND suite = :suite_id
433 AND component = :component_id
434 AND type = :type_id""",
435 {
436 "priority_id": i[1],
437 "section_id": i[2],
438 "maintainer": i[3],
439 "package": package,
440 "suite_id": osuite_id,
441 "component_id": component_id,
442 "type_id": type_id,
443 },
444 )
445 continue
446 # we can copy
447 packages[package] = 1
448 Logger.log(
449 [
450 "copying missing override",
451 osuite,
452 component,
453 otype,
454 package,
455 priorities[i[1]],
456 sections[i[2]],
457 i[3],
458 ]
459 )
460 if not Options["No-Action"]: 460 ↛ 406line 460 didn't jump to line 406, because the condition on line 460 was never false
461 session.execute(
462 """INSERT INTO override (package, suite, component,
463 priority, section, type, maintainer)
464 VALUES (:package, :suite_id, :component_id,
465 :priority_id, :section_id, :type_id, :maintainer)""",
466 {
467 "package": package,
468 "suite_id": osuite_id,
469 "component_id": component_id,
470 "priority_id": i[1],
471 "section_id": i[2],
472 "type_id": type_id,
473 "maintainer": i[3],
474 },
475 )
477 for package, hasoverride in list(packages.items()):
478 if not hasoverride: 478 ↛ 479line 478 didn't jump to line 479, because the condition on line 478 was never true
479 utils.warn("%s has no override!" % package)
481 session.commit()
482 sys.stdout.flush()
485################################################################################
488def main():
489 global Logger, Options, sections, priorities
491 cnf = Config()
493 Arguments = [
494 ("h", "help", "Check-Overrides::Options::Help"),
495 ("n", "no-action", "Check-Overrides::Options::No-Action"),
496 ]
497 for i in ["help", "no-action"]:
498 key = "Check-Overrides::Options::%s" % i
499 if key not in cnf: 499 ↛ 497line 499 didn't jump to line 497, because the condition on line 499 was never false
500 cnf[key] = ""
501 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
502 Options = cnf.subtree("Check-Overrides::Options")
504 if Options["Help"]:
505 usage()
507 session = DBConn().session()
509 # init sections, priorities:
511 # We need forward and reverse
512 sections = get_sections(session)
513 for name, entry in list(sections.items()):
514 sections[entry] = name
516 priorities = get_priorities(session)
517 for name, entry in list(priorities.items()):
518 priorities[entry] = name
520 if not Options["No-Action"]: 520 ↛ 523line 520 didn't jump to line 523, because the condition on line 520 was never false
521 Logger = daklog.Logger("check-overrides")
522 else:
523 Logger = daklog.Logger("check-overrides", 1)
525 for suite in session.query(Suite).filter(
526 Suite.overrideprocess == True # noqa:E712
527 ):
528 originosuite = None
529 originremark = ""
531 if suite.overrideorigin is not None:
532 originosuite = get_suite(suite.overrideorigin, session)
533 if originosuite is None: 533 ↛ 534line 533 didn't jump to line 534, because the condition on line 533 was never true
534 utils.fubar(
535 "%s has an override origin suite of %s but it doesn't exist!"
536 % (suite.suite_name, suite.overrideorigin)
537 )
538 originosuite = originosuite.suite_name
539 originremark = " taking missing from %s" % originosuite
541 print("Processing %s%s..." % (suite.suite_name, originremark))
543 # Get a list of all suites that use the override file of 'suite.suite_name' as
544 # well as the suite
545 ocodename = suite.codename
546 suiteids = [
547 x.suite_id
548 for x in session.query(Suite)
549 .filter(Suite.overridecodename == ocodename)
550 .all()
551 ]
552 if suite.suite_id not in suiteids: 552 ↛ 555line 552 didn't jump to line 555, because the condition on line 552 was never false
553 suiteids.append(suite.suite_id)
555 if len(suiteids) < 1: 555 ↛ 556line 555 didn't jump to line 556, because the condition on line 555 was never true
556 utils.fubar("Couldn't find id's of all suites: %s" % suiteids)
558 for component in session.query(Component).all():
559 # It is crucial for the dsc override creation based on binary
560 # overrides that 'dsc' goes first
561 component_name = component.component_name
562 otypes = ["dsc"]
563 for ot in session.query(OverrideType):
564 if ot.overridetype == "dsc":
565 continue
566 otypes.append(ot.overridetype)
568 for otype in otypes:
569 print(
570 "Processing %s [%s - %s]"
571 % (suite.suite_name, component_name, otype)
572 )
573 sys.stdout.flush()
574 process(
575 suite.suite_name,
576 suiteids,
577 originosuite,
578 component_name,
579 otype,
580 session,
581 )
583 Logger.close()
586################################################################################
589if __name__ == "__main__":
590 main()