| Trees | Indices | Help |
|
|---|
|
|
1 #! /usr/bin/env python3
2
3 """Cruft checker and hole filler for overrides
4
5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2004, 2006 James Troup <james@nocrew.org>
7 @opyright: 2005 Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
8 @copyright: 2011 Joerg Jaspert <joerg@debian.org>
9 @license: GNU General Public License version 2 or later
10
11 """
12
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
17
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
22
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26
27 ################################################################################
28
29 ######################################################################
30 # NB: dak check-overrides is not a good idea with New Incoming as it #
31 # doesn't take into account accepted. You can minimize the impact #
32 # of this by running it immediately after dak process-accepted but #
33 # that's still racy because 'dak process-new' doesn't lock with 'dak #
34 # process-accepted'. A better long term fix is the evil plan for #
35 # accepted to be in the DB. #
36 ######################################################################
37
38 # dak check-overrides should now work fine being done during
39 # cron.daily, for example just before 'dak make-overrides' (after 'dak
40 # process-accepted' and 'dak make-suite-file-list'). At that point,
41 # queue/accepted should be empty and installed, so... dak
42 # check-overrides does now take into account suites sharing overrides
43
44 # TODO:
45 # * Only update out-of-sync overrides when corresponding versions are equal to
46 # some degree
47 # * consistency checks like:
48 # - section=debian-installer only for udeb and # dsc
49 # - priority=optional if dsc
50 # - (suite, package, 'dsc') is unique,
51 # - just as (suite, package, (u)deb) (yes, across components!)
52 # - sections match their component (each component has an own set of sections,
53 # could probably be reduced...)
54
55 ################################################################################
56
57 import sys
58
59 import apt_pkg
60
61 from daklib import daklog, utils
62 from daklib.config import Config
63 from daklib.dbconn import (
64 Component,
65 DBConn,
66 OverrideType,
67 Suite,
68 get_component,
69 get_override_type,
70 get_priorities,
71 get_priority,
72 get_sections,
73 get_suite,
74 )
75
76 ################################################################################
77
78 Options = None #: Commandline arguments parsed into this
79 Logger = None #: Our logging object
80 sections = {}
81 priorities = {}
82 blacklist = {}
83
84 ################################################################################
85
86
88 print(
89 """Usage: dak check-overrides
90 Check for cruft in overrides.
91
92 -n, --no-action don't do anything
93 -h, --help show this help and exit"""
94 )
95
96 sys.exit(exit_code)
97
98
99 ################################################################################
100
101
103 global Logger, Options, sections, priorities
104
105 o = get_suite(osuite, session)
106 if o is None:
107 utils.fubar("Suite '%s' not recognised." % (osuite))
108 osuite_id = o.suite_id
109
110 originosuite_id = None
111 if originosuite:
112 oo = get_suite(originosuite, session)
113 if oo is None:
114 utils.fubar("Suite '%s' not recognised." % (originosuite))
115 originosuite_id = oo.suite_id
116
117 c = get_component(component, session)
118 if c is None:
119 utils.fubar("Component '%s' not recognised." % (component))
120 component_id = c.component_id
121
122 ot = get_override_type(otype, session)
123 if ot is None:
124 utils.fubar(
125 "Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype)
126 )
127 type_id = ot.overridetype_id
128 dsc_type_id = get_override_type("dsc", session).overridetype_id
129
130 source_priority_id = get_priority("optional", session).priority_id
131
132 if otype == "deb" or otype == "udeb":
133 packages = {}
134 # TODO: Fix to use placeholders (check how to with arrays)
135 q = session.execute(
136 """
137 SELECT b.package
138 FROM binaries b
139 JOIN bin_associations ba ON b.id = ba.bin
140 JOIN suite ON ba.suite = suite.id
141 JOIN files_archive_map af ON b.file = af.file_id AND suite.archive_id = af.archive_id
142 WHERE b.type = :otype AND ba.suite IN :affected_suites AND af.component_id = :component_id
143 """,
144 {
145 "otype": otype,
146 "affected_suites": tuple(affected_suites),
147 "component_id": component_id,
148 },
149 )
150 for i in q.fetchall():
151 packages[i[0]] = 0
152
153 src_packages = {}
154 q = session.execute(
155 """
156 SELECT s.source FROM source s
157 JOIN src_associations sa ON s.id = sa.source
158 JOIN suite ON sa.suite = suite.id
159 JOIN files_archive_map af ON s.file = af.file_id AND suite.archive_id = af.archive_id
160 WHERE sa.suite IN :affected_suites AND af.component_id = :component_id
161 """,
162 {"affected_suites": tuple(affected_suites), "component_id": component_id},
163 )
164 for i in q.fetchall():
165 src_packages[i[0]] = 0
166
167 # -----------
168 # Drop unused overrides
169
170 q = session.execute(
171 """SELECT package, priority, section, maintainer
172 FROM override WHERE suite = :suite_id
173 AND component = :component_id AND type = :type_id""",
174 {"suite_id": osuite_id, "component_id": component_id, "type_id": type_id},
175 )
176 # We're already within a transaction
177 if otype == "dsc":
178 for i in q.fetchall():
179 package = i[0]
180 if package in src_packages:
181 src_packages[package] = 1
182 else:
183 if package in blacklist:
184 utils.warn("%s in incoming, not touching" % package)
185 continue
186 Logger.log(
187 [
188 "removing unused override",
189 osuite,
190 component,
191 otype,
192 package,
193 priorities[i[1]],
194 sections[i[2]],
195 i[3],
196 ]
197 )
198 if not Options["No-Action"]:
199 session.execute(
200 """DELETE FROM override WHERE package = :package
201 AND suite = :suite_id AND component = :component_id
202 AND type = :type_id
203 AND created < now() - interval '14 days'""",
204 {
205 "package": package,
206 "suite_id": osuite_id,
207 "component_id": component_id,
208 "type_id": type_id,
209 },
210 )
211 # create source overrides based on binary overrides, as source
212 # overrides not always get created
213 q = session.execute(
214 """SELECT package, priority, section, maintainer
215 FROM override WHERE suite = :suite_id AND component = :component_id""",
216 {"suite_id": osuite_id, "component_id": component_id},
217 )
218 for i in q.fetchall():
219 package = i[0]
220 if package not in src_packages or src_packages[package]:
221 continue
222 src_packages[package] = 1
223
224 Logger.log(
225 [
226 "add missing override",
227 osuite,
228 component,
229 otype,
230 package,
231 "source",
232 sections[i[2]],
233 i[3],
234 ]
235 )
236 if not Options["No-Action"]:
237 session.execute(
238 """INSERT INTO override (package, suite, component,
239 priority, section, type, maintainer)
240 VALUES (:package, :suite_id, :component_id,
241 :priority_id, :section_id, :type_id, :maintainer)""",
242 {
243 "package": package,
244 "suite_id": osuite_id,
245 "component_id": component_id,
246 "priority_id": source_priority_id,
247 "section_id": i[2],
248 "type_id": dsc_type_id,
249 "maintainer": i[3],
250 },
251 )
252 # Check whether originosuite has an override for us we can
253 # copy
254 if originosuite:
255 q = session.execute(
256 """SELECT origin.package, origin.priority, origin.section,
257 origin.maintainer, target.priority, target.section,
258 target.maintainer
259 FROM override origin
260 LEFT JOIN override target ON (origin.package = target.package
261 AND target.suite = :suite_id
262 AND origin.component = target.component
263 AND origin.type = target.type)
264 WHERE origin.suite = :originsuite_id
265 AND origin.component = :component_id
266 AND origin.type = :type_id""",
267 {
268 "suite_id": osuite_id,
269 "originsuite_id": originosuite_id,
270 "component_id": component_id,
271 "type_id": type_id,
272 },
273 )
274 for i in q.fetchall():
275 package = i[0]
276 if package not in src_packages or src_packages[package]:
277 if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
278 Logger.log(
279 [
280 "syncing override",
281 osuite,
282 component,
283 otype,
284 package,
285 "source",
286 sections[i[5]],
287 i[6],
288 "source",
289 sections[i[2]],
290 i[3],
291 ]
292 )
293 if not Options["No-Action"]:
294 session.execute(
295 """UPDATE override
296 SET priority = :priority,
297 section = :section,
298 maintainer = :maintainer
299 WHERE package = :package AND suite = :suite_id
300 AND component = :component_id AND type = :type_id""",
301 {
302 "priority": i[1],
303 "section": i[2],
304 "maintainer": i[3],
305 "package": package,
306 "suite_id": osuite_id,
307 "component_id": component_id,
308 "type_id": dsc_type_id,
309 },
310 )
311 continue
312
313 # we can copy
314 src_packages[package] = 1
315 Logger.log(
316 [
317 "copying missing override",
318 osuite,
319 component,
320 otype,
321 package,
322 "source",
323 sections[i[2]],
324 i[3],
325 ]
326 )
327 if not Options["No-Action"]:
328 session.execute(
329 """INSERT INTO override (package, suite, component,
330 priority, section, type, maintainer)
331 VALUES (:package, :suite_id, :component_id,
332 :priority_id, :section_id, :type_id,
333 :maintainer)""",
334 {
335 "package": package,
336 "suite_id": osuite_id,
337 "component_id": component_id,
338 "priority_id": source_priority_id,
339 "section_id": i[2],
340 "type_id": dsc_type_id,
341 "maintainer": i[3],
342 },
343 )
344
345 for package, hasoverride in list(src_packages.items()):
346 if not hasoverride:
347 utils.warn("%s has no override!" % package)
348
349 else: # binary override
350 for i in q.fetchall():
351 package = i[0]
352 if package in packages:
353 packages[package] = 1
354 else:
355 if package in blacklist:
356 utils.warn("%s in incoming, not touching" % package)
357 continue
358 Logger.log(
359 [
360 "removing unused override",
361 osuite,
362 component,
363 otype,
364 package,
365 priorities[i[1]],
366 sections[i[2]],
367 i[3],
368 ]
369 )
370 if not Options["No-Action"]:
371 session.execute(
372 """DELETE FROM override
373 WHERE package = :package AND suite = :suite_id
374 AND component = :component_id AND type = :type_id
375 AND created < now() - interval '14 days'""",
376 {
377 "package": package,
378 "suite_id": osuite_id,
379 "component_id": component_id,
380 "type_id": type_id,
381 },
382 )
383
384 # Check whether originosuite has an override for us we can
385 # copy
386 if originosuite:
387 q = session.execute(
388 """SELECT origin.package, origin.priority, origin.section,
389 origin.maintainer, target.priority, target.section,
390 target.maintainer
391 FROM override origin LEFT JOIN override target
392 ON (origin.package = target.package
393 AND target.suite = :suite_id
394 AND origin.component = target.component
395 AND origin.type = target.type)
396 WHERE origin.suite = :originsuite_id
397 AND origin.component = :component_id
398 AND origin.type = :type_id""",
399 {
400 "suite_id": osuite_id,
401 "originsuite_id": originosuite_id,
402 "component_id": component_id,
403 "type_id": type_id,
404 },
405 )
406 for i in q.fetchall():
407 package = i[0]
408 if package not in packages or packages[package]:
409 if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
410 Logger.log(
411 [
412 "syncing override",
413 osuite,
414 component,
415 otype,
416 package,
417 priorities[i[4]],
418 sections[i[5]],
419 i[6],
420 priorities[i[1]],
421 sections[i[2]],
422 i[3],
423 ]
424 )
425 if not Options["No-Action"]:
426 session.execute(
427 """UPDATE override
428 SET priority = :priority_id,
429 section = :section_id,
430 maintainer = :maintainer
431 WHERE package = :package
432 AND suite = :suite_id
433 AND component = :component_id
434 AND type = :type_id""",
435 {
436 "priority_id": i[1],
437 "section_id": i[2],
438 "maintainer": i[3],
439 "package": package,
440 "suite_id": osuite_id,
441 "component_id": component_id,
442 "type_id": type_id,
443 },
444 )
445 continue
446 # we can copy
447 packages[package] = 1
448 Logger.log(
449 [
450 "copying missing override",
451 osuite,
452 component,
453 otype,
454 package,
455 priorities[i[1]],
456 sections[i[2]],
457 i[3],
458 ]
459 )
460 if not Options["No-Action"]:
461 session.execute(
462 """INSERT INTO override (package, suite, component,
463 priority, section, type, maintainer)
464 VALUES (:package, :suite_id, :component_id,
465 :priority_id, :section_id, :type_id, :maintainer)""",
466 {
467 "package": package,
468 "suite_id": osuite_id,
469 "component_id": component_id,
470 "priority_id": i[1],
471 "section_id": i[2],
472 "type_id": type_id,
473 "maintainer": i[3],
474 },
475 )
476
477 for package, hasoverride in list(packages.items()):
478 if not hasoverride:
479 utils.warn("%s has no override!" % package)
480
481 session.commit()
482 sys.stdout.flush()
483
484
485 ################################################################################
486
487
489 global Logger, Options, sections, priorities
490
491 cnf = Config()
492
493 Arguments = [
494 ("h", "help", "Check-Overrides::Options::Help"),
495 ("n", "no-action", "Check-Overrides::Options::No-Action"),
496 ]
497 for i in ["help", "no-action"]:
498 key = "Check-Overrides::Options::%s" % i
499 if key not in cnf:
500 cnf[key] = ""
501 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
502 Options = cnf.subtree("Check-Overrides::Options")
503
504 if Options["Help"]:
505 usage()
506
507 session = DBConn().session()
508
509 # init sections, priorities:
510
511 # We need forward and reverse
512 sections = get_sections(session)
513 for name, entry in list(sections.items()):
514 sections[entry] = name
515
516 priorities = get_priorities(session)
517 for name, entry in list(priorities.items()):
518 priorities[entry] = name
519
520 if not Options["No-Action"]:
521 Logger = daklog.Logger("check-overrides")
522 else:
523 Logger = daklog.Logger("check-overrides", 1)
524
525 for suite in session.query(Suite).filter(
526 Suite.overrideprocess == True # noqa:E712
527 ):
528 originosuite = None
529 originremark = ""
530
531 if suite.overrideorigin is not None:
532 originosuite = get_suite(suite.overrideorigin, session)
533 if originosuite is None:
534 utils.fubar(
535 "%s has an override origin suite of %s but it doesn't exist!"
536 % (suite.suite_name, suite.overrideorigin)
537 )
538 originosuite = originosuite.suite_name
539 originremark = " taking missing from %s" % originosuite
540
541 print("Processing %s%s..." % (suite.suite_name, originremark))
542
543 # Get a list of all suites that use the override file of 'suite.suite_name' as
544 # well as the suite
545 ocodename = suite.codename
546 suiteids = [
547 x.suite_id
548 for x in session.query(Suite)
549 .filter(Suite.overridecodename == ocodename)
550 .all()
551 ]
552 if suite.suite_id not in suiteids:
553 suiteids.append(suite.suite_id)
554
555 if len(suiteids) < 1:
556 utils.fubar("Couldn't find id's of all suites: %s" % suiteids)
557
558 for component in session.query(Component).all():
559 # It is crucial for the dsc override creation based on binary
560 # overrides that 'dsc' goes first
561 component_name = component.component_name
562 otypes = ["dsc"]
563 for ot in session.query(OverrideType):
564 if ot.overridetype == "dsc":
565 continue
566 otypes.append(ot.overridetype)
567
568 for otype in otypes:
569 print(
570 "Processing %s [%s - %s]"
571 % (suite.suite_name, component_name, otype)
572 )
573 sys.stdout.flush()
574 process(
575 suite.suite_name,
576 suiteids,
577 originosuite,
578 component_name,
579 otype,
580 session,
581 )
582
583 Logger.close()
584
585
586 ################################################################################
587
588
589 if __name__ == "__main__":
590 main()
591
| Trees | Indices | Help |
|
|---|
| Generated by Epydoc 3.0.1 on Sun Mar 30 12:42:53 2025 | http://epydoc.sourceforge.net |