1#! /usr/bin/env python3
3""" Cruft checker and hole filler for overrides
5@contact: Debian FTPMaster <ftpmaster@debian.org>
6@copyright: 2000, 2001, 2002, 2004, 2006 James Troup <james@nocrew.org>
7@opyright: 2005 Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
8@copyright: 2011 Joerg Jaspert <joerg@debian.org>
9@license: GNU General Public License version 2 or later
11"""
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27################################################################################
29######################################################################
30# NB: dak check-overrides is not a good idea with New Incoming as it #
31# doesn't take into account accepted. You can minimize the impact #
32# of this by running it immediately after dak process-accepted but #
33# that's still racy because 'dak process-new' doesn't lock with 'dak #
34# process-accepted'. A better long term fix is the evil plan for #
35# accepted to be in the DB. #
36######################################################################
38# dak check-overrides should now work fine being done during
39# cron.daily, for example just before 'dak make-overrides' (after 'dak
40# process-accepted' and 'dak make-suite-file-list'). At that point,
41# queue/accepted should be empty and installed, so... dak
42# check-overrides does now take into account suites sharing overrides
44# TODO:
45# * Only update out-of-sync overrides when corresponding versions are equal to
46# some degree
47# * consistency checks like:
48# - section=debian-installer only for udeb and # dsc
49# - priority=optional if dsc
50# - (suite, package, 'dsc') is unique,
51# - just as (suite, package, (u)deb) (yes, across components!)
52# - sections match their component (each component has an own set of sections,
53# could probably be reduced...)
55################################################################################
57import sys
58import apt_pkg
60from daklib.config import Config
61from daklib.dbconn import *
62from daklib import daklog
63from daklib import utils
65################################################################################
67Options = None #: Commandline arguments parsed into this
68Logger = None #: Our logging object
69sections = {}
70priorities = {}
71blacklist = {}
73################################################################################
76def usage(exit_code=0):
77 print("""Usage: dak check-overrides
78Check for cruft in overrides.
80 -n, --no-action don't do anything
81 -h, --help show this help and exit""")
83 sys.exit(exit_code)
85################################################################################
88def process(osuite, affected_suites, originosuite, component, otype, session):
89 global Logger, Options, sections, priorities
91 o = get_suite(osuite, session)
92 if o is None: 92 ↛ 93line 92 didn't jump to line 93, because the condition on line 92 was never true
93 utils.fubar("Suite '%s' not recognised." % (osuite))
94 osuite_id = o.suite_id
96 originosuite_id = None
97 if originosuite:
98 oo = get_suite(originosuite, session)
99 if oo is None: 99 ↛ 100line 99 didn't jump to line 100, because the condition on line 99 was never true
100 utils.fubar("Suite '%s' not recognised." % (originosuite))
101 originosuite_id = oo.suite_id
103 c = get_component(component, session)
104 if c is None: 104 ↛ 105line 104 didn't jump to line 105, because the condition on line 104 was never true
105 utils.fubar("Component '%s' not recognised." % (component))
106 component_id = c.component_id
108 ot = get_override_type(otype, session)
109 if ot is None: 109 ↛ 110line 109 didn't jump to line 110, because the condition on line 109 was never true
110 utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype))
111 type_id = ot.overridetype_id
112 dsc_type_id = get_override_type("dsc", session).overridetype_id
114 source_priority_id = get_priority("optional", session).priority_id
116 if otype == "deb" or otype == "udeb":
117 packages = {}
118 # TODO: Fix to use placeholders (check how to with arrays)
119 q = session.execute("""
120SELECT b.package
121 FROM binaries b
122 JOIN bin_associations ba ON b.id = ba.bin
123 JOIN suite ON ba.suite = suite.id
124 JOIN files_archive_map af ON b.file = af.file_id AND suite.archive_id = af.archive_id
125 WHERE b.type = :otype AND ba.suite IN :affected_suites AND af.component_id = :component_id
126""", {'otype': otype, 'affected_suites': tuple(affected_suites), 'component_id': component_id})
127 for i in q.fetchall():
128 packages[i[0]] = 0
130 src_packages = {}
131 q = session.execute("""
132SELECT s.source FROM source s
133 JOIN src_associations sa ON s.id = sa.source
134 JOIN suite ON sa.suite = suite.id
135 JOIN files_archive_map af ON s.file = af.file_id AND suite.archive_id = af.archive_id
136 WHERE sa.suite IN :affected_suites AND af.component_id = :component_id
137""", {'affected_suites': tuple(affected_suites), 'component_id': component_id})
138 for i in q.fetchall():
139 src_packages[i[0]] = 0
141 # -----------
142 # Drop unused overrides
144 q = session.execute("""SELECT package, priority, section, maintainer
145 FROM override WHERE suite = :suite_id
146 AND component = :component_id AND type = :type_id""",
147 {'suite_id': osuite_id, 'component_id': component_id,
148 'type_id': type_id})
149 # We're already within a transaction
150 if otype == "dsc":
151 for i in q.fetchall():
152 package = i[0]
153 if package in src_packages: 153 ↛ 156line 153 didn't jump to line 156, because the condition on line 153 was never false
154 src_packages[package] = 1
155 else:
156 if package in blacklist:
157 utils.warn("%s in incoming, not touching" % package)
158 continue
159 Logger.log(["removing unused override", osuite, component,
160 otype, package, priorities[i[1]], sections[i[2]], i[3]])
161 if not Options["No-Action"]:
162 session.execute("""DELETE FROM override WHERE package = :package
163 AND suite = :suite_id AND component = :component_id
164 AND type = :type_id
165 AND created < now() - interval '14 days'""",
166 {'package': package, 'suite_id': osuite_id,
167 'component_id': component_id, 'type_id': type_id})
168 # create source overrides based on binary overrides, as source
169 # overrides not always get created
170 q = session.execute("""SELECT package, priority, section, maintainer
171 FROM override WHERE suite = :suite_id AND component = :component_id""",
172 {'suite_id': osuite_id, 'component_id': component_id})
173 for i in q.fetchall():
174 package = i[0]
175 if package not in src_packages or src_packages[package]: 175 ↛ 177line 175 didn't jump to line 177, because the condition on line 175 was never false
176 continue
177 src_packages[package] = 1
179 Logger.log(["add missing override", osuite, component,
180 otype, package, "source", sections[i[2]], i[3]])
181 if not Options["No-Action"]:
182 session.execute("""INSERT INTO override (package, suite, component,
183 priority, section, type, maintainer)
184 VALUES (:package, :suite_id, :component_id,
185 :priority_id, :section_id, :type_id, :maintainer)""",
186 {'package': package, 'suite_id': osuite_id,
187 'component_id': component_id, 'priority_id': source_priority_id,
188 'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]})
189 # Check whether originosuite has an override for us we can
190 # copy
191 if originosuite:
192 q = session.execute("""SELECT origin.package, origin.priority, origin.section,
193 origin.maintainer, target.priority, target.section,
194 target.maintainer
195 FROM override origin
196 LEFT JOIN override target ON (origin.package = target.package
197 AND target.suite = :suite_id
198 AND origin.component = target.component
199 AND origin.type = target.type)
200 WHERE origin.suite = :originsuite_id
201 AND origin.component = :component_id
202 AND origin.type = :type_id""",
203 {'suite_id': osuite_id, 'originsuite_id': originosuite_id,
204 'component_id': component_id, 'type_id': type_id})
205 for i in q.fetchall():
206 package = i[0]
207 if package not in src_packages or src_packages[package]:
208 if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]): 208 ↛ 209, 208 ↛ 2222 missed branches: 1) line 208 didn't jump to line 209, because the condition on line 208 was never true, 2) line 208 didn't jump to line 222, because the condition on line 208 was never false
209 Logger.log(["syncing override", osuite, component,
210 otype, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]])
211 if not Options["No-Action"]:
212 session.execute("""UPDATE override
213 SET priority = :priority,
214 section = :section,
215 maintainer = :maintainer
216 WHERE package = :package AND suite = :suite_id
217 AND component = :component_id AND type = :type_id""",
218 {'priority': i[1],
219 'section': i[2], 'maintainer': i[3],
220 'package': package, 'suite_id': osuite_id,
221 'component_id': component_id, 'type_id': dsc_type_id})
222 continue
224 # we can copy
225 src_packages[package] = 1
226 Logger.log(["copying missing override", osuite, component,
227 otype, package, "source", sections[i[2]], i[3]])
228 if not Options["No-Action"]: 228 ↛ 205line 228 didn't jump to line 205, because the condition on line 228 was never false
229 session.execute("""INSERT INTO override (package, suite, component,
230 priority, section, type, maintainer)
231 VALUES (:package, :suite_id, :component_id,
232 :priority_id, :section_id, :type_id,
233 :maintainer)""",
234 {'package': package, 'suite_id': osuite_id,
235 'component_id': component_id, 'priority_id': source_priority_id,
236 'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]})
238 for package, hasoverride in list(src_packages.items()):
239 if not hasoverride: 239 ↛ 240line 239 didn't jump to line 240, because the condition on line 239 was never true
240 utils.warn("%s has no override!" % package)
242 else: # binary override
243 for i in q.fetchall():
244 package = i[0]
245 if package in packages:
246 packages[package] = 1
247 else:
248 if package in blacklist: 248 ↛ 249line 248 didn't jump to line 249, because the condition on line 248 was never true
249 utils.warn("%s in incoming, not touching" % package)
250 continue
251 Logger.log(["removing unused override", osuite, component,
252 otype, package, priorities[i[1]], sections[i[2]], i[3]])
253 if not Options["No-Action"]: 253 ↛ 243line 253 didn't jump to line 243, because the condition on line 253 was never false
254 session.execute("""DELETE FROM override
255 WHERE package = :package AND suite = :suite_id
256 AND component = :component_id AND type = :type_id
257 AND created < now() - interval '14 days'""",
258 {'package': package, 'suite_id': osuite_id,
259 'component_id': component_id, 'type_id': type_id})
261 # Check whether originosuite has an override for us we can
262 # copy
263 if originosuite:
264 q = session.execute("""SELECT origin.package, origin.priority, origin.section,
265 origin.maintainer, target.priority, target.section,
266 target.maintainer
267 FROM override origin LEFT JOIN override target
268 ON (origin.package = target.package
269 AND target.suite = :suite_id
270 AND origin.component = target.component
271 AND origin.type = target.type)
272 WHERE origin.suite = :originsuite_id
273 AND origin.component = :component_id
274 AND origin.type = :type_id""",
275 {'suite_id': osuite_id, 'originsuite_id': originosuite_id,
276 'component_id': component_id, 'type_id': type_id})
277 for i in q.fetchall():
278 package = i[0]
279 if package not in packages or packages[package]:
280 if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]): 280 ↛ 281, 280 ↛ 2972 missed branches: 1) line 280 didn't jump to line 281, because the condition on line 280 was never true, 2) line 280 didn't jump to line 297, because the condition on line 280 was never false
281 Logger.log(["syncing override", osuite, component,
282 otype, package, priorities[i[4]], sections[i[5]],
283 i[6], priorities[i[1]], sections[i[2]], i[3]])
284 if not Options["No-Action"]:
285 session.execute("""UPDATE override
286 SET priority = :priority_id,
287 section = :section_id,
288 maintainer = :maintainer
289 WHERE package = :package
290 AND suite = :suite_id
291 AND component = :component_id
292 AND type = :type_id""",
293 {'priority_id': i[1], 'section_id': i[2],
294 'maintainer': i[3], 'package': package,
295 'suite_id': osuite_id, 'component_id': component_id,
296 'type_id': type_id})
297 continue
298 # we can copy
299 packages[package] = 1
300 Logger.log(["copying missing override", osuite, component,
301 otype, package, priorities[i[1]], sections[i[2]], i[3]])
302 if not Options["No-Action"]: 302 ↛ 277line 302 didn't jump to line 277, because the condition on line 302 was never false
303 session.execute("""INSERT INTO override (package, suite, component,
304 priority, section, type, maintainer)
305 VALUES (:package, :suite_id, :component_id,
306 :priority_id, :section_id, :type_id, :maintainer)""",
307 {'package': package, 'suite_id': osuite_id,
308 'component_id': component_id, 'priority_id': i[1],
309 'section_id': i[2], 'type_id': type_id, 'maintainer': i[3]})
311 for package, hasoverride in list(packages.items()):
312 if not hasoverride: 312 ↛ 313line 312 didn't jump to line 313, because the condition on line 312 was never true
313 utils.warn("%s has no override!" % package)
315 session.commit()
316 sys.stdout.flush()
319################################################################################
321def main():
322 global Logger, Options, sections, priorities
324 cnf = Config()
326 Arguments = [('h', "help", "Check-Overrides::Options::Help"),
327 ('n', "no-action", "Check-Overrides::Options::No-Action")]
328 for i in ["help", "no-action"]:
329 key = "Check-Overrides::Options::%s" % i
330 if key not in cnf: 330 ↛ 328line 330 didn't jump to line 328, because the condition on line 330 was never false
331 cnf[key] = ""
332 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
333 Options = cnf.subtree("Check-Overrides::Options")
335 if Options["Help"]:
336 usage()
338 session = DBConn().session()
340 # init sections, priorities:
342 # We need forward and reverse
343 sections = get_sections(session)
344 for name, entry in list(sections.items()):
345 sections[entry] = name
347 priorities = get_priorities(session)
348 for name, entry in list(priorities.items()):
349 priorities[entry] = name
351 if not Options["No-Action"]: 351 ↛ 354line 351 didn't jump to line 354, because the condition on line 351 was never false
352 Logger = daklog.Logger("check-overrides")
353 else:
354 Logger = daklog.Logger("check-overrides", 1)
356 for suite in session.query(Suite).filter(Suite.overrideprocess == True): # noqa:E712
357 originosuite = None
358 originremark = ''
360 if suite.overrideorigin is not None:
361 originosuite = get_suite(suite.overrideorigin, session)
362 if originosuite is None: 362 ↛ 363line 362 didn't jump to line 363, because the condition on line 362 was never true
363 utils.fubar("%s has an override origin suite of %s but it doesn't exist!" % (suite.suite_name, suite.overrideorigin))
364 originosuite = originosuite.suite_name
365 originremark = " taking missing from %s" % originosuite
367 print("Processing %s%s..." % (suite.suite_name, originremark))
369 # Get a list of all suites that use the override file of 'suite.suite_name' as
370 # well as the suite
371 ocodename = suite.codename
372 suiteids = [x.suite_id for x in session.query(Suite).filter(Suite.overridecodename == ocodename).all()]
373 if suite.suite_id not in suiteids: 373 ↛ 376line 373 didn't jump to line 376, because the condition on line 373 was never false
374 suiteids.append(suite.suite_id)
376 if len(suiteids) < 1: 376 ↛ 377line 376 didn't jump to line 377, because the condition on line 376 was never true
377 utils.fubar("Couldn't find id's of all suites: %s" % suiteids)
379 for component in session.query(Component).all():
380 # It is crucial for the dsc override creation based on binary
381 # overrides that 'dsc' goes first
382 component_name = component.component_name
383 otypes = ['dsc']
384 for ot in session.query(OverrideType):
385 if ot.overridetype == 'dsc':
386 continue
387 otypes.append(ot.overridetype)
389 for otype in otypes:
390 print("Processing %s [%s - %s]"
391 % (suite.suite_name, component_name, otype))
392 sys.stdout.flush()
393 process(suite.suite_name, suiteids, originosuite, component_name, otype, session)
395 Logger.close()
397################################################################################
400if __name__ == '__main__':
401 main()