Package dak :: Module check_overrides
[hide private]
[frames] | no frames]

Source Code for Module dak.check_overrides

  1  #! /usr/bin/env python3 
  2   
  3  """ Cruft checker and hole filler for overrides 
  4   
  5  @contact: Debian FTPMaster <ftpmaster@debian.org> 
  6  @copyright: 2000, 2001, 2002, 2004, 2006  James Troup <james@nocrew.org> 
  7  @opyright: 2005  Jeroen van Wolffelaar <jeroen@wolffelaar.nl> 
  8  @copyright: 2011  Joerg Jaspert <joerg@debian.org> 
  9  @license: GNU General Public License version 2 or later 
 10   
 11  """ 
 12   
 13  # This program is free software; you can redistribute it and/or modify 
 14  # it under the terms of the GNU General Public License as published by 
 15  # the Free Software Foundation; either version 2 of the License, or 
 16  # (at your option) any later version. 
 17   
 18  # This program is distributed in the hope that it will be useful, 
 19  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 20  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 21  # GNU General Public License for more details. 
 22   
 23  # You should have received a copy of the GNU General Public License 
 24  # along with this program; if not, write to the Free Software 
 25  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
 26   
 27  ################################################################################ 
 28   
 29  ###################################################################### 
 30  # NB: dak check-overrides is not a good idea with New Incoming as it # 
 31  # doesn't take into account accepted.  You can minimize the impact   # 
 32  # of this by running it immediately after dak process-accepted but   # 
 33  # that's still racy because 'dak process-new' doesn't lock with 'dak # 
 34  # process-accepted'.  A better long term fix is the evil plan for    # 
 35  # accepted to be in the DB.                                          # 
 36  ###################################################################### 
 37   
 38  # dak check-overrides should now work fine being done during 
 39  # cron.daily, for example just before 'dak make-overrides' (after 'dak 
 40  # process-accepted' and 'dak make-suite-file-list'). At that point, 
 41  # queue/accepted should be empty and installed, so... dak 
 42  # check-overrides does now take into account suites sharing overrides 
 43   
 44  # TODO: 
 45  # * Only update out-of-sync overrides when corresponding versions are equal to 
 46  #   some degree 
 47  # * consistency checks like: 
 48  #   - section=debian-installer only for udeb and # dsc 
 49  #   - priority=optional if dsc 
 50  #   - (suite, package, 'dsc') is unique, 
 51  #   - just as (suite, package, (u)deb) (yes, across components!) 
 52  #   - sections match their component (each component has an own set of sections, 
 53  #     could probably be reduced...) 
 54   
 55  ################################################################################ 
 56   
 57  import sys 
 58  import apt_pkg 
 59   
 60  from daklib.config import Config 
 61  from daklib.dbconn import * 
 62  from daklib import daklog 
 63  from daklib import utils 
 64   
 65  ################################################################################ 
 66   
 67  Options = None                 #: Commandline arguments parsed into this 
 68  Logger = None                  #: Our logging object 
 69  sections = {} 
 70  priorities = {} 
 71  blacklist = {} 
 72   
 73  ################################################################################ 
 74   
 75   
76 -def usage(exit_code=0):
77 print("""Usage: dak check-overrides 78 Check for cruft in overrides. 79 80 -n, --no-action don't do anything 81 -h, --help show this help and exit""") 82 83 sys.exit(exit_code)
84 85 ################################################################################ 86 87
88 -def process(osuite, affected_suites, originosuite, component, otype, session):
89 global Logger, Options, sections, priorities 90 91 o = get_suite(osuite, session) 92 if o is None: 93 utils.fubar("Suite '%s' not recognised." % (osuite)) 94 osuite_id = o.suite_id 95 96 originosuite_id = None 97 if originosuite: 98 oo = get_suite(originosuite, session) 99 if oo is None: 100 utils.fubar("Suite '%s' not recognised." % (originosuite)) 101 originosuite_id = oo.suite_id 102 103 c = get_component(component, session) 104 if c is None: 105 utils.fubar("Component '%s' not recognised." % (component)) 106 component_id = c.component_id 107 108 ot = get_override_type(otype, session) 109 if ot is None: 110 utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype)) 111 type_id = ot.overridetype_id 112 dsc_type_id = get_override_type("dsc", session).overridetype_id 113 114 source_priority_id = get_priority("optional", session).priority_id 115 116 if otype == "deb" or otype == "udeb": 117 packages = {} 118 # TODO: Fix to use placeholders (check how to with arrays) 119 q = session.execute(""" 120 SELECT b.package 121 FROM binaries b 122 JOIN bin_associations ba ON b.id = ba.bin 123 JOIN suite ON ba.suite = suite.id 124 JOIN files_archive_map af ON b.file = af.file_id AND suite.archive_id = af.archive_id 125 WHERE b.type = :otype AND ba.suite IN :affected_suites AND af.component_id = :component_id 126 """, {'otype': otype, 'affected_suites': tuple(affected_suites), 'component_id': component_id}) 127 for i in q.fetchall(): 128 packages[i[0]] = 0 129 130 src_packages = {} 131 q = session.execute(""" 132 SELECT s.source FROM source s 133 JOIN src_associations sa ON s.id = sa.source 134 JOIN suite ON sa.suite = suite.id 135 JOIN files_archive_map af ON s.file = af.file_id AND suite.archive_id = af.archive_id 136 WHERE sa.suite IN :affected_suites AND af.component_id = :component_id 137 """, {'affected_suites': tuple(affected_suites), 'component_id': component_id}) 138 for i in q.fetchall(): 139 src_packages[i[0]] = 0 140 141 # ----------- 142 # Drop unused overrides 143 144 q = session.execute("""SELECT package, priority, section, maintainer 145 FROM override WHERE suite = :suite_id 146 AND component = :component_id AND type = :type_id""", 147 {'suite_id': osuite_id, 'component_id': component_id, 148 'type_id': type_id}) 149 # We're already within a transaction 150 if otype == "dsc": 151 for i in q.fetchall(): 152 package = i[0] 153 if package in src_packages: 154 src_packages[package] = 1 155 else: 156 if package in blacklist: 157 utils.warn("%s in incoming, not touching" % package) 158 continue 159 Logger.log(["removing unused override", osuite, component, 160 otype, package, priorities[i[1]], sections[i[2]], i[3]]) 161 if not Options["No-Action"]: 162 session.execute("""DELETE FROM override WHERE package = :package 163 AND suite = :suite_id AND component = :component_id 164 AND type = :type_id 165 AND created < now() - interval '14 days'""", 166 {'package': package, 'suite_id': osuite_id, 167 'component_id': component_id, 'type_id': type_id}) 168 # create source overrides based on binary overrides, as source 169 # overrides not always get created 170 q = session.execute("""SELECT package, priority, section, maintainer 171 FROM override WHERE suite = :suite_id AND component = :component_id""", 172 {'suite_id': osuite_id, 'component_id': component_id}) 173 for i in q.fetchall(): 174 package = i[0] 175 if package not in src_packages or src_packages[package]: 176 continue 177 src_packages[package] = 1 178 179 Logger.log(["add missing override", osuite, component, 180 otype, package, "source", sections[i[2]], i[3]]) 181 if not Options["No-Action"]: 182 session.execute("""INSERT INTO override (package, suite, component, 183 priority, section, type, maintainer) 184 VALUES (:package, :suite_id, :component_id, 185 :priority_id, :section_id, :type_id, :maintainer)""", 186 {'package': package, 'suite_id': osuite_id, 187 'component_id': component_id, 'priority_id': source_priority_id, 188 'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]}) 189 # Check whether originosuite has an override for us we can 190 # copy 191 if originosuite: 192 q = session.execute("""SELECT origin.package, origin.priority, origin.section, 193 origin.maintainer, target.priority, target.section, 194 target.maintainer 195 FROM override origin 196 LEFT JOIN override target ON (origin.package = target.package 197 AND target.suite = :suite_id 198 AND origin.component = target.component 199 AND origin.type = target.type) 200 WHERE origin.suite = :originsuite_id 201 AND origin.component = :component_id 202 AND origin.type = :type_id""", 203 {'suite_id': osuite_id, 'originsuite_id': originosuite_id, 204 'component_id': component_id, 'type_id': type_id}) 205 for i in q.fetchall(): 206 package = i[0] 207 if package not in src_packages or src_packages[package]: 208 if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]): 209 Logger.log(["syncing override", osuite, component, 210 otype, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]]) 211 if not Options["No-Action"]: 212 session.execute("""UPDATE override 213 SET priority = :priority, 214 section = :section, 215 maintainer = :maintainer 216 WHERE package = :package AND suite = :suite_id 217 AND component = :component_id AND type = :type_id""", 218 {'priority': i[1], 219 'section': i[2], 'maintainer': i[3], 220 'package': package, 'suite_id': osuite_id, 221 'component_id': component_id, 'type_id': dsc_type_id}) 222 continue 223 224 # we can copy 225 src_packages[package] = 1 226 Logger.log(["copying missing override", osuite, component, 227 otype, package, "source", sections[i[2]], i[3]]) 228 if not Options["No-Action"]: 229 session.execute("""INSERT INTO override (package, suite, component, 230 priority, section, type, maintainer) 231 VALUES (:package, :suite_id, :component_id, 232 :priority_id, :section_id, :type_id, 233 :maintainer)""", 234 {'package': package, 'suite_id': osuite_id, 235 'component_id': component_id, 'priority_id': source_priority_id, 236 'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]}) 237 238 for package, hasoverride in list(src_packages.items()): 239 if not hasoverride: 240 utils.warn("%s has no override!" % package) 241 242 else: # binary override 243 for i in q.fetchall(): 244 package = i[0] 245 if package in packages: 246 packages[package] = 1 247 else: 248 if package in blacklist: 249 utils.warn("%s in incoming, not touching" % package) 250 continue 251 Logger.log(["removing unused override", osuite, component, 252 otype, package, priorities[i[1]], sections[i[2]], i[3]]) 253 if not Options["No-Action"]: 254 session.execute("""DELETE FROM override 255 WHERE package = :package AND suite = :suite_id 256 AND component = :component_id AND type = :type_id 257 AND created < now() - interval '14 days'""", 258 {'package': package, 'suite_id': osuite_id, 259 'component_id': component_id, 'type_id': type_id}) 260 261 # Check whether originosuite has an override for us we can 262 # copy 263 if originosuite: 264 q = session.execute("""SELECT origin.package, origin.priority, origin.section, 265 origin.maintainer, target.priority, target.section, 266 target.maintainer 267 FROM override origin LEFT JOIN override target 268 ON (origin.package = target.package 269 AND target.suite = :suite_id 270 AND origin.component = target.component 271 AND origin.type = target.type) 272 WHERE origin.suite = :originsuite_id 273 AND origin.component = :component_id 274 AND origin.type = :type_id""", 275 {'suite_id': osuite_id, 'originsuite_id': originosuite_id, 276 'component_id': component_id, 'type_id': type_id}) 277 for i in q.fetchall(): 278 package = i[0] 279 if package not in packages or packages[package]: 280 if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]): 281 Logger.log(["syncing override", osuite, component, 282 otype, package, priorities[i[4]], sections[i[5]], 283 i[6], priorities[i[1]], sections[i[2]], i[3]]) 284 if not Options["No-Action"]: 285 session.execute("""UPDATE override 286 SET priority = :priority_id, 287 section = :section_id, 288 maintainer = :maintainer 289 WHERE package = :package 290 AND suite = :suite_id 291 AND component = :component_id 292 AND type = :type_id""", 293 {'priority_id': i[1], 'section_id': i[2], 294 'maintainer': i[3], 'package': package, 295 'suite_id': osuite_id, 'component_id': component_id, 296 'type_id': type_id}) 297 continue 298 # we can copy 299 packages[package] = 1 300 Logger.log(["copying missing override", osuite, component, 301 otype, package, priorities[i[1]], sections[i[2]], i[3]]) 302 if not Options["No-Action"]: 303 session.execute("""INSERT INTO override (package, suite, component, 304 priority, section, type, maintainer) 305 VALUES (:package, :suite_id, :component_id, 306 :priority_id, :section_id, :type_id, :maintainer)""", 307 {'package': package, 'suite_id': osuite_id, 308 'component_id': component_id, 'priority_id': i[1], 309 'section_id': i[2], 'type_id': type_id, 'maintainer': i[3]}) 310 311 for package, hasoverride in list(packages.items()): 312 if not hasoverride: 313 utils.warn("%s has no override!" % package) 314 315 session.commit() 316 sys.stdout.flush()
317 318 319 ################################################################################ 320
321 -def main():
322 global Logger, Options, sections, priorities 323 324 cnf = Config() 325 326 Arguments = [('h', "help", "Check-Overrides::Options::Help"), 327 ('n', "no-action", "Check-Overrides::Options::No-Action")] 328 for i in ["help", "no-action"]: 329 key = "Check-Overrides::Options::%s" % i 330 if key not in cnf: 331 cnf[key] = "" 332 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) 333 Options = cnf.subtree("Check-Overrides::Options") 334 335 if Options["Help"]: 336 usage() 337 338 session = DBConn().session() 339 340 # init sections, priorities: 341 342 # We need forward and reverse 343 sections = get_sections(session) 344 for name, entry in list(sections.items()): 345 sections[entry] = name 346 347 priorities = get_priorities(session) 348 for name, entry in list(priorities.items()): 349 priorities[entry] = name 350 351 if not Options["No-Action"]: 352 Logger = daklog.Logger("check-overrides") 353 else: 354 Logger = daklog.Logger("check-overrides", 1) 355 356 for suite in session.query(Suite).filter(Suite.overrideprocess == True): # noqa:E712 357 originosuite = None 358 originremark = '' 359 360 if suite.overrideorigin is not None: 361 originosuite = get_suite(suite.overrideorigin, session) 362 if originosuite is None: 363 utils.fubar("%s has an override origin suite of %s but it doesn't exist!" % (suite.suite_name, suite.overrideorigin)) 364 originosuite = originosuite.suite_name 365 originremark = " taking missing from %s" % originosuite 366 367 print("Processing %s%s..." % (suite.suite_name, originremark)) 368 369 # Get a list of all suites that use the override file of 'suite.suite_name' as 370 # well as the suite 371 ocodename = suite.codename 372 suiteids = [x.suite_id for x in session.query(Suite).filter(Suite.overridecodename == ocodename).all()] 373 if suite.suite_id not in suiteids: 374 suiteids.append(suite.suite_id) 375 376 if len(suiteids) < 1: 377 utils.fubar("Couldn't find id's of all suites: %s" % suiteids) 378 379 for component in session.query(Component).all(): 380 # It is crucial for the dsc override creation based on binary 381 # overrides that 'dsc' goes first 382 component_name = component.component_name 383 otypes = ['dsc'] 384 for ot in session.query(OverrideType): 385 if ot.overridetype == 'dsc': 386 continue 387 otypes.append(ot.overridetype) 388 389 for otype in otypes: 390 print("Processing %s [%s - %s]" 391 % (suite.suite_name, component_name, otype)) 392 sys.stdout.flush() 393 process(suite.suite_name, suiteids, originosuite, component_name, otype, session) 394 395 Logger.close()
396 397 ################################################################################ 398 399 400 if __name__ == '__main__': 401 main() 402