Source Code for Module dak.check_overrides

  1  #! /usr/bin/env python3 
  2   
  3  """ Cruft checker and hole filler for overrides 
  4   
  5  @contact: Debian FTPMaster <ftpmaster@debian.org> 
  6  @copyright: 2000, 2001, 2002, 2004, 2006  James Troup <james@nocrew.org> 
  7  @opyright: 2005  Jeroen van Wolffelaar <jeroen@wolffelaar.nl> 
  8  @copyright: 2011  Joerg Jaspert <joerg@debian.org> 
  9  @license: GNU General Public License version 2 or later 
 10   
 11  """ 
 12   
 13  # This program is free software; you can redistribute it and/or modify 
 14  # it under the terms of the GNU General Public License as published by 
 15  # the Free Software Foundation; either version 2 of the License, or 
 16  # (at your option) any later version. 
 17   
 18  # This program is distributed in the hope that it will be useful, 
 19  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 20  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 21  # GNU General Public License for more details. 
 22   
 23  # You should have received a copy of the GNU General Public License 
 24  # along with this program; if not, write to the Free Software 
 25  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
 26   
 27  ################################################################################ 
 28   
 29  ###################################################################### 
 30  # NB: dak check-overrides is not a good idea with New Incoming as it # 
 31  # doesn't take into account accepted.  You can minimize the impact   # 
 32  # of this by running it immediately after dak process-accepted but   # 
 33  # that's still racy because 'dak process-new' doesn't lock with 'dak # 
 34  # process-accepted'.  A better long term fix is the evil plan for    # 
 35  # accepted to be in the DB.                                          # 
 36  ###################################################################### 
 37   
 38  # dak check-overrides should now work fine being done during 
 39  # cron.daily, for example just before 'dak make-overrides' (after 'dak 
 40  # process-accepted' and 'dak make-suite-file-list'). At that point, 
 41  # queue/accepted should be empty and installed, so... dak 
 42  # check-overrides does now take into account suites sharing overrides 
 43   
 44  # TODO: 
 45  # * Only update out-of-sync overrides when corresponding versions are equal to 
 46  #   some degree 
 47  # * consistency checks like: 
 48  #   - section=debian-installer only for udeb and # dsc 
 49  #   - priority=optional if dsc 
 50  #   - (suite, package, 'dsc') is unique, 
 51  #   - just as (suite, package, (u)deb) (yes, across components!) 
 52  #   - sections match their component (each component has an own set of sections, 
 53  #     could probably be reduced...) 
 54   
 55  ################################################################################ 
 56   
 57  import sys 
 58  import apt_pkg 
 59   
 60  from daklib.config import Config 
 61  from daklib.dbconn import * 
 62  from daklib import daklog 
 63  from daklib import utils 
 64   
 65  ################################################################################ 
 66   
 67  Options = None                 #: Commandline arguments parsed into this 
 68  Logger = None                  #: Our logging object 
 69  sections = {} 
 70  priorities = {} 
 71  blacklist = {} 
 72   
 73  ################################################################################ 
 74   
 75   
 76 -def usage(exit_code=0): 
 77      print("""Usage: dak check-overrides 
 78  Check for cruft in overrides. 
 79   
 80    -n, --no-action            don't do anything 
 81    -h, --help                 show this help and exit""") 
 82   
 83      sys.exit(exit_code) 
 84   
 85  ################################################################################ 
 86   
 87   
 88 -def process(osuite, affected_suites, originosuite, component, otype, session): 
 89      global Logger, Options, sections, priorities 
 90   
 91      o = get_suite(osuite, session) 
 92      if o is None: 
 93          utils.fubar("Suite '%s' not recognised." % (osuite)) 
 94      osuite_id = o.suite_id 
 95   
 96      originosuite_id = None 
 97      if originosuite: 
 98          oo = get_suite(originosuite, session) 
 99          if oo is None: 
100              utils.fubar("Suite '%s' not recognised." % (originosuite)) 
101          originosuite_id = oo.suite_id 
102   
103      c = get_component(component, session) 
104      if c is None: 
105          utils.fubar("Component '%s' not recognised." % (component)) 
106      component_id = c.component_id 
107   
108      ot = get_override_type(otype, session) 
109      if ot is None: 
110          utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype)) 
111      type_id = ot.overridetype_id 
112      dsc_type_id = get_override_type("dsc", session).overridetype_id 
113   
114      source_priority_id = get_priority("optional", session).priority_id 
115   
116      if otype == "deb" or otype == "udeb": 
117          packages = {} 
118          # TODO: Fix to use placeholders (check how to with arrays) 
119          q = session.execute(""" 
120  SELECT b.package 
121    FROM binaries b 
122    JOIN bin_associations ba ON b.id = ba.bin 
123    JOIN suite ON ba.suite = suite.id 
124    JOIN files_archive_map af ON b.file = af.file_id AND suite.archive_id = af.archive_id 
125   WHERE b.type = :otype AND ba.suite IN :affected_suites AND af.component_id = :component_id 
126  """, {'otype': otype, 'affected_suites': tuple(affected_suites), 'component_id': component_id}) 
127          for i in q.fetchall(): 
128              packages[i[0]] = 0 
129   
130      src_packages = {} 
131      q = session.execute(""" 
132  SELECT s.source FROM source s 
133    JOIN src_associations sa ON s.id = sa.source 
134    JOIN suite ON sa.suite = suite.id 
135    JOIN files_archive_map af ON s.file = af.file_id AND suite.archive_id = af.archive_id 
136   WHERE sa.suite IN :affected_suites AND af.component_id = :component_id 
137  """, {'affected_suites': tuple(affected_suites), 'component_id': component_id}) 
138      for i in q.fetchall(): 
139          src_packages[i[0]] = 0 
140   
141      # ----------- 
142      # Drop unused overrides 
143   
144      q = session.execute("""SELECT package, priority, section, maintainer 
145                               FROM override WHERE suite = :suite_id 
146                                AND component = :component_id AND type = :type_id""", 
147                          {'suite_id': osuite_id, 'component_id': component_id, 
148                           'type_id': type_id}) 
149      # We're already within a transaction 
150      if otype == "dsc": 
151          for i in q.fetchall(): 
152              package = i[0] 
153              if package in src_packages: 
154                  src_packages[package] = 1 
155              else: 
156                  if package in blacklist: 
157                      utils.warn("%s in incoming, not touching" % package) 
158                      continue 
159                  Logger.log(["removing unused override", osuite, component, 
160                      otype, package, priorities[i[1]], sections[i[2]], i[3]]) 
161                  if not Options["No-Action"]: 
162                      session.execute("""DELETE FROM override WHERE package = :package 
163                                            AND suite = :suite_id AND component = :component_id 
164                                            AND type = :type_id 
165                                            AND created < now() - interval '14 days'""", 
166                                      {'package': package, 'suite_id': osuite_id, 
167                                       'component_id': component_id, 'type_id': type_id}) 
168          # create source overrides based on binary overrides, as source 
169          # overrides not always get created 
170          q = session.execute("""SELECT package, priority, section, maintainer 
171                                   FROM override WHERE suite = :suite_id AND component = :component_id""", 
172                              {'suite_id': osuite_id, 'component_id': component_id}) 
173          for i in q.fetchall(): 
174              package = i[0] 
175              if package not in src_packages or src_packages[package]: 
176                  continue 
177              src_packages[package] = 1 
178   
179              Logger.log(["add missing override", osuite, component, 
180                  otype, package, "source", sections[i[2]], i[3]]) 
181              if not Options["No-Action"]: 
182                  session.execute("""INSERT INTO override (package, suite, component, 
183                                                          priority, section, type, maintainer) 
184                                           VALUES (:package, :suite_id, :component_id, 
185                                                   :priority_id, :section_id, :type_id, :maintainer)""", 
186                                 {'package': package, 'suite_id': osuite_id, 
187                                  'component_id': component_id, 'priority_id': source_priority_id, 
188                                  'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]}) 
189          # Check whether originosuite has an override for us we can 
190          # copy 
191          if originosuite: 
192              q = session.execute("""SELECT origin.package, origin.priority, origin.section, 
193                                           origin.maintainer, target.priority, target.section, 
194                                           target.maintainer 
195                                      FROM override origin 
196                                 LEFT JOIN override target ON (origin.package = target.package 
197                                                               AND target.suite = :suite_id 
198                                                               AND origin.component = target.component 
199                                                               AND origin.type = target.type) 
200                                     WHERE origin.suite = :originsuite_id 
201                                       AND origin.component = :component_id 
202                                       AND origin.type = :type_id""", 
203                                  {'suite_id': osuite_id, 'originsuite_id': originosuite_id, 
204                                   'component_id': component_id, 'type_id': type_id}) 
205              for i in q.fetchall(): 
206                  package = i[0] 
207                  if package not in src_packages or src_packages[package]: 
208                      if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]): 
209                          Logger.log(["syncing override", osuite, component, 
210                              otype, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]]) 
211                          if not Options["No-Action"]: 
212                              session.execute("""UPDATE override 
213                                                   SET priority = :priority, 
214                                                       section = :section, 
215                                                       maintainer = :maintainer 
216                                                 WHERE package = :package AND suite = :suite_id 
217                                                   AND component = :component_id AND type = :type_id""", 
218                                              {'priority': i[1], 
219                                               'section': i[2], 'maintainer': i[3], 
220                                               'package': package, 'suite_id': osuite_id, 
221                                               'component_id': component_id, 'type_id': dsc_type_id}) 
222                      continue 
223   
224                  # we can copy 
225                  src_packages[package] = 1 
226                  Logger.log(["copying missing override", osuite, component, 
227                      otype, package, "source", sections[i[2]], i[3]]) 
228                  if not Options["No-Action"]: 
229                      session.execute("""INSERT INTO override (package, suite, component, 
230                                                               priority, section, type, maintainer) 
231                                              VALUES (:package, :suite_id, :component_id, 
232                                                      :priority_id, :section_id, :type_id, 
233                                                      :maintainer)""", 
234                                      {'package': package, 'suite_id': osuite_id, 
235                                       'component_id': component_id, 'priority_id': source_priority_id, 
236                                       'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]}) 
237   
238          for package, hasoverride in list(src_packages.items()): 
239              if not hasoverride: 
240                  utils.warn("%s has no override!" % package) 
241   
242      else: # binary override 
243          for i in q.fetchall(): 
244              package = i[0] 
245              if package in packages: 
246                  packages[package] = 1 
247              else: 
248                  if package in blacklist: 
249                      utils.warn("%s in incoming, not touching" % package) 
250                      continue 
251                  Logger.log(["removing unused override", osuite, component, 
252                      otype, package, priorities[i[1]], sections[i[2]], i[3]]) 
253                  if not Options["No-Action"]: 
254                      session.execute("""DELETE FROM override 
255                                          WHERE package = :package AND suite = :suite_id 
256                                            AND component = :component_id AND type = :type_id 
257                                            AND created < now() - interval '14 days'""", 
258                                      {'package': package, 'suite_id': osuite_id, 
259                                       'component_id': component_id, 'type_id': type_id}) 
260   
261          # Check whether originosuite has an override for us we can 
262          # copy 
263          if originosuite: 
264              q = session.execute("""SELECT origin.package, origin.priority, origin.section, 
265                                            origin.maintainer, target.priority, target.section, 
266                                            target.maintainer 
267                                       FROM override origin LEFT JOIN override target 
268                                                            ON (origin.package = target.package 
269                                                                AND target.suite = :suite_id 
270                                                                AND origin.component = target.component 
271                                                                AND origin.type = target.type) 
272                                      WHERE origin.suite = :originsuite_id 
273                                        AND origin.component = :component_id 
274                                        AND origin.type = :type_id""", 
275                                   {'suite_id': osuite_id, 'originsuite_id': originosuite_id, 
276                                    'component_id': component_id, 'type_id': type_id}) 
277              for i in q.fetchall(): 
278                  package = i[0] 
279                  if package not in packages or packages[package]: 
280                      if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]): 
281                          Logger.log(["syncing override", osuite, component, 
282                              otype, package, priorities[i[4]], sections[i[5]], 
283                              i[6], priorities[i[1]], sections[i[2]], i[3]]) 
284                          if not Options["No-Action"]: 
285                              session.execute("""UPDATE override 
286                                                    SET priority = :priority_id, 
287                                                        section = :section_id, 
288                                                        maintainer = :maintainer 
289                                                  WHERE package = :package 
290                                                    AND suite = :suite_id 
291                                                    AND component = :component_id 
292                                                    AND type = :type_id""", 
293                                              {'priority_id': i[1], 'section_id': i[2], 
294                                               'maintainer': i[3], 'package': package, 
295                                               'suite_id': osuite_id, 'component_id': component_id, 
296                                               'type_id': type_id}) 
297                      continue 
298                  # we can copy 
299                  packages[package] = 1 
300                  Logger.log(["copying missing override", osuite, component, 
301                      otype, package, priorities[i[1]], sections[i[2]], i[3]]) 
302                  if not Options["No-Action"]: 
303                      session.execute("""INSERT INTO override (package, suite, component, 
304                                                               priority, section, type, maintainer) 
305                                              VALUES (:package, :suite_id, :component_id, 
306                                                      :priority_id, :section_id, :type_id, :maintainer)""", 
307                                      {'package': package, 'suite_id': osuite_id, 
308                                       'component_id': component_id, 'priority_id': i[1], 
309                                       'section_id': i[2], 'type_id': type_id, 'maintainer': i[3]}) 
310   
311          for package, hasoverride in list(packages.items()): 
312              if not hasoverride: 
313                  utils.warn("%s has no override!" % package) 
314   
315      session.commit() 
316      sys.stdout.flush() 
317   
318   
319  ################################################################################ 
320   
321 -def main(): 
322      global Logger, Options, sections, priorities 
323   
324      cnf = Config() 
325   
326      Arguments = [('h', "help", "Check-Overrides::Options::Help"), 
327                   ('n', "no-action", "Check-Overrides::Options::No-Action")] 
328      for i in ["help", "no-action"]: 
329          key = "Check-Overrides::Options::%s" % i 
330          if key not in cnf: 
331              cnf[key] = "" 
332      apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) 
333      Options = cnf.subtree("Check-Overrides::Options") 
334   
335      if Options["Help"]: 
336          usage() 
337   
338      session = DBConn().session() 
339   
340      # init sections, priorities: 
341   
342      # We need forward and reverse 
343      sections = get_sections(session) 
344      for name, entry in list(sections.items()): 
345          sections[entry] = name 
346   
347      priorities = get_priorities(session) 
348      for name, entry in list(priorities.items()): 
349          priorities[entry] = name 
350   
351      if not Options["No-Action"]: 
352          Logger = daklog.Logger("check-overrides") 
353      else: 
354          Logger = daklog.Logger("check-overrides", 1) 
355   
356      for suite in session.query(Suite).filter(Suite.overrideprocess == True):  # noqa:E712 
357          originosuite = None 
358          originremark = '' 
359   
360          if suite.overrideorigin is not None: 
361              originosuite = get_suite(suite.overrideorigin, session) 
362              if originosuite is None: 
363                  utils.fubar("%s has an override origin suite of %s but it doesn't exist!" % (suite.suite_name, suite.overrideorigin)) 
364              originosuite = originosuite.suite_name 
365              originremark = " taking missing from %s" % originosuite 
366   
367          print("Processing %s%s..." % (suite.suite_name, originremark)) 
368   
369          # Get a list of all suites that use the override file of 'suite.suite_name' as 
370          # well as the suite 
371          ocodename = suite.codename 
372          suiteids = [x.suite_id for x in session.query(Suite).filter(Suite.overridecodename == ocodename).all()] 
373          if suite.suite_id not in suiteids: 
374              suiteids.append(suite.suite_id) 
375   
376          if len(suiteids) < 1: 
377              utils.fubar("Couldn't find id's of all suites: %s" % suiteids) 
378   
379          for component in session.query(Component).all(): 
380              # It is crucial for the dsc override creation based on binary 
381              # overrides that 'dsc' goes first 
382              component_name = component.component_name 
383              otypes = ['dsc'] 
384              for ot in session.query(OverrideType): 
385                  if ot.overridetype == 'dsc': 
386                      continue 
387                  otypes.append(ot.overridetype) 
388   
389              for otype in otypes: 
390                  print("Processing %s [%s - %s]" 
391                      % (suite.suite_name, component_name, otype)) 
392                  sys.stdout.flush() 
393                  process(suite.suite_name, suiteids, originosuite, component_name, otype, session) 
394   
395      Logger.close() 
396   
397  ################################################################################ 
398   
399   
400  if __name__ == '__main__': 
401      main() 
402