Source Code for Module dak.dominate

  1  #! /usr/bin/env python3 
  2   
  3  """ 
  4  Remove obsolete source and binary associations from suites. 
  5   
  6  @contact: Debian FTP Master <ftpmaster@debian.org> 
  7  @copyright: 2009  Torsten Werner <twerner@debian.org> 
  8  @license: GNU General Public License version 2 or later 
  9  """ 
 10   
 11  # This program is free software; you can redistribute it and/or modify 
 12  # it under the terms of the GNU General Public License as published by 
 13  # the Free Software Foundation; either version 2 of the License, or 
 14  # (at your option) any later version. 
 15   
 16  # This program is distributed in the hope that it will be useful, 
 17  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 18  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 19  # GNU General Public License for more details. 
 20   
 21  # You should have received a copy of the GNU General Public License 
 22  # along with this program; if not, write to the Free Software 
 23  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
 24   
 25  from daklib.dbconn import * 
 26  from daklib.config import Config 
 27  from daklib import daklog, utils 
 28  import apt_pkg 
 29  import sys 
 30   
 31  from sqlalchemy.sql import exists, text 
 32  from tabulate import tabulate 
 33   
 34   
 35  Options = None 
 36  Logger = None 
 37   
 38   
 39 -def retrieve_associations(suites, session): 
 40      return session.execute(text(''' 
 41  WITH 
 42    -- Provide (source, suite) tuple of all source packages to remain 
 43    remain_source AS ( 
 44      SELECT 
 45          * 
 46        FROM ( 
 47          SELECT 
 48              source.id AS source_id, 
 49              src_associations.suite AS suite_id, 
 50              -- generate rank over versions of a source package in one suite 
 51              -- "1" being the newest 
 52              dense_rank() OVER ( 
 53                PARTITION BY source.source, src_associations.suite 
 54                ORDER BY source.version DESC 
 55              ) AS version_rank 
 56            FROM 
 57              source 
 58              INNER JOIN src_associations ON 
 59                src_associations.source = source.id 
 60                AND src_associations.suite = ANY(:suite_ids) 
 61          ) AS source_ranked 
 62        WHERE 
 63          version_rank = 1 
 64    ), 
 65    -- Provide (source, arch, suite) tuple of all binary packages to remain 
 66    remain_binaries AS ( 
 67      SELECT 
 68          * 
 69        FROM ( 
 70          SELECT 
 71              binaries.id, 
 72              binaries.architecture AS arch_id, 
 73              bin_associations.suite AS suite_id, 
 74              source.id AS source_id, 
 75              architecture.arch_string AS arch, 
 76              -- arch of newest version 
 77              first_value(architecture.arch_string) OVER ( 
 78                PARTITION BY binaries.package, bin_associations.suite 
 79                ORDER BY binaries.version DESC 
 80                ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING 
 81              ) as arch_first, 
 82              -- generate rank over versions of a source package in one suite 
 83              -- "1" being the newest 
 84              -- if newest package is arch-any, we use the rank only over current arch 
 85              dense_rank() OVER ( 
 86                PARTITION BY binaries.package, binaries.architecture, bin_associations.suite 
 87                ORDER BY binaries.version DESC 
 88              ) AS version_rank_any, 
 89              -- if newest package is arch-all, we use the rank over all arches 
 90              -- this makes it possible to replace all by any and any by all 
 91              dense_rank() OVER ( 
 92                PARTITION BY binaries.package, bin_associations.suite 
 93                ORDER BY binaries.version DESC 
 94              ) AS version_rank_all 
 95            FROM 
 96              binaries 
 97              INNER JOIN source ON source.id = binaries.source 
 98              INNER JOIN bin_associations ON 
 99                bin_associations.bin = binaries.id 
100                AND bin_associations.suite = ANY(:suite_ids) 
101              INNER JOIN architecture ON architecture.id = binaries.architecture 
102          ) AS source_rank 
103        WHERE 
104          -- we only want to retain the newest of each 
105          CASE 
106            WHEN arch != 'all' AND arch_first != 'all' THEN version_rank_any = 1 
107            ELSE version_rank_all = 1 
108          END 
109    ), 
110    -- Figure out which source we should remove 
111    -- A binary forces the corresponding source to remain 
112    dominate_source AS ( 
113      SELECT 
114          source.source AS source_package, 
115          source.version AS source_version, 
116          source.source AS package, 
117          source.version, 
118          'source'::text AS arch, 
119          suite.suite_name AS suite, 
120          src_associations.id AS assoc_id 
121        FROM 
122          source 
123          INNER JOIN src_associations ON 
124            src_associations.source = source.id 
125            AND src_associations.suite = ANY(:suite_ids) 
126          INNER join suite ON suite.id = src_associations.suite 
127          LEFT JOIN remain_binaries ON 
128            remain_binaries.source_id = source.id 
129            AND remain_binaries.suite_id = src_associations.suite 
130          LEFT JOIN remain_source ON 
131            remain_source.source_id = source.id 
132            AND remain_source.suite_id = src_associations.suite 
133        WHERE 
134          remain_binaries.source_id IS NULL 
135          AND remain_source.source_id IS NULL 
136    ), 
137    -- Figure out which arch-any binaries we should remove 
138    dominate_binaries AS ( 
139      SELECT 
140          source.source AS source_package, 
141          source.version AS source_version, 
142          binaries.package AS package, 
143          binaries.version, 
144          architecture.arch_string AS arch, 
145          suite.suite_name AS suite, 
146          bin_associations.id AS assoc_id 
147        FROM 
148          binaries 
149          INNER JOIN source ON source.id = binaries.source 
150          INNER JOIN bin_associations ON 
151            bin_associations.bin = binaries.id 
152            AND bin_associations.suite = ANY(:suite_ids) 
153          INNER JOIN architecture ON architecture.id = binaries.architecture 
154          INNER join suite ON suite.id = bin_associations.suite 
155          LEFT JOIN remain_binaries ON 
156            remain_binaries.id = binaries.id 
157            AND remain_binaries.arch_id = binaries.architecture 
158            AND remain_binaries.suite_id = bin_associations.suite 
159        WHERE 
160          remain_binaries.source_id IS NULL 
161          AND binaries.architecture != (SELECT id from architecture WHERE arch_string = 'all') 
162    ), 
163    -- Figure out which arch-all binaries we should remove 
164    -- A arch-any binary forces the related arch-all binaries to remain 
165    dominate_binaries_all AS ( 
166      SELECT 
167          source.source AS source_package, 
168          source.version AS source_version, 
169          binaries.package AS package, 
170          binaries.version, 
171          architecture.arch_string AS arch, 
172          suite.suite_name AS suite, 
173          bin_associations.id AS assoc_id 
174        FROM 
175          binaries 
176          INNER JOIN source ON source.id = binaries.source 
177          INNER JOIN bin_associations ON 
178            bin_associations.bin = binaries.id 
179            AND bin_associations.suite = ANY(:suite_ids) 
180          INNER JOIN architecture ON architecture.id = binaries.architecture 
181          INNER join suite ON suite.id = bin_associations.suite 
182          LEFT JOIN remain_binaries ON 
183            remain_binaries.id = binaries.id 
184            AND remain_binaries.arch_id = binaries.architecture 
185            AND remain_binaries.suite_id = bin_associations.suite 
186          LEFT JOIN remain_binaries AS remain_binaries_any ON 
187            remain_binaries_any.source_id = source.id 
188            AND remain_binaries_any.suite_id = bin_associations.suite 
189            AND remain_binaries_any.arch_id != (SELECT id from architecture WHERE arch_string = 'all') 
190        WHERE 
191          remain_binaries.source_id IS NULL 
192          AND remain_binaries_any.source_id IS NULL 
193          AND binaries.architecture = (SELECT id from architecture WHERE arch_string = 'all') 
194    ) 
195  SELECT 
196      * 
197    FROM 
198      dominate_source 
199    UNION SELECT 
200      * 
201    FROM 
202      dominate_binaries 
203    UNION SELECT 
204      * 
205    FROM 
206      dominate_binaries_all 
207    ORDER BY 
208      source_package, source_version, package, version, arch, suite 
209  ''').params( 
210      suite_ids=[s.suite_id for s in suites], 
211  )) 
212   
213   
214 -def delete_associations_table(table, ids, session): 
215      result = session.execute(text(''' 
216          DELETE 
217              FROM {} 
218              WHERE id = ANY(:assoc_ids) 
219      '''.format(table)).params( 
220          assoc_ids=list(ids), 
221      )) 
222   
223      assert result.rowcount == len(ids), 'Rows deleted are not equal to deletion requests' 
224   
225   
226 -def delete_associations(assocs, session): 
227      ids_bin = set() 
228      ids_src = set() 
229   
230      for e in assocs: 
231          Logger.log(['newer', e.package, e.version, e.suite, e.arch, e.assoc_id]) 
232   
233          if e.arch == 'source': 
234              ids_src.add(e.assoc_id) 
235          else: 
236              ids_bin.add(e.assoc_id) 
237   
238      delete_associations_table('bin_associations', ids_bin, session) 
239      delete_associations_table('src_associations', ids_src, session) 
240   
241   
242 -def usage(): 
243      print("""Usage: dak dominate [OPTIONS] 
244  Remove obsolete source and binary associations from suites. 
245   
246      -s, --suite=SUITE          act on this suite 
247      -h, --help                 show this help and exit 
248      -n, --no-action            don't commit changes 
249      -f, --force                also clean up untouchable suites 
250   
251  SUITE can be comma (or space) separated list, e.g. 
252      --suite=testing,unstable""") 
253      sys.exit() 
254   
255   
256 -def main(): 
257      global Options, Logger 
258      cnf = Config() 
259      Arguments = [('h', "help",      "Obsolete::Options::Help"), 
260                   ('s', "suite",     "Obsolete::Options::Suite", "HasArg"), 
261                   ('n', "no-action", "Obsolete::Options::No-Action"), 
262                   ('f', "force",     "Obsolete::Options::Force")] 
263      cnf['Obsolete::Options::Help'] = '' 
264      cnf['Obsolete::Options::No-Action'] = '' 
265      cnf['Obsolete::Options::Force'] = '' 
266      apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) 
267      Options = cnf.subtree("Obsolete::Options") 
268      if Options['Help']: 
269          usage() 
270   
271      if not Options['No-Action']: 
272          Logger = daklog.Logger("dominate") 
273      session = DBConn().session() 
274   
275      suites_query = (session 
276              .query(Suite) 
277              .order_by(Suite.suite_name) 
278              .filter(~exists().where(Suite.suite_id == PolicyQueue.suite_id))) 
279      if 'Suite' in Options: 
280          suites_query = suites_query.filter(Suite.suite_name.in_(utils.split_args(Options['Suite']))) 
281      if not Options['Force']: 
282          suites_query = suites_query.filter_by(untouchable=False) 
283      suites = suites_query.all() 
284   
285      assocs = list(retrieve_associations(suites, session)) 
286   
287      if Options['No-Action']: 
288          headers = ('source package', 'source version', 'package', 'version', 'arch', 'suite', 'id') 
289          print((tabulate(assocs, headers, tablefmt="orgtbl"))) 
290          session.rollback() 
291   
292      else: 
293          delete_associations(assocs, session) 
294          session.commit() 
295   
296      if Logger: 
297          Logger.close() 
298   
299   
300  if __name__ == '__main__': 
301      main() 
302