Source Code for Module dak.manage_build_queues

  1  #! /usr/bin/env python3 
  2   
  3  """ Manage build queues 
  4   
  5  @contact: Debian FTPMaster <ftpmaster@debian.org> 
  6  @copyright: 2000, 2001, 2002, 2006  James Troup <james@nocrew.org> 
  7  @copyright: 2009  Mark Hymers <mhy@debian.org> 
  8  @copyright: 2012, Ansgar Burchardt <ansgar@debian.org> 
  9   
 10  """ 
 11   
 12  # This program is free software; you can redistribute it and/or modify 
 13  # it under the terms of the GNU General Public License as published by 
 14  # the Free Software Foundation; either version 2 of the License, or 
 15  # (at your option) any later version. 
 16   
 17  # This program is distributed in the hope that it will be useful, 
 18  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 19  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 20  # GNU General Public License for more details. 
 21   
 22  # You should have received a copy of the GNU General Public License 
 23  # along with this program; if not, write to the Free Software 
 24  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
 25   
 26  ################################################################################ 
 27   
 28  import apt_pkg 
 29  from datetime import datetime, timedelta 
 30  import sys 
 31  import sqlalchemy.sql as sql 
 32   
 33  from daklib import daklog 
 34  from daklib.archive import ArchiveTransaction 
 35  from daklib.dbconn import * 
 36  from daklib.config import Config 
 37   
 38  ################################################################################ 
 39   
 40  Options = None 
 41  Logger = None 
 42   
 43  ################################################################################ 
 44   
 45   
 46 -def usage(exit_code=0): 
 47      print("""Usage: dak manage-build-queues [OPTIONS] buildqueue1 buildqueue2 
 48  Manage the contents of one or more build queues 
 49   
 50    -a, --all                  run on all known build queues 
 51    -n, --no-action            don't do anything 
 52    -h, --help                 show this help and exit""") 
 53   
 54      sys.exit(exit_code) 
 55   
 56  ################################################################################ 
 57   
 58   
 59 -def clean(build_queue, transaction, now=None): 
 60      session = transaction.session 
 61      if now is None: 
 62          now = datetime.now() 
 63   
 64      delete_before = now - timedelta(seconds=build_queue.stay_of_execution) 
 65      suite = build_queue.suite 
 66      suite_was_changed = False 
 67   
 68      # Remove binaries subject to the following conditions: 
 69      # 1. Keep binaries that are in policy queues. 
 70      # 2. Remove binaries that are not in suites. 
 71      # 3. Remove binaries that have been in the build queue for some time. 
 72      query = sql.text(""" 
 73          SELECT b.* 
 74            FROM binaries b 
 75            JOIN bin_associations ba ON b.id = ba.bin 
 76           WHERE ba.suite = :suite_id 
 77             AND NOT EXISTS 
 78                 (SELECT 1 FROM policy_queue_upload_binaries_map pqubm 
 79                           JOIN policy_queue_upload pqu ON pqu.id = pqubm.policy_queue_upload_id 
 80                           JOIN policy_queue pq ON pq.id = pqu.policy_queue_id 
 81                           JOIN suite s ON s.policy_queue_id = pq.id 
 82                           JOIN suite_build_queue_copy sbqc ON sbqc.suite = s.id 
 83                          WHERE pqubm.binary_id = ba.bin AND pq.send_to_build_queues 
 84                            AND sbqc.build_queue_id = :build_queue_id) 
 85             AND (ba.created < :delete_before 
 86                  OR NOT EXISTS 
 87                     (SELECT 1 FROM bin_associations ba2 
 88                               JOIN suite_build_queue_copy sbqc ON sbqc.suite = ba2.suite 
 89                              WHERE ba2.bin = ba.bin AND sbqc.build_queue_id = :build_queue_id))""") 
 90      binaries = session.query(DBBinary).from_statement(query) \ 
 91          .params({'build_queue_id': build_queue.queue_id, 'suite_id': suite.suite_id, 'delete_before': delete_before}) 
 92      for binary in binaries: 
 93          Logger.log(["removed binary from build queue", build_queue.queue_name, binary.package, binary.version]) 
 94          transaction.remove_binary(binary, suite) 
 95          suite_was_changed = True 
 96   
 97      # Remove sources 
 98      # Conditions are similar as for binaries, but we also keep sources 
 99      # if there is a binary in the build queue that uses it. 
100      query = sql.text(""" 
101          SELECT s.* 
102            FROM source s 
103            JOIN src_associations sa ON s.id = sa.source 
104           WHERE sa.suite = :suite_id 
105             AND NOT EXISTS 
106                 (SELECT 1 FROM policy_queue_upload pqu 
107                           JOIN policy_queue pq ON pq.id = pqu.policy_queue_id 
108                           JOIN suite s ON s.policy_queue_id = pq.id 
109                           JOIN suite_build_queue_copy sbqc ON sbqc.suite = s.id 
110                          WHERE pqu.source_id = sa.source AND pq.send_to_build_queues 
111                            AND sbqc.build_queue_id = :build_queue_id) 
112             AND (sa.created < :delete_before 
113                  OR NOT EXISTS 
114                     (SELECT 1 FROM src_associations sa2 
115                               JOIN suite_build_queue_copy sbqc ON sbqc.suite = sa2.suite 
116                              WHERE sbqc.build_queue_id = :build_queue_id 
117                                AND sa2.source = sa.source)) 
118             AND NOT EXISTS 
119                 (SELECT 1 FROM bin_associations ba 
120                           JOIN binaries b ON ba.bin = b.id 
121                          WHERE ba.suite = :suite_id 
122                            AND b.source = s.id)""") 
123      sources = session.query(DBSource).from_statement(query) \ 
124          .params({'build_queue_id': build_queue.queue_id, 'suite_id': suite.suite_id, 'delete_before': delete_before}) 
125      for source in sources: 
126          Logger.log(["removed source from build queue", build_queue.queue_name, source.source, source.version]) 
127          transaction.remove_source(source, suite) 
128          suite_was_changed = True 
129   
130      if suite_was_changed: 
131          suite.update_last_changed() 
132   
133   
134 -def main(): 
135      global Options, Logger 
136   
137      cnf = Config() 
138   
139      for i in ["Help", "No-Action", "All"]: 
140          key = "Manage-Build-Queues::Options::%s" % i 
141          if key not in cnf: 
142              cnf[key] = "" 
143   
144      Arguments = [('h', "help", "Manage-Build-Queues::Options::Help"), 
145                   ('n', "no-action", "Manage-Build-Queues::Options::No-Action"), 
146                   ('a', "all", "Manage-Build-Queues::Options::All")] 
147   
148      queue_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) 
149      Options = cnf.subtree("Manage-Build-Queues::Options") 
150   
151      if Options["Help"]: 
152          usage() 
153   
154      Logger = daklog.Logger('manage-build-queues', Options['No-Action']) 
155   
156      starttime = datetime.now() 
157   
158      with ArchiveTransaction() as transaction: 
159          session = transaction.session 
160          if Options['All']: 
161              if len(queue_names) != 0: 
162                  print("E: Cannot use both -a and a queue name") 
163                  sys.exit(1) 
164              queues = session.query(BuildQueue) 
165          else: 
166              queues = session.query(BuildQueue).filter(BuildQueue.queue_name.in_(queue_names)) 
167   
168          for q in queues: 
169              Logger.log(['cleaning queue %s using datetime %s' % (q.queue_name, starttime)]) 
170              clean(q, transaction, now=starttime) 
171          if not Options['No-Action']: 
172              transaction.commit() 
173          else: 
174              transaction.rollback() 
175   
176      Logger.close() 
177   
178  ####################################################################################### 
179   
180   
181  if __name__ == '__main__': 
182      main() 
183