dak.manage_build

1 #! /usr/bin/env python3 2 3 """Manage build queues 4 5 @contact: Debian FTPMaster <ftpmaster@debian.org> 6 @copyright: 2000, 2001, 2002, 2006 James Troup <james@nocrew.org> 7 @copyright: 2009 Mark Hymers <mhy@debian.org> 8 @copyright: 2012, Ansgar Burchardt <ansgar@debian.org> 9 10 """ 11 12 # This program is free software; you can redistribute it and/or modify 13 # it under the terms of the GNU General Public License as published by 14 # the Free Software Foundation; either version 2 of the License, or 15 # (at your option) any later version. 16 17 # This program is distributed in the hope that it will be useful, 18 # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 20 # GNU General Public License for more details. 21 22 # You should have received a copy of the GNU General Public License 23 # along with this program; if not, write to the Free Software 24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 25 26 ################################################################################ 27 28 import sys 29 from datetime import datetime, timedelta 30 31 import apt_pkg 32 import sqlalchemy.sql as sql 33 34 from daklib import daklog 35 from daklib.archive import ArchiveTransaction 36 from daklib.config import Config 37 from daklib.dbconn import BuildQueue, DBBinary, DBSource 38 39 ################################################################################ 40 41 Options = None 42 Logger = None 43 44 ################################################################################ 45 46

47 -def usage(exit_code=0):

48 print( 49 """Usage: dak manage-build-queues [OPTIONS] buildqueue1 buildqueue2 50 Manage the contents of one or more build queues 51 52 -a, --all run on all known build queues 53 -n, --no-action don't do anything 54 -h, --help show this help and exit""" 55 ) 56 57 sys.exit(exit_code)

58 59 60 ################################################################################ 61 62

63 -def clean(build_queue, transaction, now=None):

64 session = transaction.session 65 if now is None: 66 now = datetime.now() 67 68 delete_before = now - timedelta(seconds=build_queue.stay_of_execution) 69 suite = build_queue.suite 70 suite_was_changed = False 71 72 # Remove binaries subject to the following conditions: 73 # 1. Keep binaries that are in policy queues. 74 # 2. Remove binaries that are not in suites. 75 # 3. Remove binaries that have been in the build queue for some time. 76 query = sql.text( 77 """ 78 SELECT b.* 79 FROM binaries b 80 JOIN bin_associations ba ON b.id = ba.bin 81 WHERE ba.suite = :suite_id 82 AND NOT EXISTS 83 (SELECT 1 FROM policy_queue_upload_binaries_map pqubm 84 JOIN policy_queue_upload pqu ON pqu.id = pqubm.policy_queue_upload_id 85 JOIN policy_queue pq ON pq.id = pqu.policy_queue_id 86 JOIN suite s ON s.policy_queue_id = pq.id 87 JOIN suite_build_queue_copy sbqc ON sbqc.suite = s.id 88 WHERE pqubm.binary_id = ba.bin AND pq.send_to_build_queues 89 AND sbqc.build_queue_id = :build_queue_id) 90 AND (ba.created < :delete_before 91 OR NOT EXISTS 92 (SELECT 1 FROM bin_associations ba2 93 JOIN suite_build_queue_copy sbqc ON sbqc.suite = ba2.suite 94 WHERE ba2.bin = ba.bin AND sbqc.build_queue_id = :build_queue_id))""" 95 ) 96 binaries = ( 97 session.query(DBBinary) 98 .from_statement(query) 99 .params( 100 { 101 "build_queue_id": build_queue.queue_id, 102 "suite_id": suite.suite_id, 103 "delete_before": delete_before, 104 } 105 ) 106 ) 107 for binary in binaries: 108 Logger.log( 109 [ 110 "removed binary from build queue", 111 build_queue.queue_name, 112 binary.package, 113 binary.version, 114 ] 115 ) 116 transaction.remove_binary(binary, suite) 117 suite_was_changed = True 118 119 # Remove sources 120 # Conditions are similar as for binaries, but we also keep sources 121 # if there is a binary in the build queue that uses it. 122 query = sql.text( 123 """ 124 SELECT s.* 125 FROM source s 126 JOIN src_associations sa ON s.id = sa.source 127 WHERE sa.suite = :suite_id 128 AND NOT EXISTS 129 (SELECT 1 FROM policy_queue_upload pqu 130 JOIN policy_queue pq ON pq.id = pqu.policy_queue_id 131 JOIN suite s ON s.policy_queue_id = pq.id 132 JOIN suite_build_queue_copy sbqc ON sbqc.suite = s.id 133 WHERE pqu.source_id = sa.source AND pq.send_to_build_queues 134 AND sbqc.build_queue_id = :build_queue_id) 135 AND (sa.created < :delete_before 136 OR NOT EXISTS 137 (SELECT 1 FROM src_associations sa2 138 JOIN suite_build_queue_copy sbqc ON sbqc.suite = sa2.suite 139 WHERE sbqc.build_queue_id = :build_queue_id 140 AND sa2.source = sa.source)) 141 AND NOT EXISTS 142 (SELECT 1 FROM bin_associations ba 143 JOIN binaries b ON ba.bin = b.id 144 WHERE ba.suite = :suite_id 145 AND b.source = s.id)""" 146 ) 147 sources = ( 148 session.query(DBSource) 149 .from_statement(query) 150 .params( 151 { 152 "build_queue_id": build_queue.queue_id, 153 "suite_id": suite.suite_id, 154 "delete_before": delete_before, 155 } 156 ) 157 ) 158 for source in sources: 159 Logger.log( 160 [ 161 "removed source from build queue", 162 build_queue.queue_name, 163 source.source, 164 source.version, 165 ] 166 ) 167 transaction.remove_source(source, suite) 168 suite_was_changed = True 169 170 if suite_was_changed: 171 suite.update_last_changed()

172 173

174 -def main():

175 global Options, Logger 176 177 cnf = Config() 178 179 for i in ["Help", "No-Action", "All"]: 180 key = "Manage-Build-Queues::Options::%s" % i 181 if key not in cnf: 182 cnf[key] = "" 183 184 Arguments = [ 185 ("h", "help", "Manage-Build-Queues::Options::Help"), 186 ("n", "no-action", "Manage-Build-Queues::Options::No-Action"), 187 ("a", "all", "Manage-Build-Queues::Options::All"), 188 ] 189 190 queue_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) 191 Options = cnf.subtree("Manage-Build-Queues::Options") 192 193 if Options["Help"]: 194 usage() 195 196 Logger = daklog.Logger("manage-build-queues", Options["No-Action"]) 197 198 starttime = datetime.now() 199 200 with ArchiveTransaction() as transaction: 201 session = transaction.session 202 if Options["All"]: 203 if len(queue_names) != 0: 204 print("E: Cannot use both -a and a queue name") 205 sys.exit(1) 206 queues = session.query(BuildQueue) 207 else: 208 queues = session.query(BuildQueue).filter( 209 BuildQueue.queue_name.in_(queue_names) 210 ) 211 212 for q in queues: 213 Logger.log( 214 ["cleaning queue %s using datetime %s" % (q.queue_name, starttime)] 215 ) 216 clean(q, transaction, now=starttime) 217 if not Options["No-Action"]: 218 transaction.commit() 219 else: 220 transaction.rollback() 221 222 Logger.close()

223 224 225 ####################################################################################### 226 227 228 if __name__ == "__main__": 229 main() 230

Source Code for Module dak.manage_build_queues