Package dak :: Module process_policy
[hide private]
[frames] | no frames]

Source Code for Module dak.process_policy

  1  #! /usr/bin/env python3 
  2  # vim:set et ts=4 sw=4: 
  3   
  4  """ Handles packages from policy queues 
  5   
  6  @contact: Debian FTP Master <ftpmaster@debian.org> 
  7  @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org> 
  8  @copyright: 2009 Joerg Jaspert <joerg@debian.org> 
  9  @copyright: 2009 Frank Lichtenheld <djpig@debian.org> 
 10  @copyright: 2009 Mark Hymers <mhy@debian.org> 
 11  @license: GNU General Public License version 2 or later 
 12  """ 
 13  # This program is free software; you can redistribute it and/or modify 
 14  # it under the terms of the GNU General Public License as published by 
 15  # the Free Software Foundation; either version 2 of the License, or 
 16  # (at your option) any later version. 
 17   
 18  # This program is distributed in the hope that it will be useful, 
 19  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 20  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 21  # GNU General Public License for more details. 
 22   
 23  # You should have received a copy of the GNU General Public License 
 24  # along with this program; if not, write to the Free Software 
 25  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
 26   
 27  ################################################################################ 
 28   
 29  # <mhy> So how do we handle that at the moment? 
 30  # <stew> Probably incorrectly. 
 31   
 32  ################################################################################ 
 33   
 34  import os 
 35  import datetime 
 36  import re 
 37  import sys 
 38  import traceback 
 39  import apt_pkg 
 40  from sqlalchemy.orm.exc import NoResultFound 
 41  import sqlalchemy.sql as sql 
 42   
 43  from daklib.dbconn import * 
 44  from daklib import daklog 
 45  from daklib import utils 
 46  from daklib.externalsignature import check_upload_for_external_signature_request 
 47  from daklib.config import Config 
 48  from daklib.archive import ArchiveTransaction, source_component_from_package_list 
 49  from daklib.urgencylog import UrgencyLog 
 50  from daklib.packagelist import PackageList 
 51   
 52  import daklib.announce 
 53  import daklib.upload 
 54  import daklib.utils 
 55   
 56  # Globals 
 57  Options = None 
 58  Logger = None 
59 60 ################################################################################ 61 62 63 -def do_comments(dir, srcqueue, opref, npref, line, fn, transaction):
64 session = transaction.session 65 actions = [] 66 for comm in [x for x in os.listdir(dir) if x.startswith(opref)]: 67 with open(os.path.join(dir, comm)) as fd: 68 lines = fd.readlines() 69 if len(lines) == 0 or lines[0] != line + "\n": 70 continue 71 72 # If the ACCEPT includes a _<arch> we only accept that .changes. 73 # Otherwise we accept all .changes that start with the given prefix 74 changes_prefix = comm[len(opref):] 75 if changes_prefix.count('_') < 2: 76 changes_prefix = changes_prefix + '_' 77 else: 78 changes_prefix = changes_prefix + '.changes' 79 80 # We need to escape "_" as we use it with the LIKE operator (via the 81 # SQLA startwith) later. 82 changes_prefix = changes_prefix.replace("_", r"\_") 83 84 uploads = session.query(PolicyQueueUpload).filter_by(policy_queue=srcqueue) \ 85 .join(PolicyQueueUpload.changes).filter(DBChange.changesname.startswith(changes_prefix)) \ 86 .order_by(PolicyQueueUpload.source_id) 87 reason = "".join(lines[1:]) 88 actions.extend((u, reason) for u in uploads) 89 90 if opref != npref: 91 newcomm = npref + comm[len(opref):] 92 newcomm = utils.find_next_free(os.path.join(dir, newcomm)) 93 transaction.fs.move(os.path.join(dir, comm), newcomm) 94 95 actions.sort() 96 97 for u, reason in actions: 98 print(("Processing changes file: {0}".format(u.changes.changesname))) 99 fn(u, srcqueue, reason, transaction)
100
101 ################################################################################ 102 103 104 -def try_or_reject(function):
105 def wrapper(upload, srcqueue, comments, transaction): 106 try: 107 function(upload, srcqueue, comments, transaction) 108 except Exception as e: 109 comments = 'An exception was raised while processing the package:\n{0}\nOriginal comments:\n{1}'.format(traceback.format_exc(), comments) 110 try: 111 transaction.rollback() 112 real_comment_reject(upload, srcqueue, comments, transaction) 113 except Exception as e: 114 comments = 'In addition an exception was raised while trying to reject the upload:\n{0}\nOriginal rejection:\n{1}'.format(traceback.format_exc(), comments) 115 transaction.rollback() 116 real_comment_reject(upload, srcqueue, comments, transaction, notify=False) 117 if not Options['No-Action']: 118 transaction.commit() 119 else: 120 transaction.rollback()
121 return wrapper 122
123 ################################################################################ 124 125 126 @try_or_reject 127 -def comment_accept(upload, srcqueue, comments, transaction):
128 for byhand in upload.byhand: 129 path = os.path.join(srcqueue.path, byhand.filename) 130 if os.path.exists(path): 131 raise Exception('E: cannot ACCEPT upload with unprocessed byhand file {0}'.format(byhand.filename)) 132 133 cnf = Config() 134 135 fs = transaction.fs 136 session = transaction.session 137 changesname = upload.changes.changesname 138 allow_tainted = srcqueue.suite.archive.tainted 139 140 # We need overrides to get the target component 141 overridesuite = upload.target_suite 142 if overridesuite.overridesuite is not None: 143 overridesuite = session.query(Suite).filter_by(suite_name=overridesuite.overridesuite).one() 144 145 def binary_component_func(db_binary): 146 section = db_binary.proxy['Section'] 147 component_name = 'main' 148 if section.find('/') != -1: 149 component_name = section.split('/', 1)[0] 150 return get_mapped_component(component_name, session=session)
151 152 def is_debug_binary(db_binary): 153 return daklib.utils.is_in_debug_section(db_binary.proxy) 154 155 def has_debug_binaries(upload): 156 return any((is_debug_binary(x) for x in upload.binaries)) 157 158 def source_component_func(db_source): 159 package_list = PackageList(db_source.proxy) 160 component = source_component_from_package_list(package_list, upload.target_suite) 161 if component is not None: 162 return get_mapped_component(component.component_name, session=session) 163 164 # Fallback for packages without Package-List field 165 query = session.query(Override).filter_by(suite=overridesuite, package=db_source.source) \ 166 .join(OverrideType).filter(OverrideType.overridetype == 'dsc') \ 167 .join(Component) 168 return query.one().component 169 170 policy_queue = upload.target_suite.policy_queue 171 if policy_queue == srcqueue: 172 policy_queue = None 173 174 all_target_suites = [upload.target_suite if policy_queue is None else policy_queue.suite] 175 if policy_queue is None or policy_queue.send_to_build_queues: 176 all_target_suites.extend([q.suite for q in upload.target_suite.copy_queues]) 177 178 throw_away_binaries = False 179 if upload.source is not None: 180 source_component = source_component_func(upload.source) 181 if upload.target_suite.suite_name in cnf.value_list('Dinstall::ThrowAwayNewBinarySuites') and \ 182 source_component.component_name in cnf.value_list('Dinstall::ThrowAwayNewBinaryComponents'): 183 throw_away_binaries = True 184 185 for suite in all_target_suites: 186 debug_suite = suite.debug_suite 187 188 if upload.source is not None: 189 # If we have Source in this upload, let's include it into 190 # upload suite. 191 transaction.copy_source( 192 upload.source, 193 suite, 194 source_component, 195 allow_tainted=allow_tainted, 196 ) 197 198 if not throw_away_binaries: 199 if debug_suite is not None and has_debug_binaries(upload): 200 # If we're handing a debug package, we also need to include the 201 # source in the debug suite as well. 202 transaction.copy_source( 203 upload.source, 204 debug_suite, 205 source_component_func(upload.source), 206 allow_tainted=allow_tainted, 207 ) 208 209 if not throw_away_binaries: 210 for db_binary in upload.binaries: 211 # Now, let's work out where to copy this guy to -- if it's 212 # a debug binary, and the suite has a debug suite, let's go 213 # ahead and target the debug suite rather then the stock 214 # suite. 215 copy_to_suite = suite 216 if debug_suite is not None and is_debug_binary(db_binary): 217 copy_to_suite = debug_suite 218 219 # build queues and debug suites may miss the source package 220 # if this is a binary-only upload. 221 if copy_to_suite != upload.target_suite: 222 transaction.copy_source( 223 db_binary.source, 224 copy_to_suite, 225 source_component_func(db_binary.source), 226 allow_tainted=allow_tainted, 227 ) 228 229 transaction.copy_binary( 230 db_binary, 231 copy_to_suite, 232 binary_component_func(db_binary), 233 allow_tainted=allow_tainted, 234 extra_archives=[upload.target_suite.archive], 235 ) 236 237 check_upload_for_external_signature_request(session, suite, copy_to_suite, db_binary) 238 239 suite.update_last_changed() 240 241 # Copy .changes if needed 242 if policy_queue is None and upload.target_suite.copychanges: 243 src = os.path.join(upload.policy_queue.path, upload.changes.changesname) 244 dst = os.path.join(upload.target_suite.path, upload.changes.changesname) 245 fs.copy(src, dst, mode=upload.target_suite.archive.mode) 246 247 # List of files in the queue directory 248 queue_files = [changesname] 249 chg = daklib.upload.Changes(upload.policy_queue.path, changesname, keyrings=[], require_signature=False) 250 queue_files.extend(f.filename for f in chg.buildinfo_files) 251 252 # TODO: similar code exists in archive.py's `ArchiveUpload._install_policy` 253 if policy_queue is not None: 254 # register upload in policy queue 255 new_upload = PolicyQueueUpload() 256 new_upload.policy_queue = policy_queue 257 new_upload.target_suite = upload.target_suite 258 new_upload.changes = upload.changes 259 new_upload.source = upload.source 260 new_upload.binaries = upload.binaries 261 session.add(new_upload) 262 session.flush() 263 264 # copy .changes & similar to policy queue 265 for fn in queue_files: 266 src = os.path.join(upload.policy_queue.path, fn) 267 dst = os.path.join(policy_queue.path, fn) 268 transaction.fs.copy(src, dst, mode=policy_queue.change_perms) 269 270 # Copy upload to Process-Policy::CopyDir 271 # Used on security.d.o to sync accepted packages to ftp-master, but this 272 # should eventually be replaced by something else. 273 copydir = cnf.get('Process-Policy::CopyDir') or None 274 if policy_queue is None and copydir is not None: 275 mode = upload.target_suite.archive.mode 276 if upload.source is not None: 277 for f in [df.poolfile for df in upload.source.srcfiles]: 278 dst = os.path.join(copydir, f.basename) 279 if not os.path.exists(dst): 280 fs.copy(f.fullpath, dst, mode=mode) 281 282 for db_binary in upload.binaries: 283 f = db_binary.poolfile 284 dst = os.path.join(copydir, f.basename) 285 if not os.path.exists(dst): 286 fs.copy(f.fullpath, dst, mode=mode) 287 288 for fn in queue_files: 289 src = os.path.join(upload.policy_queue.path, fn) 290 dst = os.path.join(copydir, fn) 291 # We check for `src` to exist as old uploads in policy queues 292 # might still miss the `.buildinfo` files. 293 if os.path.exists(src) and not os.path.exists(dst): 294 fs.copy(src, dst, mode=mode) 295 296 if policy_queue is None: 297 utils.process_buildinfos(upload.policy_queue.path, chg.buildinfo_files, 298 fs, Logger) 299 300 if policy_queue is None and upload.source is not None and not Options['No-Action']: 301 urgency = upload.changes.urgency 302 # As per policy 5.6.17, the urgency can be followed by a space and a 303 # comment. Extract only the urgency from the string. 304 if ' ' in urgency: 305 urgency, comment = urgency.split(' ', 1) 306 if urgency not in cnf.value_list('Urgency::Valid'): 307 urgency = cnf['Urgency::Default'] 308 UrgencyLog().log(upload.source.source, upload.source.version, urgency) 309 310 if policy_queue is None: 311 print(" ACCEPT") 312 else: 313 print(" ACCEPT-TO-QUEUE") 314 if not Options['No-Action']: 315 Logger.log(["Policy Queue ACCEPT", srcqueue.queue_name, changesname]) 316 317 if policy_queue is None: 318 pu = get_processed_upload(upload) 319 daklib.announce.announce_accept(pu) 320 321 # TODO: code duplication. Similar code is in process-upload. 322 # Move .changes to done 323 now = datetime.datetime.now() 324 donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d')) 325 if policy_queue is None: 326 for fn in queue_files: 327 src = os.path.join(upload.policy_queue.path, fn) 328 if os.path.exists(src): 329 dst = os.path.join(donedir, fn) 330 dst = utils.find_next_free(dst) 331 fs.copy(src, dst, mode=0o644) 332 333 if throw_away_binaries and upload.target_suite.archive.use_morgue: 334 morguesubdir = cnf.get("New::MorgueSubDir", 'new') 335 336 utils.move_to_morgue(morguesubdir, 337 [db_binary.poolfile.fullpath for db_binary in upload.binaries], 338 fs, Logger) 339 340 remove_upload(upload, transaction) 341
342 ################################################################################ 343 344 345 @try_or_reject 346 -def comment_reject(*args):
347 real_comment_reject(*args, manual=True)
348
349 350 -def real_comment_reject(upload, srcqueue, comments, transaction, notify=True, manual=False):
351 cnf = Config() 352 353 fs = transaction.fs 354 session = transaction.session 355 changesname = upload.changes.changesname 356 queuedir = upload.policy_queue.path 357 rejectdir = cnf['Dir::Reject'] 358 359 ### Copy files to reject/ 360 361 poolfiles = [b.poolfile for b in upload.binaries] 362 if upload.source is not None: 363 poolfiles.extend([df.poolfile for df in upload.source.srcfiles]) 364 # Not beautiful... 365 files = [af.path for af in session.query(ArchiveFile) 366 .filter_by(archive=upload.policy_queue.suite.archive) 367 .join(ArchiveFile.file) 368 .filter(PoolFile.file_id.in_([f.file_id for f in poolfiles]))] 369 for byhand in upload.byhand: 370 path = os.path.join(queuedir, byhand.filename) 371 if os.path.exists(path): 372 files.append(path) 373 chg = daklib.upload.Changes(queuedir, changesname, keyrings=[], require_signature=False) 374 for f in chg.buildinfo_files: 375 path = os.path.join(queuedir, f.filename) 376 if os.path.exists(path): 377 files.append(path) 378 files.append(os.path.join(queuedir, changesname)) 379 380 for fn in files: 381 dst = utils.find_next_free(os.path.join(rejectdir, os.path.basename(fn))) 382 fs.copy(fn, dst, link=True) 383 384 ### Write reason 385 386 dst = utils.find_next_free(os.path.join(rejectdir, '{0}.reason'.format(changesname))) 387 fh = fs.create(dst) 388 fh.write(comments) 389 fh.close() 390 391 ### Send mail notification 392 393 if notify: 394 rejected_by = None 395 reason = comments 396 397 # Try to use From: from comment file if there is one. 398 # This is not very elegant... 399 match = re.match(r"\AFrom: ([^\n]+)\n\n", comments) 400 if match: 401 rejected_by = match.group(1) 402 reason = '\n'.join(comments.splitlines()[2:]) 403 404 pu = get_processed_upload(upload) 405 daklib.announce.announce_reject(pu, reason, rejected_by) 406 407 print(" REJECT") 408 if not Options["No-Action"]: 409 Logger.log(["Policy Queue REJECT", srcqueue.queue_name, upload.changes.changesname]) 410 411 changes = upload.changes 412 remove_upload(upload, transaction) 413 session.delete(changes)
414
415 ################################################################################ 416 417 418 -def remove_upload(upload, transaction):
419 fs = transaction.fs 420 session = transaction.session 421 changes = upload.changes 422 423 # Remove byhand and changes files. Binary and source packages will be 424 # removed from {bin,src}_associations and eventually removed by clean-suites automatically. 425 queuedir = upload.policy_queue.path 426 for byhand in upload.byhand: 427 path = os.path.join(queuedir, byhand.filename) 428 if os.path.exists(path): 429 fs.unlink(path) 430 session.delete(byhand) 431 432 chg = daklib.upload.Changes(queuedir, upload.changes.changesname, keyrings=[], require_signature=False) 433 queue_files = [upload.changes.changesname] 434 queue_files.extend(f.filename for f in chg.buildinfo_files) 435 for fn in queue_files: 436 # We check for `path` to exist as old uploads in policy queues 437 # might still miss the `.buildinfo` files. 438 path = os.path.join(queuedir, fn) 439 if os.path.exists(path): 440 fs.unlink(path) 441 442 session.delete(upload) 443 session.flush()
444
445 ################################################################################ 446 447 448 -def get_processed_upload(upload):
449 pu = daklib.announce.ProcessedUpload() 450 451 pu.maintainer = upload.changes.maintainer 452 pu.changed_by = upload.changes.changedby 453 pu.fingerprint = upload.changes.fingerprint 454 455 pu.suites = [upload.target_suite] 456 pu.from_policy_suites = [upload.target_suite] 457 458 changes_path = os.path.join(upload.policy_queue.path, upload.changes.changesname) 459 with open(changes_path, 'r') as fd: 460 pu.changes = fd.read() 461 pu.changes_filename = upload.changes.changesname 462 pu.sourceful = upload.source is not None 463 pu.source = upload.changes.source 464 pu.version = upload.changes.version 465 pu.architecture = upload.changes.architecture 466 pu.bugs = upload.changes.closes 467 468 pu.program = "process-policy" 469 470 return pu
471
472 ################################################################################ 473 474 475 -def remove_unreferenced_binaries(policy_queue, transaction):
476 """Remove binaries that are no longer referenced by an upload 477 478 @type policy_queue: L{daklib.dbconn.PolicyQueue} 479 480 @type transaction: L{daklib.archive.ArchiveTransaction} 481 """ 482 session = transaction.session 483 suite = policy_queue.suite 484 485 query = sql.text(""" 486 SELECT b.* 487 FROM binaries b 488 JOIN bin_associations ba ON b.id = ba.bin 489 WHERE ba.suite = :suite_id 490 AND NOT EXISTS (SELECT 1 FROM policy_queue_upload_binaries_map pqubm 491 JOIN policy_queue_upload pqu ON pqubm.policy_queue_upload_id = pqu.id 492 WHERE pqu.policy_queue_id = :policy_queue_id 493 AND pqubm.binary_id = b.id)""") 494 binaries = session.query(DBBinary).from_statement(query) \ 495 .params({'suite_id': policy_queue.suite_id, 'policy_queue_id': policy_queue.policy_queue_id}) 496 497 for binary in binaries: 498 Logger.log(["removed binary from policy queue", policy_queue.queue_name, binary.package, binary.version]) 499 transaction.remove_binary(binary, suite)
500
501 502 -def remove_unreferenced_sources(policy_queue, transaction):
503 """Remove sources that are no longer referenced by an upload or a binary 504 505 @type policy_queue: L{daklib.dbconn.PolicyQueue} 506 507 @type transaction: L{daklib.archive.ArchiveTransaction} 508 """ 509 session = transaction.session 510 suite = policy_queue.suite 511 512 query = sql.text(""" 513 SELECT s.* 514 FROM source s 515 JOIN src_associations sa ON s.id = sa.source 516 WHERE sa.suite = :suite_id 517 AND NOT EXISTS (SELECT 1 FROM policy_queue_upload pqu 518 WHERE pqu.policy_queue_id = :policy_queue_id 519 AND pqu.source_id = s.id) 520 AND NOT EXISTS (SELECT 1 FROM binaries b 521 JOIN bin_associations ba ON b.id = ba.bin 522 WHERE b.source = s.id 523 AND ba.suite = :suite_id)""") 524 sources = session.query(DBSource).from_statement(query) \ 525 .params({'suite_id': policy_queue.suite_id, 'policy_queue_id': policy_queue.policy_queue_id}) 526 527 for source in sources: 528 Logger.log(["removed source from policy queue", policy_queue.queue_name, source.source, source.version]) 529 transaction.remove_source(source, suite)
530
531 ################################################################################ 532 533 534 -def usage(status=0):
535 print("""Usage: dak process-policy QUEUE""") 536 sys.exit(status)
537
538 ################################################################################ 539 540 541 -def main():
542 global Options, Logger 543 544 cnf = Config() 545 session = DBConn().session() 546 547 Arguments = [('h', "help", "Process-Policy::Options::Help"), 548 ('n', "no-action", "Process-Policy::Options::No-Action")] 549 550 for i in ["help", "no-action"]: 551 key = "Process-Policy::Options::%s" % i 552 if key not in cnf: 553 cnf[key] = "" 554 555 queue_name = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) 556 557 Options = cnf.subtree("Process-Policy::Options") 558 if Options["Help"]: 559 usage() 560 561 if len(queue_name) != 1: 562 print("E: Specify exactly one policy queue") 563 sys.exit(1) 564 565 queue_name = queue_name[0] 566 567 Logger = daklog.Logger("process-policy") 568 if not Options["No-Action"]: 569 urgencylog = UrgencyLog() 570 571 with ArchiveTransaction() as transaction: 572 session = transaction.session 573 try: 574 pq = session.query(PolicyQueue).filter_by(queue_name=queue_name).one() 575 except NoResultFound: 576 print("E: Cannot find policy queue %s" % queue_name) 577 sys.exit(1) 578 579 commentsdir = os.path.join(pq.path, 'COMMENTS') 580 # The comments stuff relies on being in the right directory 581 os.chdir(pq.path) 582 583 do_comments(commentsdir, pq, "REJECT.", "REJECTED.", "NOTOK", comment_reject, transaction) 584 do_comments(commentsdir, pq, "ACCEPT.", "ACCEPTED.", "OK", comment_accept, transaction) 585 do_comments(commentsdir, pq, "ACCEPTED.", "ACCEPTED.", "OK", comment_accept, transaction) 586 587 remove_unreferenced_binaries(pq, transaction) 588 remove_unreferenced_sources(pq, transaction) 589 590 if not Options['No-Action']: 591 urgencylog.close()
592 593 ################################################################################ 594 595 596 if __name__ == '__main__': 597 main() 598