Package dak :: Module process_upload
[hide private]
[frames] | no frames]

Source Code for Module dak.process_upload

  1  #! /usr/bin/env python3 
  2   
  3  """ 
  4  Checks Debian packages from Incoming 
  5  @contact: Debian FTP Master <ftpmaster@debian.org> 
  6  @copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org> 
  7  @copyright: 2009  Joerg Jaspert <joerg@debian.org> 
  8  @copyright: 2009  Mark Hymers <mhy@debian.org> 
  9  @copyright: 2009  Frank Lichtenheld <djpig@debian.org> 
 10  @license: GNU General Public License version 2 or later 
 11  """ 
 12   
 13  # This program is free software; you can redistribute it and/or modify 
 14  # it under the terms of the GNU General Public License as published by 
 15  # the Free Software Foundation; either version 2 of the License, or 
 16  # (at your option) any later version. 
 17   
 18  # This program is distributed in the hope that it will be useful, 
 19  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 20  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 21  # GNU General Public License for more details. 
 22   
 23  # You should have received a copy of the GNU General Public License 
 24  # along with this program; if not, write to the Free Software 
 25  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
 26   
 27  # based on process-unchecked and process-accepted 
 28   
 29  ## pu|pa: locking (daily.lock) 
 30  ## pu|pa: parse arguments -> list of changes files 
 31  ## pa: initialize urgency log 
 32  ## pu|pa: sort changes list 
 33   
 34  ## foreach changes: 
 35  ###  pa: load dak file 
 36  ##   pu: copy CHG to tempdir 
 37  ##   pu: check CHG signature 
 38  ##   pu: parse changes file 
 39  ##   pu: checks: 
 40  ##     pu: check distribution (mappings, rejects) 
 41  ##     pu: copy FILES to tempdir 
 42  ##     pu: check whether CHG already exists in CopyChanges 
 43  ##     pu: check whether FILES already exist in one of the policy queues 
 44  ##     for deb in FILES: 
 45  ##       pu: extract control information 
 46  ##       pu: various checks on control information 
 47  ##       pu|pa: search for source (in CHG, projectb, policy queues) 
 48  ##       pu|pa: check whether "Version" fulfills target suite requirements/suite propagation 
 49  ##       pu|pa: check whether deb already exists in the pool 
 50  ##     for src in FILES: 
 51  ##       pu: various checks on filenames and CHG consistency 
 52  ##       pu: if isdsc: check signature 
 53  ##     for file in FILES: 
 54  ##       pu: various checks 
 55  ##       pu: NEW? 
 56  ##       //pu: check whether file already exists in the pool 
 57  ##       pu: store what "Component" the package is currently in 
 58  ##     pu: check whether we found everything we were looking for in CHG 
 59  ##     pu: check the DSC: 
 60  ##       pu: check whether we need and have ONE DSC 
 61  ##       pu: parse the DSC 
 62  ##       pu: various checks //maybe drop some of the in favor of lintian 
 63  ##       pu|pa: check whether "Version" fulfills target suite requirements/suite propagation 
 64  ##       pu: check whether DSC_FILES is consistent with "Format" 
 65  ##       for src in DSC_FILES: 
 66  ##         pu|pa: check whether file already exists in the pool (with special handling for .orig.tar.gz) 
 67  ##     pu: create new tempdir 
 68  ##     pu: create symlink mirror of source 
 69  ##     pu: unpack source 
 70  ##     pu: extract changelog information for BTS 
 71  ##     //pu: create missing .orig symlink 
 72  ##     pu: check with lintian 
 73  ##     for file in FILES: 
 74  ##       pu: check checksums and sizes 
 75  ##     for file in DSC_FILES: 
 76  ##       pu: check checksums and sizes 
 77  ##     pu: CHG: check urgency 
 78  ##     for deb in FILES: 
 79  ##       pu: extract contents list and check for dubious timestamps 
 80  ##     pu: check that the uploader is actually allowed to upload the package 
 81  ###  pa: install: 
 82  ###    if stable_install: 
 83  ###      pa: remove from p-u 
 84  ###      pa: add to stable 
 85  ###      pa: move CHG to morgue 
 86  ###      pa: append data to ChangeLog 
 87  ###      pa: send mail 
 88  ###      pa: remove .dak file 
 89  ###    else: 
 90  ###      pa: add dsc to db: 
 91  ###        for file in DSC_FILES: 
 92  ###          pa: add file to file 
 93  ###          pa: add file to dsc_files 
 94  ###        pa: create source entry 
 95  ###        pa: update source associations 
 96  ###        pa: update src_uploaders 
 97  ###      for deb in FILES: 
 98  ###        pa: add deb to db: 
 99  ###          pa: add file to file 
100  ###          pa: find source entry 
101  ###          pa: create binaries entry 
102  ###          pa: update binary associations 
103  ###      pa: .orig component move 
104  ###      pa: move files to pool 
105  ###      pa: save CHG 
106  ###      pa: move CHG to done/ 
107  ###      pa: change entry in queue_build 
108  ##   pu: use dispatch table to choose target queue: 
109  ##     if NEW: 
110  ##       pu: write .dak file 
111  ##       pu: move to NEW 
112  ##       pu: send mail 
113  ##     elsif AUTOBYHAND: 
114  ##       pu: run autobyhand script 
115  ##       pu: if stuff left, do byhand or accept 
116  ##     elsif targetqueue in (oldstable, stable, embargo, unembargo): 
117  ##       pu: write .dak file 
118  ##       pu: check overrides 
119  ##       pu: move to queue 
120  ##       pu: send mail 
121  ##     else: 
122  ##       pu: write .dak file 
123  ##       pu: move to ACCEPTED 
124  ##       pu: send mails 
125  ##       pu: create files for BTS 
126  ##       pu: create entry in queue_build 
127  ##       pu: check overrides 
128   
129  # Integrity checks 
130  ## GPG 
131  ## Parsing changes (check for duplicates) 
132  ## Parse dsc 
133  ## file list checks 
134   
135  # New check layout (TODO: Implement) 
136  ## Permission checks 
137  ### suite mappings 
138  ### ACLs 
139  ### version checks (suite) 
140  ### override checks 
141   
142  ## Source checks 
143  ### copy orig 
144  ### unpack 
145  ### BTS changelog 
146  ### src contents 
147  ### lintian 
148  ### urgency log 
149   
150  ## Binary checks 
151  ### timestamps 
152  ### control checks 
153  ### src relation check 
154  ### contents 
155   
156  ## Database insertion (? copy from stuff) 
157  ### BYHAND / NEW / Policy queues 
158  ### Pool 
159   
160  ## Queue builds 
161   
162  import datetime 
163  import errno 
164  import fcntl 
165  import os 
166  import sys 
167  import traceback 
168  import apt_pkg 
169  import time 
170   
171  from daklib import daklog 
172  from daklib.dbconn import * 
173  from daklib.urgencylog import UrgencyLog 
174  from daklib.summarystats import SummaryStats 
175  from daklib.config import Config 
176  import daklib.utils as utils 
177  from daklib.regexes import * 
178   
179  import daklib.announce 
180  import daklib.archive 
181  import daklib.checks 
182  import daklib.upload 
183   
184  ############################################################################### 
185   
186  Options = None 
187  Logger = None 
188 189 ############################################################################### 190 191 192 -def usage(exit_code=0):
193 print("""Usage: dak process-upload [OPTION]... [CHANGES]... 194 -a, --automatic automatic run 195 -d, --directory <DIR> process uploads in <DIR> 196 -h, --help show this help and exit. 197 -n, --no-action don't do anything 198 -p, --no-lock don't check lockfile !! for cron.daily only !! 199 -s, --no-mail don't send any mail 200 -V, --version display the version number and exit""") 201 sys.exit(exit_code)
202
203 ############################################################################### 204 205 206 -def try_or_reject(function):
207 """Try to call function or reject the upload if that fails 208 """ 209 def wrapper(directory, upload, *args, **kwargs): 210 reason = 'No exception caught. This should not happen.' 211 212 try: 213 return function(directory, upload, *args, **kwargs) 214 except (daklib.archive.ArchiveException, daklib.checks.Reject) as e: 215 reason = str(e) 216 except Exception as e: 217 reason = "There was an uncaught exception when processing your upload:\n{0}\nAny original reject reason follows below.".format(traceback.format_exc()) 218 219 try: 220 upload.rollback() 221 return real_reject(directory, upload, reason=reason) 222 except Exception as e: 223 reason = "In addition there was an exception when rejecting the package:\n{0}\nPrevious reasons:\n{1}".format(traceback.format_exc(), reason) 224 upload.rollback() 225 return real_reject(directory, upload, reason=reason, notify=False) 226 227 raise Exception('Rejecting upload failed after multiple tries. Giving up. Last reason:\n{0}'.format(reason))
228 229 return wrapper 230
231 232 -def get_processed_upload(upload):
233 changes = upload.changes 234 control = upload.changes.changes 235 236 pu = daklib.announce.ProcessedUpload() 237 238 pu.maintainer = control.get('Maintainer') 239 pu.changed_by = control.get('Changed-By') 240 pu.fingerprint = changes.primary_fingerprint 241 242 pu.suites = upload.final_suites or [] 243 pu.from_policy_suites = [] 244 245 with open(upload.changes.path, 'r') as fd: 246 pu.changes = fd.read() 247 pu.changes_filename = upload.changes.filename 248 pu.sourceful = upload.changes.sourceful 249 pu.source = control.get('Source') 250 pu.version = control.get('Version') 251 pu.architecture = control.get('Architecture') 252 pu.bugs = changes.closed_bugs 253 254 pu.program = "process-upload" 255 256 pu.warnings = upload.warnings 257 258 return pu
259
260 261 @try_or_reject 262 -def accept(directory, upload):
263 cnf = Config() 264 265 Logger.log(['ACCEPT', upload.changes.filename]) 266 print("ACCEPT") 267 268 upload.install() 269 utils.process_buildinfos(upload.directory, upload.changes.buildinfo_files, 270 upload.transaction.fs, Logger) 271 272 accepted_to_real_suite = any(suite.policy_queue is None for suite in upload.final_suites) 273 sourceful_upload = upload.changes.sourceful 274 275 control = upload.changes.changes 276 if sourceful_upload and not Options['No-Action']: 277 urgency = control.get('Urgency') 278 # As per policy 5.6.17, the urgency can be followed by a space and a 279 # comment. Extract only the urgency from the string. 280 if ' ' in urgency: 281 urgency, comment = urgency.split(' ', 1) 282 if urgency not in cnf.value_list('Urgency::Valid'): 283 urgency = cnf['Urgency::Default'] 284 UrgencyLog().log(control['Source'], control['Version'], urgency) 285 286 pu = get_processed_upload(upload) 287 daklib.announce.announce_accept(pu) 288 289 # Move .changes to done, but only for uploads that were accepted to a 290 # real suite. process-policy will handle this for uploads to queues. 291 if accepted_to_real_suite: 292 src = os.path.join(upload.directory, upload.changes.filename) 293 294 now = datetime.datetime.now() 295 donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d')) 296 dst = os.path.join(donedir, upload.changes.filename) 297 dst = utils.find_next_free(dst) 298 299 upload.transaction.fs.copy(src, dst, mode=0o644) 300 301 SummaryStats().accept_count += 1 302 SummaryStats().accept_bytes += upload.changes.bytes
303
304 305 @try_or_reject 306 -def accept_to_new(directory, upload):
307 308 Logger.log(['ACCEPT-TO-NEW', upload.changes.filename]) 309 print("ACCEPT-TO-NEW") 310 311 upload.install_to_new() 312 # TODO: tag bugs pending 313 314 pu = get_processed_upload(upload) 315 daklib.announce.announce_new(pu) 316 317 SummaryStats().accept_count += 1 318 SummaryStats().accept_bytes += upload.changes.bytes
319
320 321 @try_or_reject 322 -def reject(directory, upload, reason=None, notify=True):
323 real_reject(directory, upload, reason, notify)
324
325 326 -def real_reject(directory, upload, reason=None, notify=True):
327 # XXX: rejection itself should go to daklib.archive.ArchiveUpload 328 cnf = Config() 329 330 Logger.log(['REJECT', upload.changes.filename]) 331 print("REJECT") 332 333 fs = upload.transaction.fs 334 rejectdir = cnf['Dir::Reject'] 335 336 files = [f.filename for f in upload.changes.files.values()] 337 files.append(upload.changes.filename) 338 339 for fn in files: 340 src = os.path.join(upload.directory, fn) 341 dst = utils.find_next_free(os.path.join(rejectdir, fn)) 342 if not os.path.exists(src): 343 continue 344 fs.copy(src, dst) 345 346 if upload.reject_reasons is not None: 347 if reason is None: 348 reason = '' 349 reason = reason + '\n' + '\n'.join(upload.reject_reasons) 350 351 if reason is None: 352 reason = '(Unknown reason. Please check logs.)' 353 354 dst = utils.find_next_free(os.path.join(rejectdir, '{0}.reason'.format(upload.changes.filename))) 355 fh = fs.create(dst) 356 fh.write(reason) 357 fh.close() 358 359 if notify: 360 pu = get_processed_upload(upload) 361 daklib.announce.announce_reject(pu, reason) 362 363 SummaryStats().reject_count += 1
364
365 ############################################################################### 366 367 368 -def action(directory, upload):
369 changes = upload.changes 370 processed = True 371 372 global Logger 373 374 cnf = Config() 375 376 okay = upload.check() 377 378 try: 379 summary = changes.changes.get('Changes', '') 380 except UnicodeDecodeError as e: 381 summary = "Reading changes failed: %s" % (e) 382 # the upload checks should have detected this, but make sure this 383 # upload gets rejected in any case 384 upload.reject_reasons.append(summary) 385 386 package_info = [] 387 if okay: 388 if changes.source is not None: 389 package_info.append("source:{0}".format(changes.source.dsc['Source'])) 390 for binary in changes.binaries: 391 package_info.append("binary:{0}".format(binary.control['Package'])) 392 393 (prompt, answer) = ("", "XXX") 394 if Options["No-Action"] or Options["Automatic"]: 395 answer = 'S' 396 397 print(summary) 398 print() 399 print("\n".join(package_info)) 400 print() 401 if len(upload.warnings) > 0: 402 print("\n".join(upload.warnings)) 403 print() 404 405 if len(upload.reject_reasons) > 0: 406 print("Reason:") 407 print("\n".join(upload.reject_reasons)) 408 print() 409 410 path = os.path.join(directory, changes.filename) 411 created = os.stat(path).st_mtime 412 now = time.time() 413 too_new = (now - created < int(cnf['Dinstall::SkipTime'])) 414 415 if too_new: 416 print("SKIP (too new)") 417 prompt = "[S]kip, Quit ?" 418 else: 419 prompt = "[R]eject, Skip, Quit ?" 420 if Options["Automatic"]: 421 answer = 'R' 422 elif upload.new: 423 prompt = "[N]ew, Skip, Quit ?" 424 if Options['Automatic']: 425 answer = 'N' 426 else: 427 prompt = "[A]ccept, Skip, Quit ?" 428 if Options['Automatic']: 429 answer = 'A' 430 431 while prompt.find(answer) == -1: 432 answer = utils.input_or_exit(prompt) 433 m = re_default_answer.match(prompt) 434 if answer == "": 435 answer = m.group(1) 436 answer = answer[:1].upper() 437 438 if answer == 'R': 439 reject(directory, upload) 440 elif answer == 'A': 441 # upload.try_autobyhand must not be run with No-Action. 442 if Options['No-Action']: 443 accept(directory, upload) 444 elif upload.try_autobyhand(): 445 accept(directory, upload) 446 else: 447 print("W: redirecting to BYHAND as automatic processing failed.") 448 accept_to_new(directory, upload) 449 elif answer == 'N': 450 accept_to_new(directory, upload) 451 elif answer == 'Q': 452 sys.exit(0) 453 elif answer == 'S': 454 processed = False 455 456 if not Options['No-Action']: 457 upload.commit() 458 459 return processed
460 470
471 472 -def process_it(directory, changes, keyrings):
473 global Logger 474 475 print("\n{0}\n".format(changes.filename)) 476 Logger.log(["Processing changes file", changes.filename]) 477 478 with daklib.archive.ArchiveUpload(directory, changes, keyrings) as upload: 479 processed = action(directory, upload) 480 if processed and not Options['No-Action']: 481 session = DBConn().session() 482 history = SignatureHistory.from_signed_file(upload.changes) 483 if history.query(session) is None: 484 session.add(history) 485 session.commit() 486 session.close() 487 488 unlink_if_exists(os.path.join(directory, changes.filename)) 489 for fn in changes.files: 490 unlink_if_exists(os.path.join(directory, fn))
491
492 ############################################################################### 493 494 495 -def process_changes(changes_filenames):
496 session = DBConn().session() 497 keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority) 498 keyring_files = [k.keyring_name for k in keyrings] 499 session.close() 500 501 changes = [] 502 for fn in changes_filenames: 503 try: 504 directory, filename = os.path.split(fn) 505 c = daklib.upload.Changes(directory, filename, keyring_files) 506 changes.append([directory, c]) 507 except Exception as e: 508 try: 509 Logger.log([filename, "Error while loading changes file {0}: {1}".format(fn, e)]) 510 except Exception as e: 511 Logger.log([filename, "Error while loading changes file {0}, with additional error while printing exception: {1}".format(fn, repr(e))]) 512 513 changes.sort(key=lambda x: x[1]) 514 515 for directory, c in changes: 516 process_it(directory, c, keyring_files)
517
518 ############################################################################### 519 520 521 -def main():
522 global Options, Logger 523 524 cnf = Config() 525 summarystats = SummaryStats() 526 527 Arguments = [('a', "automatic", "Dinstall::Options::Automatic"), 528 ('h', "help", "Dinstall::Options::Help"), 529 ('n', "no-action", "Dinstall::Options::No-Action"), 530 ('p', "no-lock", "Dinstall::Options::No-Lock"), 531 ('s', "no-mail", "Dinstall::Options::No-Mail"), 532 ('d', "directory", "Dinstall::Options::Directory", "HasArg")] 533 534 for i in ["automatic", "help", "no-action", "no-lock", "no-mail", 535 "version", "directory"]: 536 key = "Dinstall::Options::%s" % i 537 if key not in cnf: 538 cnf[key] = "" 539 540 changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) 541 Options = cnf.subtree("Dinstall::Options") 542 543 if Options["Help"]: 544 usage() 545 546 # -n/--dry-run invalidates some other options which would involve things happening 547 if Options["No-Action"]: 548 Options["Automatic"] = "" 549 550 # Obtain lock if not in no-action mode and initialize the log 551 if not Options["No-Action"]: 552 lock_fd = os.open(os.path.join(cnf["Dir::Lock"], 'process-upload.lock'), os.O_RDWR | os.O_CREAT) 553 try: 554 fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) 555 except OSError as e: 556 if e.errno in (errno.EACCES, errno.EAGAIN): 557 utils.fubar("Couldn't obtain lock; assuming another 'dak process-upload' is already running.") 558 else: 559 raise 560 561 # Initialise UrgencyLog() - it will deal with the case where we don't 562 # want to log urgencies 563 urgencylog = UrgencyLog() 564 565 Logger = daklog.Logger("process-upload", Options["No-Action"]) 566 567 # If we have a directory flag, use it to find our files 568 if cnf["Dinstall::Options::Directory"] != "": 569 # Note that we clobber the list of files we were given in this case 570 # so warn if the user has done both 571 if len(changes_files) > 0: 572 utils.warn("Directory provided so ignoring files given on command line") 573 574 changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"]) 575 Logger.log(["Using changes files from directory", cnf["Dinstall::Options::Directory"], len(changes_files)]) 576 elif not len(changes_files) > 0: 577 utils.fubar("No changes files given and no directory specified") 578 else: 579 Logger.log(["Using changes files from command-line", len(changes_files)]) 580 581 process_changes(changes_files) 582 583 if summarystats.accept_count: 584 sets = "set" 585 if summarystats.accept_count > 1: 586 sets = "sets" 587 print("Installed %d package %s, %s." % (summarystats.accept_count, sets, 588 utils.size_type(int(summarystats.accept_bytes)))) 589 Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes]) 590 591 if summarystats.reject_count: 592 sets = "set" 593 if summarystats.reject_count > 1: 594 sets = "sets" 595 print("Rejected %d package %s." % (summarystats.reject_count, sets)) 596 Logger.log(["rejected", summarystats.reject_count]) 597 598 if not Options["No-Action"]: 599 urgencylog.close() 600 601 Logger.close()
602 603 ############################################################################### 604 605 606 if __name__ == '__main__': 607 main() 608