Package dak :: Module transitions
[hide private]
[frames] | no frames]

Source Code for Module dak.transitions

  1  #! /usr/bin/env python3 
  2   
  3  """ 
  4  Display, edit and check the release manager's transition file. 
  5   
  6  @contact: Debian FTP Master <ftpmaster@debian.org> 
  7  @copyright: 2008 Joerg Jaspert <joerg@debian.org> 
  8  @license: GNU General Public License version 2 or later 
  9  """ 
 10   
 11  # This program is free software; you can redistribute it and/or modify 
 12  # it under the terms of the GNU General Public License as published by 
 13  # the Free Software Foundation; either version 2 of the License, or 
 14  # (at your option) any later version. 
 15   
 16  # This program is distributed in the hope that it will be useful, 
 17  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 18  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 19  # GNU General Public License for more details. 
 20   
 21  # You should have received a copy of the GNU General Public License 
 22  # along with this program; if not, write to the Free Software 
 23  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
 24   
 25  ################################################################################ 
 26   
 27  # <elmo> if klecker.d.o died, I swear to god, I'm going to migrate to gentoo. 
 28   
 29  ################################################################################ 
 30   
 31  import os 
 32  import subprocess 
 33  import sys 
 34  import time 
 35  import errno 
 36  import fcntl 
 37  import tempfile 
 38  import apt_pkg 
 39   
 40  from daklib.dbconn import * 
 41  from daklib import utils 
 42  from daklib.dak_exceptions import TransitionsError 
 43  from daklib.regexes import re_broken_package 
 44  import yaml 
 45   
 46  # Globals 
 47  Cnf = None      #: Configuration, apt_pkg.Configuration 
 48  Options = None  #: Parsed CommandLine arguments 
 49   
 50  ################################################################################ 
 51   
 52  ##################################### 
 53  #### This may run within sudo !! #### 
 54  ##################################### 
 55   
 56   
57 -def init():
58 """ 59 Initialize. Sets up database connection, parses commandline arguments. 60 61 @attention: This function may run B{within sudo} 62 63 """ 64 global Cnf, Options 65 66 apt_pkg.init() 67 68 Cnf = utils.get_conf() 69 70 Arguments = [('a', "automatic", "Edit-Transitions::Options::Automatic"), 71 ('h', "help", "Edit-Transitions::Options::Help"), 72 ('e', "edit", "Edit-Transitions::Options::Edit"), 73 ('i', "import", "Edit-Transitions::Options::Import", "HasArg"), 74 ('c', "check", "Edit-Transitions::Options::Check"), 75 ('s', "sudo", "Edit-Transitions::Options::Sudo"), 76 ('n', "no-action", "Edit-Transitions::Options::No-Action")] 77 78 for i in ["automatic", "help", "no-action", "edit", "import", "check", "sudo"]: 79 key = "Edit-Transitions::Options::%s" % i 80 if key not in Cnf: 81 Cnf[key] = "" 82 83 apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) 84 85 Options = Cnf.subtree("Edit-Transitions::Options") 86 87 if Options["help"]: 88 usage() 89 90 username = utils.getusername() 91 if username != "dak": 92 print("Non-dak user: %s" % username) 93 Options["sudo"] = "y" 94 95 # Initialise DB connection 96 DBConn()
97 98 ################################################################################ 99 100
101 -def usage(exit_code=0):
102 print("""Usage: transitions [OPTION]... 103 Update and check the release managers transition file. 104 105 Options: 106 107 -h, --help show this help and exit. 108 -e, --edit edit the transitions file 109 -i, --import <file> check and import transitions from file 110 -c, --check check the transitions file, remove outdated entries 111 -S, --sudo use sudo to update transitions file 112 -a, --automatic don't prompt (only affects check). 113 -n, --no-action don't do anything (only affects check)""") 114 115 sys.exit(exit_code)
116 117 ################################################################################ 118 119 ##################################### 120 #### This may run within sudo !! #### 121 ##################################### 122 123
124 -def load_transitions(trans_file):
125 """ 126 Parse a transition yaml file and check it for validity. 127 128 @attention: This function may run B{within sudo} 129 130 @type trans_file: string 131 @param trans_file: filename to parse 132 133 @rtype: dict or None 134 @return: validated dictionary of transition entries or None 135 if validation fails, empty string if reading C{trans_file} 136 returned something else than a dict 137 138 """ 139 # Parse the yaml file 140 with open(trans_file, 'r') as sourcefile: 141 sourcecontent = sourcefile.read() 142 failure = False 143 try: 144 trans = yaml.safe_load(sourcecontent) 145 except yaml.YAMLError as exc: 146 # Someone fucked it up 147 print("ERROR: %s" % (exc)) 148 return None 149 150 # lets do further validation here 151 checkkeys = ["source", "reason", "packages", "new", "rm"] 152 153 # If we get an empty definition - we just have nothing to check, no transitions defined 154 if not isinstance(trans, dict): 155 # This can be anything. We could have no transitions defined. Or someone totally fucked up the 156 # file, adding stuff in a way we dont know or want. Then we set it empty - and simply have no 157 # transitions anymore. User will see it in the information display after he quit the editor and 158 # could fix it 159 trans = "" 160 return trans 161 162 try: 163 for test in trans: 164 t = trans[test] 165 166 # First check if we know all the keys for the transition and if they have 167 # the right type (and for the packages also if the list has the right types 168 # included, ie. not a list in list, but only str in the list) 169 for key in t: 170 if key not in checkkeys: 171 print("ERROR: Unknown key %s in transition %s" % (key, test)) 172 failure = True 173 174 if key == "packages": 175 if not isinstance(t[key], list): 176 print("ERROR: Unknown type %s for packages in transition %s." % (type(t[key]), test)) 177 failure = True 178 try: 179 for package in t["packages"]: 180 if not isinstance(package, str): 181 print("ERROR: Packages list contains invalid type %s (as %s) in transition %s" % (type(package), package, test)) 182 failure = True 183 if re_broken_package.match(package): 184 # Someone had a space too much (or not enough), we have something looking like 185 # "package1 - package2" now. 186 print("ERROR: Invalid indentation of package list in transition %s, around package(s): %s" % (test, package)) 187 failure = True 188 except TypeError: 189 # In case someone has an empty packages list 190 print("ERROR: No packages defined in transition %s" % (test)) 191 failure = True 192 continue 193 194 elif not isinstance(t[key], str): 195 if key == "new" and isinstance(t[key], int): 196 # Ok, debian native version 197 continue 198 else: 199 print("ERROR: Unknown type %s for key %s in transition %s" % (type(t[key]), key, test)) 200 failure = True 201 202 # And now the other way round - are all our keys defined? 203 for key in checkkeys: 204 if key not in t: 205 print("ERROR: Missing key %s in transition %s" % (key, test)) 206 failure = True 207 except TypeError: 208 # In case someone defined very broken things 209 print("ERROR: Unable to parse the file") 210 failure = True 211 212 if failure: 213 return None 214 215 return trans
216 217 ################################################################################ 218 219 ##################################### 220 #### This may run within sudo !! #### 221 ##################################### 222 223
224 -def lock_file(f):
225 """ 226 Lock a file 227 228 @attention: This function may run B{within sudo} 229 230 """ 231 for retry in range(10): 232 lock_fd = os.open(f, os.O_RDWR | os.O_CREAT) 233 try: 234 fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) 235 return lock_fd 236 except OSError as e: 237 if e.errno in (errno.EACCES, errno.EEXIST): 238 print("Unable to get lock for %s (try %d of 10)" % 239 (file, retry + 1)) 240 time.sleep(60) 241 else: 242 raise 243 244 utils.fubar("Couldn't obtain lock for %s." % (f))
245 246 ################################################################################ 247 248 ##################################### 249 #### This may run within sudo !! #### 250 ##################################### 251 252
253 -def write_transitions(from_trans):
254 """ 255 Update the active transitions file safely. 256 This function takes a parsed input file (which avoids invalid 257 files or files that may be be modified while the function is 258 active) and ensure the transitions file is updated atomically 259 to avoid locks. 260 261 @attention: This function may run B{within sudo} 262 263 @type from_trans: dict 264 @param from_trans: transitions dictionary, as returned by L{load_transitions} 265 266 """ 267 268 trans_file = Cnf["Dinstall::ReleaseTransitions"] 269 trans_temp = trans_file + ".tmp" 270 271 trans_lock = lock_file(trans_file) 272 temp_lock = lock_file(trans_temp) 273 274 with open(trans_temp, 'w') as destfile: 275 yaml.safe_dump(from_trans, destfile, default_flow_style=False) 276 277 os.rename(trans_temp, trans_file) 278 os.close(temp_lock) 279 os.close(trans_lock)
280 281 ################################################################################ 282 283 ########################################## 284 #### This usually runs within sudo !! #### 285 ########################################## 286 287
288 -def write_transitions_from_file(from_file):
289 """ 290 We have a file we think is valid; if we're using sudo, we invoke it 291 here, otherwise we just parse the file and call write_transitions 292 293 @attention: This function usually runs B{within sudo} 294 295 @type from_file: filename 296 @param from_file: filename of a transitions file 297 298 """ 299 300 # Lets check if from_file is in the directory we expect it to be in 301 if not os.path.abspath(from_file).startswith(Cnf["Dir::TempPath"]): 302 print("Will not accept transitions file outside of %s" % (Cnf["Dir::TempPath"])) 303 sys.exit(3) 304 305 if Options["sudo"]: 306 subprocess.check_call( 307 ["/usr/bin/sudo", "-u", "dak", "-H", 308 "/usr/local/bin/dak", "transitions", "--import", from_file]) 309 else: 310 trans = load_transitions(from_file) 311 if trans is None: 312 raise TransitionsError("Unparsable transitions file %s" % (file)) 313 write_transitions(trans)
314 315 ################################################################################ 316 317
318 -def temp_transitions_file(transitions):
319 """ 320 Open a temporary file and dump the current transitions into it, so users 321 can edit them. 322 323 @type transitions: dict 324 @param transitions: current defined transitions 325 326 @rtype: string 327 @return: path of newly created tempfile 328 329 @note: NB: file is unlinked by caller, but fd is never actually closed. 330 We need the chmod, as the file is (most possibly) copied from a 331 sudo-ed script and would be unreadable if it has default mkstemp mode 332 """ 333 334 (fd, path) = tempfile.mkstemp("", "transitions", Cnf["Dir::TempPath"]) 335 os.chmod(path, 0o644) 336 with open(path, "w") as f: 337 yaml.safe_dump(transitions, f, default_flow_style=False) 338 return path
339 340 ################################################################################ 341 342
343 -def edit_transitions():
344 """ Edit the defined transitions. """ 345 trans_file = Cnf["Dinstall::ReleaseTransitions"] 346 edit_file = temp_transitions_file(load_transitions(trans_file)) 347 348 editor = os.environ.get("EDITOR", "vi") 349 350 while True: 351 result = os.system("%s %s" % (editor, edit_file)) 352 if result != 0: 353 os.unlink(edit_file) 354 utils.fubar("%s invocation failed for %s, not removing tempfile." % (editor, edit_file)) 355 356 # Now try to load the new file 357 test = load_transitions(edit_file) 358 359 if test is None: 360 # Edit is broken 361 print("Edit was unparsable.") 362 prompt = "[E]dit again, Drop changes?" 363 default = "E" 364 else: 365 print("Edit looks okay.\n") 366 print("The following transitions are defined:") 367 print("------------------------------------------------------------------------") 368 transition_info(test) 369 370 prompt = "[S]ave, Edit again, Drop changes?" 371 default = "S" 372 373 answer = "XXX" 374 while prompt.find(answer) == -1: 375 answer = utils.input_or_exit(prompt) 376 if answer == "": 377 answer = default 378 answer = answer[:1].upper() 379 380 if answer == 'E': 381 continue 382 elif answer == 'D': 383 os.unlink(edit_file) 384 print("OK, discarding changes") 385 sys.exit(0) 386 elif answer == 'S': 387 # Ready to save 388 break 389 else: 390 print("You pressed something you shouldn't have :(") 391 sys.exit(1) 392 393 # We seem to be done and also have a working file. Copy over. 394 write_transitions_from_file(edit_file) 395 os.unlink(edit_file) 396 397 print("Transitions file updated.")
398 399 ################################################################################ 400 401
402 -def check_transitions(transitions):
403 """ 404 Check if the defined transitions still apply and remove those that no longer do. 405 @note: Asks the user for confirmation first unless -a has been set. 406 407 """ 408 global Cnf 409 410 to_dump = 0 411 to_remove = [] 412 info = {} 413 414 session = DBConn().session() 415 416 # Now look through all defined transitions 417 for trans in transitions: 418 t = transitions[trans] 419 source = t["source"] 420 expected = t["new"] 421 422 # Will be an empty list if nothing is in testing. 423 sourceobj = get_source_in_suite(source, "testing", session) 424 425 info[trans] = get_info(trans, source, expected, t["rm"], t["reason"], t["packages"]) 426 print(info[trans]) 427 428 if sourceobj is None: 429 # No package in testing 430 print("Transition source %s not in testing, transition still ongoing." % (source)) 431 else: 432 current = sourceobj.version 433 compare = apt_pkg.version_compare(current, expected) 434 if compare < 0: 435 # This is still valid, the current version in database is older than 436 # the new version we wait for 437 print("This transition is still ongoing, we currently have version %s" % (current)) 438 else: 439 print("REMOVE: This transition is over, the target package reached testing. REMOVE") 440 print("%s wanted version: %s, has %s" % (source, expected, current)) 441 to_remove.append(trans) 442 to_dump = 1 443 print("-------------------------------------------------------------------------") 444 445 if to_dump: 446 prompt = "Removing: " 447 for remove in to_remove: 448 prompt += remove 449 prompt += "," 450 451 prompt += " Commit Changes? (y/N)" 452 answer = "" 453 454 if Options["no-action"]: 455 answer = "n" 456 elif Options["automatic"]: 457 answer = "y" 458 else: 459 answer = utils.input_or_exit(prompt).lower() 460 461 if answer == "": 462 answer = "n" 463 464 if answer == 'n': 465 print("Not committing changes") 466 sys.exit(0) 467 elif answer == 'y': 468 print("Committing") 469 subst = {} 470 subst['__SUBJECT__'] = "Transitions completed: " + ", ".join(sorted(to_remove)) 471 subst['__TRANSITION_MESSAGE__'] = "The following transitions were removed:\n" 472 for remove in sorted(to_remove): 473 subst['__TRANSITION_MESSAGE__'] += info[remove] + '\n' 474 del transitions[remove] 475 476 # If we have a mail address configured for transitions, 477 # send a notification 478 subst['__TRANSITION_EMAIL__'] = Cnf.get("Transitions::Notifications", "") 479 if subst['__TRANSITION_EMAIL__'] != "": 480 print("Sending notification to %s" % subst['__TRANSITION_EMAIL__']) 481 subst['__DAK_ADDRESS__'] = Cnf["Dinstall::MyEmailAddress"] 482 subst['__BCC__'] = 'X-DAK: dak transitions' 483 if "Dinstall::Bcc" in Cnf: 484 subst["__BCC__"] += '\nBcc: %s' % Cnf["Dinstall::Bcc"] 485 message = utils.TemplateSubst(subst, 486 os.path.join(Cnf["Dir::Templates"], 'transition.removed')) 487 utils.send_mail(message) 488 489 edit_file = temp_transitions_file(transitions) 490 write_transitions_from_file(edit_file) 491 492 print("Done") 493 else: 494 print("WTF are you typing?") 495 sys.exit(0)
496 497 ################################################################################ 498 499
500 -def get_info(trans, source, expected, rm, reason, packages):
501 """ 502 Print information about a single transition. 503 504 @type trans: string 505 @param trans: Transition name 506 507 @type source: string 508 @param source: Source package 509 510 @type expected: string 511 @param expected: Expected version in testing 512 513 @type rm: string 514 @param rm: Responsible RM 515 516 @type reason: string 517 @param reason: Reason 518 519 @type packages: list 520 @param packages: list of blocked packages 521 522 """ 523 return """Looking at transition: %s 524 Source: %s 525 New Version: %s 526 Responsible: %s 527 Description: %s 528 Blocked Packages (total: %d): %s 529 """ % (trans, source, expected, rm, reason, len(packages), ", ".join(packages))
530 531 ################################################################################ 532 533
534 -def transition_info(transitions):
535 """ 536 Print information about all defined transitions. 537 Calls L{get_info} for every transition and then tells user if the transition is 538 still ongoing or if the expected version already hit testing. 539 540 @type transitions: dict 541 @param transitions: defined transitions 542 """ 543 544 session = DBConn().session() 545 546 for trans in transitions: 547 t = transitions[trans] 548 source = t["source"] 549 expected = t["new"] 550 551 # Will be None if nothing is in testing. 552 sourceobj = get_source_in_suite(source, "testing", session) 553 554 print(get_info(trans, source, expected, t["rm"], t["reason"], t["packages"])) 555 556 if sourceobj is None: 557 # No package in testing 558 print("Transition source %s not in testing, transition still ongoing." % (source)) 559 else: 560 compare = apt_pkg.version_compare(sourceobj.version, expected) 561 print("Apt compare says: %s" % (compare)) 562 if compare < 0: 563 # This is still valid, the current version in database is older than 564 # the new version we wait for 565 print("This transition is still ongoing, we currently have version %s" % (sourceobj.version)) 566 else: 567 print("This transition is over, the target package reached testing, should be removed") 568 print("%s wanted version: %s, has %s" % (source, expected, sourceobj.version)) 569 print("-------------------------------------------------------------------------")
570 571 ################################################################################ 572 573
574 -def main():
575 """ 576 Prepare the work to be done, do basic checks. 577 578 @attention: This function may run B{within sudo} 579 580 """ 581 global Cnf 582 583 ##################################### 584 #### This can run within sudo !! #### 585 ##################################### 586 init() 587 588 # Check if there is a file defined (and existant) 589 transpath = Cnf.get("Dinstall::ReleaseTransitions", "") 590 if transpath == "": 591 utils.warn("Dinstall::ReleaseTransitions not defined") 592 sys.exit(1) 593 if not os.path.exists(transpath): 594 utils.warn("ReleaseTransitions file, %s, not found." % 595 (Cnf["Dinstall::ReleaseTransitions"])) 596 sys.exit(1) 597 # Also check if our temp directory is defined and existant 598 temppath = Cnf.get("Dir::TempPath", "") 599 if temppath == "": 600 utils.warn("Dir::TempPath not defined") 601 sys.exit(1) 602 if not os.path.exists(temppath): 603 utils.warn("Temporary path %s not found." % 604 (Cnf["Dir::TempPath"])) 605 sys.exit(1) 606 607 if Options["import"]: 608 try: 609 write_transitions_from_file(Options["import"]) 610 except TransitionsError as m: 611 print(m) 612 sys.exit(2) 613 sys.exit(0) 614 ############################################## 615 #### Up to here it can run within sudo !! #### 616 ############################################## 617 618 # Parse the yaml file 619 transitions = load_transitions(transpath) 620 if transitions is None: 621 # Something very broken with the transitions, exit 622 utils.warn("Could not parse existing transitions file. Aborting.") 623 sys.exit(2) 624 625 if Options["edit"]: 626 # Let's edit the transitions file 627 edit_transitions() 628 elif Options["check"]: 629 # Check and remove outdated transitions 630 check_transitions(transitions) 631 else: 632 # Output information about the currently defined transitions. 633 print("Currently defined transitions:") 634 transition_info(transitions) 635 636 sys.exit(0)
637 638 ################################################################################ 639 640 641 if __name__ == '__main__': 642 main() 643