1#! /usr/bin/env python3
3"""
4Display, edit and check the release manager's transition file.
6@contact: Debian FTP Master <ftpmaster@debian.org>
7@copyright: 2008 Joerg Jaspert <joerg@debian.org>
8@license: GNU General Public License version 2 or later
9"""
11# This program is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation; either version 2 of the License, or
14# (at your option) any later version.
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
21# You should have received a copy of the GNU General Public License
22# along with this program; if not, write to the Free Software
23# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25################################################################################
27# <elmo> if klecker.d.o died, I swear to god, I'm going to migrate to gentoo.
29################################################################################
31import os
32import subprocess
33import sys
34import time
35import errno
36import fcntl
37import tempfile
38import apt_pkg
39from collections.abc import Iterable
40from typing import Optional
42from daklib.dbconn import *
43from daklib import utils
44from daklib.dak_exceptions import TransitionsError
45from daklib.regexes import re_broken_package
46import yaml
48# Globals
49Cnf = None #: Configuration, apt_pkg.Configuration
50Options = None #: Parsed CommandLine arguments
52################################################################################
54#####################################
55#### This may run within sudo !! ####
56#####################################
59def init():
60 """
61 Initialize. Sets up database connection, parses commandline arguments.
63 .. warning::
65 This function may run **within sudo**
67 """
68 global Cnf, Options
70 apt_pkg.init()
72 Cnf = utils.get_conf()
74 Arguments = [('a', "automatic", "Edit-Transitions::Options::Automatic"),
75 ('h', "help", "Edit-Transitions::Options::Help"),
76 ('e', "edit", "Edit-Transitions::Options::Edit"),
77 ('i', "import", "Edit-Transitions::Options::Import", "HasArg"),
78 ('c', "check", "Edit-Transitions::Options::Check"),
79 ('s', "sudo", "Edit-Transitions::Options::Sudo"),
80 ('n', "no-action", "Edit-Transitions::Options::No-Action")]
82 for i in ["automatic", "help", "no-action", "edit", "import", "check", "sudo"]:
83 key = "Edit-Transitions::Options::%s" % i
84 if key not in Cnf: 84 ↛ 82line 84 didn't jump to line 82, because the condition on line 84 was never false
85 Cnf[key] = ""
87 apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
89 Options = Cnf.subtree("Edit-Transitions::Options")
91 if Options["help"]: 91 ↛ 94line 91 didn't jump to line 94, because the condition on line 91 was never false
92 usage()
94 username = utils.getusername()
95 if username != "dak":
96 print("Non-dak user: %s" % username)
97 Options["sudo"] = "y"
99 # Initialise DB connection
100 DBConn()
102################################################################################
105def usage(exit_code=0):
106 print("""Usage: transitions [OPTION]...
107Update and check the release managers transition file.
109Options:
111 -h, --help show this help and exit.
112 -e, --edit edit the transitions file
113 -i, --import <file> check and import transitions from file
114 -c, --check check the transitions file, remove outdated entries
115 -S, --sudo use sudo to update transitions file
116 -a, --automatic don't prompt (only affects check).
117 -n, --no-action don't do anything (only affects check)""")
119 sys.exit(exit_code)
121################################################################################
123#####################################
124#### This may run within sudo !! ####
125#####################################
128def load_transitions(trans_file: str) -> Optional[dict]:
129 """
130 Parse a transition yaml file and check it for validity.
132 .. warning::
134 This function may run **within sudo**
136 :param trans_file: filename to parse
137 :return: validated dictionary of transition entries or None
138 if validation fails, empty string if reading `trans_file`
139 returned something else than a dict
141 """
142 # Parse the yaml file
143 with open(trans_file, 'r') as sourcefile:
144 sourcecontent = sourcefile.read()
145 failure = False
146 try:
147 trans = yaml.safe_load(sourcecontent)
148 except yaml.YAMLError as exc:
149 # Someone fucked it up
150 print("ERROR: %s" % (exc))
151 return None
153 # lets do further validation here
154 checkkeys = ["source", "reason", "packages", "new", "rm"]
156 # If we get an empty definition - we just have nothing to check, no transitions defined
157 if not isinstance(trans, dict):
158 # This can be anything. We could have no transitions defined. Or someone totally fucked up the
159 # file, adding stuff in a way we dont know or want. Then we set it empty - and simply have no
160 # transitions anymore. User will see it in the information display after he quit the editor and
161 # could fix it
162 trans = ""
163 return trans
165 try:
166 for test in trans:
167 t = trans[test]
169 # First check if we know all the keys for the transition and if they have
170 # the right type (and for the packages also if the list has the right types
171 # included, ie. not a list in list, but only str in the list)
172 for key in t:
173 if key not in checkkeys:
174 print("ERROR: Unknown key %s in transition %s" % (key, test))
175 failure = True
177 if key == "packages":
178 if not isinstance(t[key], list):
179 print("ERROR: Unknown type %s for packages in transition %s." % (type(t[key]), test))
180 failure = True
181 try:
182 for package in t["packages"]:
183 if not isinstance(package, str):
184 print("ERROR: Packages list contains invalid type %s (as %s) in transition %s" % (type(package), package, test))
185 failure = True
186 if re_broken_package.match(package):
187 # Someone had a space too much (or not enough), we have something looking like
188 # "package1 - package2" now.
189 print("ERROR: Invalid indentation of package list in transition %s, around package(s): %s" % (test, package))
190 failure = True
191 except TypeError:
192 # In case someone has an empty packages list
193 print("ERROR: No packages defined in transition %s" % (test))
194 failure = True
195 continue
197 elif not isinstance(t[key], str):
198 if key == "new" and isinstance(t[key], int):
199 # Ok, debian native version
200 continue
201 else:
202 print("ERROR: Unknown type %s for key %s in transition %s" % (type(t[key]), key, test))
203 failure = True
205 # And now the other way round - are all our keys defined?
206 for key in checkkeys:
207 if key not in t:
208 print("ERROR: Missing key %s in transition %s" % (key, test))
209 failure = True
210 except TypeError:
211 # In case someone defined very broken things
212 print("ERROR: Unable to parse the file")
213 failure = True
215 if failure:
216 return None
218 return trans
220################################################################################
222#####################################
223#### This may run within sudo !! ####
224#####################################
227def lock_file(f):
228 """
229 Lock a file
231 .. warning::
233 This function may run **within sudo**
234 """
235 for retry in range(10):
236 lock_fd = os.open(f, os.O_RDWR | os.O_CREAT)
237 try:
238 fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
239 return lock_fd
240 except OSError as e:
241 if e.errno in (errno.EACCES, errno.EEXIST):
242 print("Unable to get lock for %s (try %d of 10)" %
243 (file, retry + 1))
244 time.sleep(60)
245 else:
246 raise
248 utils.fubar("Couldn't obtain lock for %s." % (f))
250################################################################################
252#####################################
253#### This may run within sudo !! ####
254#####################################
257def write_transitions(from_trans: dict) -> None:
258 """
259 Update the active transitions file safely.
260 This function takes a parsed input file (which avoids invalid
261 files or files that may be be modified while the function is
262 active) and ensure the transitions file is updated atomically
263 to avoid locks.
265 .. warning::
267 This function may run **within sudo**
269 :param from_trans: transitions dictionary, as returned by :func:`load_transitions`
270 """
272 trans_file = Cnf["Dinstall::ReleaseTransitions"]
273 trans_temp = trans_file + ".tmp"
275 trans_lock = lock_file(trans_file)
276 temp_lock = lock_file(trans_temp)
278 with open(trans_temp, 'w') as destfile:
279 yaml.safe_dump(from_trans, destfile, default_flow_style=False)
281 os.rename(trans_temp, trans_file)
282 os.close(temp_lock)
283 os.close(trans_lock)
285################################################################################
287##########################################
288#### This usually runs within sudo !! ####
289##########################################
292def write_transitions_from_file(from_file: str) -> None:
293 """
294 We have a file we think is valid; if we're using sudo, we invoke it
295 here, otherwise we just parse the file and call write_transitions
297 .. warning::
299 This function usually runs **within sudo**
301 :param from_file: filename of a transitions file
302 """
304 # Lets check if from_file is in the directory we expect it to be in
305 if not os.path.abspath(from_file).startswith(Cnf["Dir::TempPath"]):
306 print("Will not accept transitions file outside of %s" % (Cnf["Dir::TempPath"]))
307 sys.exit(3)
309 if Options["sudo"]:
310 subprocess.check_call(
311 ["/usr/bin/sudo", "-u", "dak", "-H",
312 "/usr/local/bin/dak", "transitions", "--import", from_file])
313 else:
314 trans = load_transitions(from_file)
315 if trans is None:
316 raise TransitionsError("Unparsable transitions file %s" % (from_file))
317 write_transitions(trans)
319################################################################################
322def temp_transitions_file(transitions: dict) -> str:
323 """
324 Open a temporary file and dump the current transitions into it, so users
325 can edit them.
327 :param transitions: current defined transitions
328 :return: path of newly created tempfile
330 .. note::
332 file is unlinked by caller, but fd is never actually closed.
333 We need the chmod, as the file is (most possibly) copied from a
334 sudo-ed script and would be unreadable if it has default mkstemp mode
335 """
337 (fd, path) = tempfile.mkstemp("", "transitions", Cnf["Dir::TempPath"])
338 os.chmod(path, 0o644)
339 with open(path, "w") as f:
340 yaml.safe_dump(transitions, f, default_flow_style=False)
341 return path
343################################################################################
346def edit_transitions():
347 """ Edit the defined transitions. """
348 trans_file = Cnf["Dinstall::ReleaseTransitions"]
349 edit_file = temp_transitions_file(load_transitions(trans_file))
351 editor = os.environ.get("EDITOR", "vi")
353 while True:
354 result = os.system("%s %s" % (editor, edit_file))
355 if result != 0:
356 os.unlink(edit_file)
357 utils.fubar("%s invocation failed for %s, not removing tempfile." % (editor, edit_file))
359 # Now try to load the new file
360 test = load_transitions(edit_file)
362 if test is None:
363 # Edit is broken
364 print("Edit was unparsable.")
365 prompt = "[E]dit again, Drop changes?"
366 default = "E"
367 else:
368 print("Edit looks okay.\n")
369 print("The following transitions are defined:")
370 print("------------------------------------------------------------------------")
371 transition_info(test)
373 prompt = "[S]ave, Edit again, Drop changes?"
374 default = "S"
376 answer = "XXX"
377 while prompt.find(answer) == -1:
378 answer = utils.input_or_exit(prompt)
379 if answer == "":
380 answer = default
381 answer = answer[:1].upper()
383 if answer == 'E':
384 continue
385 elif answer == 'D':
386 os.unlink(edit_file)
387 print("OK, discarding changes")
388 sys.exit(0)
389 elif answer == 'S':
390 # Ready to save
391 break
392 else:
393 print("You pressed something you shouldn't have :(")
394 sys.exit(1)
396 # We seem to be done and also have a working file. Copy over.
397 write_transitions_from_file(edit_file)
398 os.unlink(edit_file)
400 print("Transitions file updated.")
402################################################################################
405def check_transitions(transitions):
406 """
407 Check if the defined transitions still apply and remove those that no longer do.
408 @note: Asks the user for confirmation first unless -a has been set.
410 """
411 global Cnf
413 to_dump = 0
414 to_remove = []
415 info = {}
417 session = DBConn().session()
419 # Now look through all defined transitions
420 for trans in transitions:
421 t = transitions[trans]
422 source = t["source"]
423 expected = t["new"]
425 # Will be an empty list if nothing is in testing.
426 sourceobj = get_source_in_suite(source, "testing", session)
428 info[trans] = get_info(trans, source, expected, t["rm"], t["reason"], t["packages"])
429 print(info[trans])
431 if sourceobj is None:
432 # No package in testing
433 print("Transition source %s not in testing, transition still ongoing." % (source))
434 else:
435 current = sourceobj.version
436 compare = apt_pkg.version_compare(current, expected)
437 if compare < 0:
438 # This is still valid, the current version in database is older than
439 # the new version we wait for
440 print("This transition is still ongoing, we currently have version %s" % (current))
441 else:
442 print("REMOVE: This transition is over, the target package reached testing. REMOVE")
443 print("%s wanted version: %s, has %s" % (source, expected, current))
444 to_remove.append(trans)
445 to_dump = 1
446 print("-------------------------------------------------------------------------")
448 if to_dump:
449 prompt = "Removing: "
450 for remove in to_remove:
451 prompt += remove
452 prompt += ","
454 prompt += " Commit Changes? (y/N)"
455 answer = ""
457 if Options["no-action"]:
458 answer = "n"
459 elif Options["automatic"]:
460 answer = "y"
461 else:
462 answer = utils.input_or_exit(prompt).lower()
464 if answer == "":
465 answer = "n"
467 if answer == 'n':
468 print("Not committing changes")
469 sys.exit(0)
470 elif answer == 'y':
471 print("Committing")
472 subst = {}
473 subst['__SUBJECT__'] = "Transitions completed: " + ", ".join(sorted(to_remove))
474 subst['__TRANSITION_MESSAGE__'] = "The following transitions were removed:\n"
475 for remove in sorted(to_remove):
476 subst['__TRANSITION_MESSAGE__'] += info[remove] + '\n'
477 del transitions[remove]
479 # If we have a mail address configured for transitions,
480 # send a notification
481 subst['__TRANSITION_EMAIL__'] = Cnf.get("Transitions::Notifications", "")
482 if subst['__TRANSITION_EMAIL__'] != "":
483 print("Sending notification to %s" % subst['__TRANSITION_EMAIL__'])
484 subst['__DAK_ADDRESS__'] = Cnf["Dinstall::MyEmailAddress"]
485 subst['__BCC__'] = 'X-DAK: dak transitions'
486 if "Dinstall::Bcc" in Cnf:
487 subst["__BCC__"] += '\nBcc: %s' % Cnf["Dinstall::Bcc"]
488 message = utils.TemplateSubst(subst,
489 os.path.join(Cnf["Dir::Templates"], 'transition.removed'))
490 utils.send_mail(message)
492 edit_file = temp_transitions_file(transitions)
493 write_transitions_from_file(edit_file)
495 print("Done")
496 else:
497 print("WTF are you typing?")
498 sys.exit(0)
500################################################################################
503def get_info(trans: str, source: str, expected: str, rm: str, reason: str, packages: Iterable[str]) -> str:
504 """
505 Print information about a single transition.
507 :param trans: Transition name
508 :param source: Source package
509 :param expected: Expected version in testing
510 :param rm: Responsible release manager (RM)
511 :param reason: Reason
512 :param packages: list of blocked packages
513 """
514 return """Looking at transition: %s
515Source: %s
516New Version: %s
517Responsible: %s
518Description: %s
519Blocked Packages (total: %d): %s
520""" % (trans, source, expected, rm, reason, len(packages), ", ".join(packages))
522################################################################################
525def transition_info(transitions: dict):
526 """
527 Print information about all defined transitions.
528 Calls :func:`get_info` for every transition and then tells user if the transition is
529 still ongoing or if the expected version already hit testing.
531 :param transitions: defined transitions
532 """
534 session = DBConn().session()
536 for trans in transitions:
537 t = transitions[trans]
538 source = t["source"]
539 expected = t["new"]
541 # Will be None if nothing is in testing.
542 sourceobj = get_source_in_suite(source, "testing", session)
544 print(get_info(trans, source, expected, t["rm"], t["reason"], t["packages"]))
546 if sourceobj is None:
547 # No package in testing
548 print("Transition source %s not in testing, transition still ongoing." % (source))
549 else:
550 compare = apt_pkg.version_compare(sourceobj.version, expected)
551 print("Apt compare says: %s" % (compare))
552 if compare < 0:
553 # This is still valid, the current version in database is older than
554 # the new version we wait for
555 print("This transition is still ongoing, we currently have version %s" % (sourceobj.version))
556 else:
557 print("This transition is over, the target package reached testing, should be removed")
558 print("%s wanted version: %s, has %s" % (source, expected, sourceobj.version))
559 print("-------------------------------------------------------------------------")
561################################################################################
564def main():
565 """
566 Prepare the work to be done, do basic checks.
568 .. warning::
570 This function may run **within sudo**
571 """
572 global Cnf
574 #####################################
575 #### This can run within sudo !! ####
576 #####################################
577 init()
579 # Check if there is a file defined (and existant)
580 transpath = Cnf.get("Dinstall::ReleaseTransitions", "")
581 if transpath == "":
582 utils.warn("Dinstall::ReleaseTransitions not defined")
583 sys.exit(1)
584 if not os.path.exists(transpath):
585 utils.warn("ReleaseTransitions file, %s, not found." %
586 (Cnf["Dinstall::ReleaseTransitions"]))
587 sys.exit(1)
588 # Also check if our temp directory is defined and existant
589 temppath = Cnf.get("Dir::TempPath", "")
590 if temppath == "":
591 utils.warn("Dir::TempPath not defined")
592 sys.exit(1)
593 if not os.path.exists(temppath):
594 utils.warn("Temporary path %s not found." %
595 (Cnf["Dir::TempPath"]))
596 sys.exit(1)
598 if Options["import"]:
599 try:
600 write_transitions_from_file(Options["import"])
601 except TransitionsError as m:
602 print(m)
603 sys.exit(2)
604 sys.exit(0)
605 ##############################################
606 #### Up to here it can run within sudo !! ####
607 ##############################################
609 # Parse the yaml file
610 transitions = load_transitions(transpath)
611 if transitions is None:
612 # Something very broken with the transitions, exit
613 utils.warn("Could not parse existing transitions file. Aborting.")
614 sys.exit(2)
616 if Options["edit"]:
617 # Let's edit the transitions file
618 edit_transitions()
619 elif Options["check"]:
620 # Check and remove outdated transitions
621 check_transitions(transitions)
622 else:
623 # Output information about the currently defined transitions.
624 print("Currently defined transitions:")
625 transition_info(transitions)
627 sys.exit(0)
629################################################################################
632if __name__ == '__main__':
633 main()