Coverage for dak/transitions.py: 12%

1"""

2Display, edit and check the release manager's transition file.

4@contact: Debian FTP Master <ftpmaster@debian.org>

5@copyright: 2008 Joerg Jaspert <joerg@debian.org>

6@license: GNU General Public License version 2 or later

7"""

9# This program is free software; you can redistribute it and/or modify

10# it under the terms of the GNU General Public License as published by

11# the Free Software Foundation; either version 2 of the License, or

12# (at your option) any later version.

14# This program is distributed in the hope that it will be useful,

15# but WITHOUT ANY WARRANTY; without even the implied warranty of

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

17# GNU General Public License for more details.

19# You should have received a copy of the GNU General Public License

20# along with this program; if not, write to the Free Software

21# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

23################################################################################

25# <elmo> if klecker.d.o died, I swear to god, I'm going to migrate to gentoo.

27################################################################################

29import errno

30import fcntl

31import os

32import subprocess

33import sys

34import tempfile

35import time

36from collections.abc import Sequence

38import apt_pkg

39import yaml

41from daklib import utils

42from daklib.dak_exceptions import TransitionsError

43from daklib.dbconn import DBConn, get_source_in_suite

44from daklib.regexes import re_broken_package

46# Globals

47Cnf: apt_pkg.Configuration #: Configuration

48Options: apt_pkg.Configuration #: Parsed CommandLine arguments

50################################################################################

52#####################################

53#### This may run within sudo !! ####

54#####################################

57def init():

58 """

59 Initialize. Sets up database connection, parses commandline arguments.

61 .. warning::

63 This function may run **within sudo**

65 """

66 global Cnf, Options

68 apt_pkg.init()

70 Cnf = utils.get_conf()

72 Arguments = [

73 ("a", "automatic", "Edit-Transitions::Options::Automatic"),

74 ("h", "help", "Edit-Transitions::Options::Help"),

75 ("e", "edit", "Edit-Transitions::Options::Edit"),

76 ("i", "import", "Edit-Transitions::Options::Import", "HasArg"),

77 ("c", "check", "Edit-Transitions::Options::Check"),

78 ("s", "sudo", "Edit-Transitions::Options::Sudo"),

79 ("n", "no-action", "Edit-Transitions::Options::No-Action"),

80 ]

82 for i in ["automatic", "help", "no-action", "edit", "import", "check", "sudo"]:

83 key = "Edit-Transitions::Options::%s" % i

84 if key not in Cnf: 84 ↛ 82line 84 didn't jump to line 82 because the condition on line 84 was always true

85 Cnf[key] = "" # type: ignore[index]

87 apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) # type: ignore[attr-defined]

89 Options = Cnf.subtree("Edit-Transitions::Options") # type: ignore[attr-defined]

91 if Options["help"]: 91 ↛ 94line 91 didn't jump to line 94 because the condition on line 91 was always true

92 usage()

94 username = utils.getusername()

95 if username != "dak":

96 print("Non-dak user: %s" % username)

97 Options["sudo"] = "y" # type: ignore[index]

99 # Initialise DB connection

100 DBConn()

101

102

103################################################################################

104

105

106def usage(exit_code=0):

107 print(

108 """Usage: transitions [OPTION]...

109Update and check the release managers transition file.

110

111Options:

112

113 -h, --help show this help and exit.

114 -e, --edit edit the transitions file

115 -i, --import <file> check and import transitions from file

116 -c, --check check the transitions file, remove outdated entries

117 -S, --sudo use sudo to update transitions file

118 -a, --automatic don't prompt (only affects check).

119 -n, --no-action don't do anything (only affects check)"""

120 )

121

122 sys.exit(exit_code)

123

124

125################################################################################

126

127#####################################

128#### This may run within sudo !! ####

129#####################################

130

131

132def load_transitions(trans_file: str) -> dict | None:

133 """

134 Parse a transition yaml file and check it for validity.

135

136 .. warning::

137

138 This function may run **within sudo**

139

140 :param trans_file: filename to parse

141 :return: validated dictionary of transition entries or None

142 if validation fails, empty string if reading `trans_file`

143 returned something else than a dict

144

145 """

146 # Parse the yaml file

147 with open(trans_file, "r") as sourcefile:

148 sourcecontent = sourcefile.read()

149 failure = False

150 try:

151 trans = yaml.safe_load(sourcecontent)

152 except yaml.YAMLError as exc:

153 # Someone fucked it up

154 print("ERROR: %s" % (exc))

155 return None

156

157 # lets do further validation here

158 checkkeys = ["source", "reason", "packages", "new", "rm"]

159

160 # If we get an empty definition - we just have nothing to check, no transitions defined

161 if not isinstance(trans, dict):

162 # This can be anything. We could have no transitions defined. Or someone totally fucked up the

163 # file, adding stuff in a way we dont know or want. Then we set it empty - and simply have no

164 # transitions anymore. User will see it in the information display after he quit the editor and

165 # could fix it

166 return None

167

168 try:

169 for test in trans:

170 t = trans[test]

171

172 # First check if we know all the keys for the transition and if they have

173 # the right type (and for the packages also if the list has the right types

174 # included, ie. not a list in list, but only str in the list)

175 for key in t:

176 if key not in checkkeys:

177 print("ERROR: Unknown key %s in transition %s" % (key, test))

178 failure = True

179

180 if key == "packages":

181 if not isinstance(t[key], list):

182 print(

183 "ERROR: Unknown type %s for packages in transition %s."

184 % (type(t[key]), test)

185 )

186 failure = True

187 try:

188 for package in t["packages"]:

189 if not isinstance(package, str):

190 print(

191 "ERROR: Packages list contains invalid type %s (as %s) in transition %s"

192 % (type(package), package, test)

193 )

194 failure = True

195 if re_broken_package.match(package):

196 # Someone had a space too much (or not enough), we have something looking like

197 # "package1 - package2" now.

198 print(

199 "ERROR: Invalid indentation of package list in transition %s, around package(s): %s"

200 % (test, package)

201 )

202 failure = True

203 except TypeError:

204 # In case someone has an empty packages list

205 print("ERROR: No packages defined in transition %s" % (test))

206 failure = True

207 continue

208

209 elif not isinstance(t[key], str):

210 if key == "new" and isinstance(t[key], int):

211 # Ok, debian native version

212 continue

213 else:

214 print(

215 "ERROR: Unknown type %s for key %s in transition %s"

216 % (type(t[key]), key, test)

217 )

218 failure = True

219

220 # And now the other way round - are all our keys defined?

221 for key in checkkeys:

222 if key not in t:

223 print("ERROR: Missing key %s in transition %s" % (key, test))

224 failure = True

225 except TypeError:

226 # In case someone defined very broken things

227 print("ERROR: Unable to parse the file")

228 failure = True

229

230 if failure:

231 return None

232

233 return trans

234

235

236################################################################################

237

238#####################################

239#### This may run within sudo !! ####

240#####################################

241

242

243def lock_file(f: str) -> int:

244 """

245 Lock a file

246

247 .. warning::

248

249 This function may run **within sudo**

250 """

251 for retry in range(10):

252 lock_fd = os.open(f, os.O_RDWR | os.O_CREAT)

253 try:

254 fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)

255 return lock_fd

256 except OSError as e:

257 if e.errno in (errno.EACCES, errno.EEXIST):

258 print("Unable to get lock for %s (try %d of 10)" % (f, retry + 1))

259 time.sleep(60)

260 else:

261 raise

262

263 utils.fubar("Couldn't obtain lock for %s." % (f))

264

265

266################################################################################

267

268#####################################

269#### This may run within sudo !! ####

270#####################################

271

272

273def write_transitions(from_trans: dict) -> None:

274 """

275 Update the active transitions file safely.

276 This function takes a parsed input file (which avoids invalid

277 files or files that may be be modified while the function is

278 active) and ensure the transitions file is updated atomically

279 to avoid locks.

280

281 .. warning::

282

283 This function may run **within sudo**

284

285 :param from_trans: transitions dictionary, as returned by :func:`load_transitions`

286 """

287

288 trans_file = Cnf["Dinstall::ReleaseTransitions"]

289 trans_temp = trans_file + ".tmp"

290

291 trans_lock = lock_file(trans_file)

292 temp_lock = lock_file(trans_temp)

293

294 with open(trans_temp, "w") as destfile:

295 yaml.safe_dump(from_trans, destfile, default_flow_style=False)

296

297 os.rename(trans_temp, trans_file)

298 os.close(temp_lock)

299 os.close(trans_lock)

300

301

302################################################################################

303

304##########################################

305#### This usually runs within sudo !! ####

306##########################################

307

308

309def write_transitions_from_file(from_file: str) -> None:

310 """

311 We have a file we think is valid; if we're using sudo, we invoke it

312 here, otherwise we just parse the file and call write_transitions

313

314 .. warning::

315

316 This function usually runs **within sudo**

317

318 :param from_file: filename of a transitions file

319 """

320

321 # Lets check if from_file is in the directory we expect it to be in

322 if not os.path.abspath(from_file).startswith(Cnf["Dir::TempPath"]):

323 print("Will not accept transitions file outside of %s" % (Cnf["Dir::TempPath"]))

324 sys.exit(3)

325

326 if Options["sudo"]:

327 subprocess.check_call(

328 [

329 "/usr/bin/sudo",

330 "-u",

331 "dak",

332 "-H",

333 "/usr/local/bin/dak",

334 "transitions",

335 "--import",

336 from_file,

337 ]

338 )

339 else:

340 trans = load_transitions(from_file)

341 if trans is None:

342 raise TransitionsError("Unparsable transitions file %s" % (from_file))

343 write_transitions(trans)

344

345

346################################################################################

347

348

349def temp_transitions_file(transitions: dict) -> str:

350 """

351 Open a temporary file and dump the current transitions into it, so users

352 can edit them.

353

354 :param transitions: current defined transitions

355 :return: path of newly created tempfile

356

357 .. note::

358

359 file is unlinked by caller, but fd is never actually closed.

360 We need the chmod, as the file is (most possibly) copied from a

361 sudo-ed script and would be unreadable if it has default mkstemp mode

362 """

363

364 (fd, path) = tempfile.mkstemp("", "transitions", Cnf["Dir::TempPath"])

365 os.chmod(path, 0o644)

366 with open(path, "w") as f:

367 yaml.safe_dump(transitions, f, default_flow_style=False)

368 return path

369

370

371################################################################################

372

373

374def edit_transitions():

375 """Edit the defined transitions."""

376 trans_file = Cnf["Dinstall::ReleaseTransitions"]

377 edit_file = temp_transitions_file(load_transitions(trans_file) or {})

378

379 editor = os.environ.get("EDITOR", "vi")

380

381 while True:

382 result = os.system("%s %s" % (editor, edit_file))

383 if result != 0:

384 os.unlink(edit_file)

385 utils.fubar(

386 "%s invocation failed for %s, not removing tempfile."

387 % (editor, edit_file)

388 )

389

390 # Now try to load the new file

391 test = load_transitions(edit_file)

392

393 if test is None:

394 # Edit is broken

395 print("Edit was unparsable.")

396 prompt = "[E]dit again, Drop changes?"

397 default = "E"

398 else:

399 print("Edit looks okay.\n")

400 print("The following transitions are defined:")

401 print(

402 "------------------------------------------------------------------------"

403 )

404 transition_info(test)

405

406 prompt = "[S]ave, Edit again, Drop changes?"

407 default = "S"

408

409 answer = "XXX"

410 while prompt.find(answer) == -1:

411 answer = utils.input_or_exit(prompt)

412 if answer == "":

413 answer = default

414 answer = answer[:1].upper()

415

416 if answer == "E":

417 continue

418 elif answer == "D":

419 os.unlink(edit_file)

420 print("OK, discarding changes")

421 sys.exit(0)

422 elif answer == "S":

423 # Ready to save

424 break

425 else:

426 print("You pressed something you shouldn't have :(")

427 sys.exit(1)

428

429 # We seem to be done and also have a working file. Copy over.

430 write_transitions_from_file(edit_file)

431 os.unlink(edit_file)

432

433 print("Transitions file updated.")

434

435

436################################################################################

437

438

439def check_transitions(transitions) -> None:

440 """

441 Check if the defined transitions still apply and remove those that no longer do.

442 @note: Asks the user for confirmation first unless -a has been set.

443

444 """

445 global Cnf

446

447 to_dump = 0

448 to_remove = []

449 info = {}

450

451 session = DBConn().session()

452

453 # Now look through all defined transitions

454 for trans in transitions:

455 t = transitions[trans]

456 source = t["source"]

457 expected = t["new"]

458

459 # Will be an empty list if nothing is in testing.

460 sourceobj = get_source_in_suite(source, "testing", session)

461

462 info[trans] = get_info(

463 trans, source, expected, t["rm"], t["reason"], t["packages"]

464 )

465 print(info[trans])

466

467 if sourceobj is None:

468 # No package in testing

469 print(

470 "Transition source %s not in testing, transition still ongoing."

471 % (source)

472 )

473 else:

474 current = sourceobj.version

475 compare = apt_pkg.version_compare(current, expected)

476 if compare < 0:

477 # This is still valid, the current version in database is older than

478 # the new version we wait for

479 print(

480 "This transition is still ongoing, we currently have version %s"

481 % (current)

482 )

483 else:

484 print(

485 "REMOVE: This transition is over, the target package reached testing. REMOVE"

486 )

487 print("%s wanted version: %s, has %s" % (source, expected, current))

488 to_remove.append(trans)

489 to_dump = 1

490 print(

491 "-------------------------------------------------------------------------"

492 )

493

494 if to_dump:

495 prompt = "Removing: "

496 for remove in to_remove:

497 prompt += remove

498 prompt += ","

499

500 prompt += " Commit Changes? (y/N)"

501 answer = ""

502

503 if Options["no-action"]:

504 answer = "n"

505 elif Options["automatic"]:

506 answer = "y"

507 else:

508 answer = utils.input_or_exit(prompt).lower()

509

510 if answer == "":

511 answer = "n"

512

513 if answer == "n":

514 print("Not committing changes")

515 sys.exit(0)

516 elif answer == "y":

517 print("Committing")

518 subst = {}

519 subst["__SUBJECT__"] = "Transitions completed: " + ", ".join(

520 sorted(to_remove)

521 )

522 subst["__TRANSITION_MESSAGE__"] = (

523 "The following transitions were removed:\n"

524 )

525 for remove in sorted(to_remove):

526 subst["__TRANSITION_MESSAGE__"] += info[remove] + "\n"

527 del transitions[remove]

528

529 # If we have a mail address configured for transitions,

530 # send a notification

531 subst["__TRANSITION_EMAIL__"] = Cnf.get("Transitions::Notifications", "")

532 if subst["__TRANSITION_EMAIL__"] != "":

533 print("Sending notification to %s" % subst["__TRANSITION_EMAIL__"])

534 subst["__DAK_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]

535 subst["__BCC__"] = "X-DAK: dak transitions"

536 if "Dinstall::Bcc" in Cnf:

537 subst["__BCC__"] += "\nBcc: %s" % Cnf["Dinstall::Bcc"]

538 message = utils.TemplateSubst(

539 subst, os.path.join(Cnf["Dir::Templates"], "transition.removed")

540 )

541 utils.send_mail(message)

542

543 edit_file = temp_transitions_file(transitions)

544 write_transitions_from_file(edit_file)

545

546 print("Done")

547 else:

548 print("WTF are you typing?")

549 sys.exit(0)

550

551

552################################################################################

553

554

555def get_info(

556 trans: str,

557 source: str,

558 expected: str,

559 rm: str,

560 reason: str,

561 packages: Sequence[str],

562) -> str:

563 """

564 Print information about a single transition.

565

566 :param trans: Transition name

567 :param source: Source package

568 :param expected: Expected version in testing

569 :param rm: Responsible release manager (RM)

570 :param reason: Reason

571 :param packages: list of blocked packages

572 """

573 return """Looking at transition: %s

574Source: %s

575New Version: %s

576Responsible: %s

577Description: %s

578Blocked Packages (total: %d): %s

579""" % (

580 trans,

581 source,

582 expected,

583 rm,

584 reason,

585 len(packages),

586 ", ".join(packages),

587 )

588

589

590################################################################################

591

592

593def transition_info(transitions) -> None:

594 """

595 Print information about all defined transitions.

596 Calls :func:`get_info` for every transition and then tells user if the transition is

597 still ongoing or if the expected version already hit testing.

598

599 :param transitions: defined transitions

600 """

601

602 session = DBConn().session()

603

604 for trans in transitions:

605 t = transitions[trans]

606 source = t["source"]

607 expected = t["new"]

608

609 # Will be None if nothing is in testing.

610 sourceobj = get_source_in_suite(source, "testing", session)

611

612 print(get_info(trans, source, expected, t["rm"], t["reason"], t["packages"]))

613

614 if sourceobj is None:

615 # No package in testing

616 print(

617 "Transition source %s not in testing, transition still ongoing."

618 % (source)

619 )

620 else:

621 compare = apt_pkg.version_compare(sourceobj.version, expected)

622 print("Apt compare says: %s" % (compare))

623 if compare < 0:

624 # This is still valid, the current version in database is older than

625 # the new version we wait for

626 print(

627 "This transition is still ongoing, we currently have version %s"

628 % (sourceobj.version)

629 )

630 else:

631 print(

632 "This transition is over, the target package reached testing, should be removed"

633 )

634 print(

635 "%s wanted version: %s, has %s"

636 % (source, expected, sourceobj.version)

637 )

638 print(

639 "-------------------------------------------------------------------------"

640 )

641

642

643################################################################################

644

645

646def main():

647 """

648 Prepare the work to be done, do basic checks.

649

650 .. warning::

651

652 This function may run **within sudo**

653 """

654 global Cnf

655

656 #####################################

657 #### This can run within sudo !! ####

658 #####################################

659 init()

660

661 # Check if there is a file defined (and existant)

662 transpath = Cnf.get("Dinstall::ReleaseTransitions", "")

663 if transpath == "":

664 utils.warn("Dinstall::ReleaseTransitions not defined")

665 sys.exit(1)

666 if not os.path.exists(transpath):

667 utils.warn(

668 "ReleaseTransitions file, %s, not found."

669 % (Cnf["Dinstall::ReleaseTransitions"])

670 )

671 sys.exit(1)

672 # Also check if our temp directory is defined and existant

673 temppath = Cnf.get("Dir::TempPath", "")

674 if temppath == "":

675 utils.warn("Dir::TempPath not defined")

676 sys.exit(1)

677 if not os.path.exists(temppath):

678 utils.warn("Temporary path %s not found." % (Cnf["Dir::TempPath"]))

679 sys.exit(1)

680

681 if Options["import"]:

682 try:

683 write_transitions_from_file(Options["import"])

684 except TransitionsError as m:

685 print(m)

686 sys.exit(2)

687 sys.exit(0)

688 ##############################################

689 #### Up to here it can run within sudo !! ####

690 ##############################################

691

692 # Parse the yaml file

693 transitions = load_transitions(transpath)

694 if transitions is None:

695 # Something very broken with the transitions, exit

696 utils.warn("Could not parse existing transitions file. Aborting.")

697 sys.exit(2)

698

699 if Options["edit"]:

700 # Let's edit the transitions file

701 edit_transitions()

702 elif Options["check"]:

703 # Check and remove outdated transitions

704 check_transitions(transitions)

705 else:

706 # Output information about the currently defined transitions.

707 print("Currently defined transitions:")

708 transition_info(transitions)

709

710 sys.exit(0)