Coverage for dak/control

1"""Manipulate suite tags"""

5# This program is free software; you can redistribute it and/or modify

6# it under the terms of the GNU General Public License as published by

7# the Free Software Foundation; either version 2 of the License, or

8# (at your option) any later version.

10# This program is distributed in the hope that it will be useful,

11# but WITHOUT ANY WARRANTY; without even the implied warranty of

12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

13# GNU General Public License for more details.

15# You should have received a copy of the GNU General Public License

16# along with this program; if not, write to the Free Software

17# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

19#######################################################################################

21# 8to6Guy: "Wow, Bob, You look rough!"

22# BTAF: "Mbblpmn..."

23# BTAF <.oO>: "You moron! This is what you get for staying up all night drinking vodka and salad dressing!"

24# BTAF <.oO>: "This coffee I.V. drip is barely even keeping me awake! I need something with more kick! But what?"

25# BTAF: "OMIGOD! I OVERDOSED ON HEROIN"

26# CoWorker#n: "Give him air!!"

27# CoWorker#n+1: "We need a syringe full of adrenaline!"

28# CoWorker#n+2: "Stab him in the heart!"

29# BTAF: "*YES!*"

30# CoWorker#n+3: "Bob's been overdosing quite a bit lately..."

31# CoWorker#n+4: "Third time this week."

33# -- http://www.angryflower.com/8to6.gif

35#######################################################################################

37# Adds or removes packages from a suite. Takes the list of files

38# either from stdin or as a command line argument. Special action

39# "set", will reset the suite (!) and add all packages from scratch.

41#######################################################################################

43import functools

44import os

45import sys

46from collections.abc import Iterable

47from typing import TYPE_CHECKING, NoReturn, cast

49import apt_pkg

50from sqlalchemy import select, sql

51from sqlalchemy.engine import CursorResult

53from daklib import daklog, utils

54from daklib.archive import ArchiveTransaction

55from daklib.config import Config

56from daklib.dbconn import (

57 Architecture,

58 DBBinary,

59 DBConn,

60 DBSource,

61 Suite,

62 get_suite,

63 get_suite_by_name,

64 get_version_checks,

65)

66from daklib.queue import get_suite_version_by_package, get_suite_version_by_source

68if TYPE_CHECKING:

69 from sqlalchemy import Select

70 from sqlalchemy.orm import Session

72#######################################################################################

74Logger: daklog.Logger

76################################################################################

79def usage(exit_code=0) -> NoReturn:

80 print(

81 """Usage: dak control-suite [OPTIONS] [FILE]

82Display or alter the contents of a suite using FILE(s), or stdin.

84 -a, --add=SUITE add to SUITE

85 -h, --help show this help and exit

86 -l, --list=SUITE list the contents of SUITE

87 -r, --remove=SUITE remove from SUITE

88 -s, --set=SUITE set SUITE

89 -b, --britney generate changelog entry for britney runs"""

90 )

92 sys.exit(exit_code)

95#######################################################################################

98def get_pkg(

99 package: str, version: str, architecture: str, session: "Session"

100) -> DBBinary | DBSource | None:

101 pkg: DBSource | DBBinary | None

102 if architecture == "source":

103 source_q: "Select[tuple[DBSource]]" = (

104 select(DBSource)

105 .filter_by(source=package, version=version)

106 .join(DBSource.poolfile)

107 .limit(1)

108 )

109 pkg = session.scalars(source_q).first()

110 else:

111 binary_q: "Select[tuple[DBBinary]]" = (

112 select(DBBinary)

113 .filter_by(package=package, version=version)

114 .join(DBBinary.architecture)

115 .where(Architecture.arch_string.in_([architecture, "all"]))

116 .join(DBBinary.poolfile)

117 .limit(1)

118 )

119 pkg = session.scalars(binary_q).first()

120 if pkg is None: 120 ↛ 121line 120 didn't jump to line 121 because the condition on line 120 was never true

121 utils.warn("Could not find {0}_{1}_{2}.".format(package, version, architecture))

122 return pkg

123

124

125#######################################################################################

126

127

128def britney_changelog(

129 packages: Iterable[tuple[str, str, str]], suite: Suite, session: "Session"

130) -> None:

131

132 old: dict[str, str] = {}

133 current: dict[str, str] = {}

134 Cnf = utils.get_conf()

135

136 try:

137 q = session.execute(

138 sql.text("SELECT changelog FROM suite WHERE id = :suiteid"),

139 {"suiteid": suite.suite_id},

140 )

141 brit_file = q.scalar_one_or_none()

142 except:

143 brit_file = None

144

145 if brit_file: 145 ↛ 148line 145 didn't jump to line 148 because the condition on line 145 was always true

146 brit_file = os.path.join(Cnf["Dir::Root"], brit_file)

147 else:

148 return

149

150 q = session.execute(

151 sql.text(

152 """SELECT s.source, s.version, sa.id

153 FROM source s, src_associations sa

154 WHERE sa.suite = :suiteid

155 AND sa.source = s.id"""

156 ),

157 {"suiteid": suite.suite_id},

158 )

159

160 for p1 in q.fetchall():

161 current[p1[0]] = p1[1]

162 for p2 in packages: 162 ↛ 163line 162 didn't jump to line 163 because the loop on line 162 never started

163 if p2[2] == "source":

164 old[p2[0]] = p2[1]

165

166 new: dict[str, tuple[str, str | None]] = {}

167 for p3, value in current.items():

168 if (old_value := old.get(p3)) is not None: 168 ↛ 169line 168 didn't jump to line 169 because the condition on line 168 was never true

169 if apt_pkg.version_compare(value, old_value) > 0:

170 new[p3] = (value, old_value)

171 else:

172 new[p3] = (value, None)

173

174 params: dict[str, str | None] = {}

175 query = "SELECT source, changelog FROM changelogs WHERE"

176 for n, p3 in enumerate(new.keys()):

177 query += f" source = :source_{n} AND (:version1_{n} IS NULL OR version > :version1_{n}) AND version <= :version2_{n}"

178 query += " AND architecture LIKE '%source%' AND distribution in \

179 ('unstable', 'experimental', 'testing-proposed-updates') OR"

180 params[f"source_{n}"] = p3

181 params[f"version1_{n}"] = new[p3][1]

182 params[f"version2_{n}"] = new[p3][0]

183 query += " False ORDER BY source, version DESC"

184 q = cast(CursorResult, session.execute(sql.text(query), params))

185

186 pu = None

187 with open(brit_file, "w") as brit:

188

189 for u in q:

190 if pu and pu != u[0]: 190 ↛ 191line 190 didn't jump to line 191 because the condition on line 190 was never true

191 brit.write("\n")

192 brit.write("%s\n" % u[1])

193 pu = u[0]

194 if q.rowcount: 194 ↛ 197line 194 didn't jump to line 197 because the condition on line 194 was always true

195 brit.write("\n\n\n")

196

197 brit.writelines(

198 "REMOVED: %s %s\n" % (p, old[p])

199 for p in list(set(old.keys()).difference(current.keys()))

200 )

201

202 brit.flush()

203

204

205#######################################################################################

206

207

208class VersionCheck:

209 def __init__(self, target_suite: str, force: bool, session: "Session") -> None:

210 self.target_suite = target_suite

211 self.force = force

212 self.session = session

213

214 self.must_be_newer_than = [

215 vc.reference.suite_name

216 for vc in get_version_checks(target_suite, "MustBeNewerThan", session)

217 ]

218 self.must_be_older_than = [

219 vc.reference.suite_name

220 for vc in get_version_checks(target_suite, "MustBeOlderThan", session)

221 ]

222

223 # Must be newer than an existing version in target_suite

224 if target_suite not in self.must_be_newer_than: 224 ↛ exitline 224 didn't return from function '__init__' because the condition on line 224 was always true

225 self.must_be_newer_than.append(target_suite)

226

227 def __call__(self, package: str, architecture: str, new_version: str) -> None:

228 if architecture == "source":

229 suite_version_list = get_suite_version_by_source(package, self.session)

230 else:

231 suite_version_list = get_suite_version_by_package(

232 package, architecture, self.session

233 )

234

235 violations = False

236

237 for suite, version in suite_version_list:

238 cmp = apt_pkg.version_compare(new_version, version)

239 # for control-suite we allow equal version (for uploads, we don't)

240 if suite in self.must_be_newer_than and cmp < 0:

241 utils.warn(

242 "%s (%s): version check violated: %s targeted at %s is *not* newer than %s in %s"

243 % (

244 package,

245 architecture,

246 new_version,

247 self.target_suite,

248 version,

249 suite,

250 )

251 )

252 violations = True

253 if suite in self.must_be_older_than and cmp > 0: 253 ↛ 254line 253 didn't jump to line 254 because the condition on line 253 was never true

254 utils.warn(

255 "%s (%s): version check violated: %s targeted at %s is *not* older than %s in %s"

256 % (

257 package,

258 architecture,

259 new_version,

260 self.target_suite,

261 version,

262 suite,

263 )

264 )

265 violations = True

266

267 if violations:

268 if self.force:

269 utils.warn("Continuing anyway (forced)...")

270 else:

271 utils.fubar("Aborting. Version checks violated and not forced.")

272

273

274#######################################################################################

275

276

277def cmp_package_version(a: tuple[str, str, str], b: tuple[str, str, str]) -> int:

278 """

279 comparison function for tuples of the form (package-name, version, arch, ...)

280 """

281 res = 0

282 if a[2] == "source" and b[2] != "source":

283 res = -1

284 elif a[2] != "source" and b[2] == "source":

285 res = 1

286 if res == 0:

287 res = (a[0] > b[0]) - (a[0] < b[0])

288 if res == 0:

289 res = apt_pkg.version_compare(a[1], b[1])

290 return res

291

292

293#######################################################################################

294

295

296def copy_to_suites(

297 transaction: ArchiveTransaction, pkg: DBBinary | DBSource, suites: Iterable[Suite]

298) -> None:

299 component = pkg.poolfile.component

300 if pkg.arch_string == "source":

301 for s in suites:

302 transaction.copy_source(cast(DBSource, pkg), s, component)

303 else:

304 for s in suites:

305 transaction.copy_binary(cast(DBBinary, pkg), s, component)

306

307

308def check_propups(

309 pkg: DBBinary | DBSource,

310 psuites_current: dict[int, dict[tuple[str, str], str]],

311 propups: dict[int, set[DBBinary | DBSource]],

312) -> None:

313 key = (pkg.name, pkg.arch_string)

314 for suite_id, suite_values in psuites_current.items():

315 if (old_version := suite_values.get(key)) is None:

316 continue

317 if apt_pkg.version_compare(pkg.version, old_version) <= 0:

318 continue

319 propups[suite_id].add(pkg)

320 if pkg.arch_string != "source":

321 source = cast(DBBinary, pkg).source

322 propups[suite_id].add(source)

323

324

325def get_propup_suites(suite: Suite, session: "Session") -> list[Suite]:

326 propup_suites: list[Suite] = []

327 for rule in Config().value_list("SuiteMappings"):

328 fields = rule.split()

329 if fields[0] == "propup-version" and fields[1] == suite.suite_name:

330 propup_suites.append(get_suite_by_name(fields[2], session))

331 return propup_suites

332

333

334def set_suite(

335 file: Iterable[str],

336 suite: Suite,

337 transaction: ArchiveTransaction,

338 britney=False,

339 force=False,

340) -> None:

341 session = transaction.session

342 suite_id = suite.suite_id

343 suites = [suite] + [q.suite for q in suite.copy_queues]

344 propup_suites = get_propup_suites(suite, session)

345

346 # Our session is already in a transaction

347

348 def get_binary_q(suite_id):

349 return session.execute(

350 sql.text(

351 """SELECT b.package, b.version, a.arch_string, ba.id

352 FROM binaries b, bin_associations ba, architecture a

353 WHERE ba.suite = :suiteid

354 AND ba.bin = b.id AND b.architecture = a.id

355 ORDER BY b.version ASC"""

356 ),

357 {"suiteid": suite_id},

358 )

359

360 def get_source_q(suite_id):

361 return session.execute(

362 sql.text(

363 """SELECT s.source, s.version, 'source', sa.id

364 FROM source s, src_associations sa

365 WHERE sa.suite = :suiteid

366 AND sa.source = s.id

367 ORDER BY s.version ASC"""

368 ),

369 {"suiteid": suite_id},

370 )

371

372 # Build up a dictionary of what is currently in the suite

373 current: dict[tuple[str, str, str], int] = {}

374

375 q = get_binary_q(suite_id)

376 for i in q:

377 key = i[:3]

378 current[key] = i[3]

379

380 q = get_source_q(suite_id)

381 for i in q:

382 key = i[:3]

383 current[key] = i[3]

384

385 # Build a dictionary of what's currently in the propup suites

386 psuites_current: dict[int, dict[tuple[str, str], str]] = {}

387 propups_needed: dict[int, set[DBBinary | DBSource]] = {}

388 for p_s in propup_suites:

389 propups_needed[p_s.suite_id] = set()

390 psuites_current[p_s.suite_id] = {}

391 q = get_binary_q(p_s.suite_id)

392 for i in q:

393 key = (i[0], i[2])

394 # the query is sorted, so we only keep the newest version

395 psuites_current[p_s.suite_id][key] = i[1]

396

397 q = get_source_q(p_s.suite_id)

398 for i in q:

399 key = (i[0], i[2])

400 # the query is sorted, so we only keep the newest version

401 psuites_current[p_s.suite_id][key] = i[1]

402

403 # Build up a dictionary of what should be in the suite

404 desired: set[tuple[str, str, str]] = set()

405 for line in file:

406 split_line = line.strip().split()

407 if len(split_line) != 3: 407 ↛ 408line 407 didn't jump to line 408 because the condition on line 407 was never true

408 utils.warn(

409 "'%s' does not break into 'package version architecture'." % (line[:-1])

410 )

411 continue

412 desired.add(tuple(split_line)) # type: ignore[arg-type]

413

414 version_check = VersionCheck(suite.suite_name, force, session)

415

416 # Check to see which packages need added and add them

417 for key in sorted(desired, key=functools.cmp_to_key(cmp_package_version)):

418 if key not in current:

419 (package, version, architecture) = key

420 version_check(package, architecture, version)

421 pkg = get_pkg(package, version, architecture, session)

422 if pkg is None: 422 ↛ 423line 422 didn't jump to line 423 because the condition on line 422 was never true

423 continue

424

425 copy_to_suites(transaction, pkg, suites)

426 Logger.log(["added", suite.suite_name, " ".join(key)])

427

428 check_propups(pkg, psuites_current, propups_needed)

429

430 # Check to see which packages need removed and remove them

431 for key, pkid in current.items():

432 if key not in desired:

433 (package, version, architecture) = key

434 if architecture == "source":

435 session.execute(

436 sql.text("""DELETE FROM src_associations WHERE id = :pkid"""),

437 {"pkid": pkid},

438 )

439 else:

440 session.execute(

441 sql.text("""DELETE FROM bin_associations WHERE id = :pkid"""),

442 {"pkid": pkid},

443 )

444 Logger.log(["removed", suite.suite_name, " ".join(key), pkid])

445

446 for p_s in propup_suites:

447 for p in propups_needed[p_s.suite_id]:

448 copy_to_suites(transaction, p, [p_s])

449 info = (p.name, p.version, p.arch_string)

450 Logger.log(["propup", p_s.suite_name, " ".join(info)])

451

452 session.commit()

453

454 if britney:

455 britney_changelog(current.keys(), suite, session)

456

457

458#######################################################################################

459

460

461def process_file(

462 file: Iterable[str],

463 suite: Suite,

464 action: str,

465 transaction: ArchiveTransaction,

466 britney=False,

467 force=False,

468) -> None:

469 session = transaction.session

470

471 if action == "set":

472 set_suite(file, suite, transaction, britney, force)

473 return

474

475 suite_id = suite.suite_id

476 suites = [suite] + [q.suite for q in suite.copy_queues]

477 extra_archives = [suite.archive]

478

479 request: list[tuple[str, str, str]] = []

480

481 # Our session is already in a transaction

482 for line in file:

483 split_line = line.strip().split()

484 if len(split_line) != 3: 484 ↛ 485line 484 didn't jump to line 485 because the condition on line 484 was never true

485 utils.warn(

486 "'%s' does not break into 'package version architecture'." % (line[:-1])

487 )

488 continue

489 request.append(tuple(split_line)) # type: ignore[arg-type]

490

491 request.sort(key=functools.cmp_to_key(cmp_package_version))

492

493 version_check = VersionCheck(suite.suite_name, force, session)

494

495 for package, version, architecture in request:

496 pkg = get_pkg(package, version, architecture, session)

497 if pkg is None: 497 ↛ 498line 497 didn't jump to line 498 because the condition on line 497 was never true

498 continue

499 if architecture == "source":

500 pkid = cast(DBSource, pkg).source_id

501 else:

502 pkid = cast(DBBinary, pkg).binary_id

503

504 component = pkg.poolfile.component

505

506 # Do version checks when adding packages

507 if action == "add":

508 version_check(package, architecture, version)

509

510 if architecture == "source":

511 assert isinstance(pkg, DBSource)

512 # Find the existing association ID, if any

513 q = session.execute(

514 sql.text(

515 """SELECT id FROM src_associations

516 WHERE suite = :suiteid and source = :pkid"""

517 ),

518 {"suiteid": suite_id, "pkid": pkid},

519 )

520 ql = q.fetchall()

521 if len(ql) < 1:

522 association_id = None

523 else:

524 association_id = ql[0][0]

525

526 # Take action

527 if action == "add":

528 if association_id: 528 ↛ 529line 528 didn't jump to line 529 because the condition on line 528 was never true

529 utils.warn(

530 "'%s_%s_%s' already exists in suite %s."

531 % (package, version, architecture, suite.suite_name)

532 )

533 continue

534 else:

535 for s in suites:

536 transaction.copy_source(pkg, s, component)

537 Logger.log(

538 [

539 "added",

540 package,

541 version,

542 architecture,

543 suite.suite_name,

544 pkid,

545 ]

546 )

547

548 elif action == "remove": 548 ↛ 495line 548 didn't jump to line 495 because the condition on line 548 was always true

549 if association_id is None: 549 ↛ 550line 549 didn't jump to line 550 because the condition on line 549 was never true

550 utils.warn(

551 "'%s_%s_%s' doesn't exist in suite %s."

552 % (package, version, architecture, suite)

553 )

554 continue

555 else:

556 session.execute(

557 sql.text("""DELETE FROM src_associations WHERE id = :pkid"""),

558 {"pkid": association_id},

559 )

560 Logger.log(

561 [

562 "removed",

563 package,

564 version,

565 architecture,

566 suite.suite_name,

567 pkid,

568 ]

569 )

570 else:

571 assert isinstance(pkg, DBBinary)

572 # Find the existing associations ID, if any

573 q = session.execute(

574 sql.text(

575 """SELECT id FROM bin_associations

576 WHERE suite = :suiteid and bin = :pkid"""

577 ),

578 {"suiteid": suite_id, "pkid": pkid},

579 )

580 ql = q.fetchall()

581 if len(ql) < 1:

582 association_id = None

583 else:

584 association_id = ql[0][0]

585

586 # Take action

587 if action == "add":

588 if association_id: 588 ↛ 589line 588 didn't jump to line 589 because the condition on line 588 was never true

589 utils.warn(

590 "'%s_%s_%s' already exists in suite %s."

591 % (package, version, architecture, suite)

592 )

593 continue

594 else:

595 for s in suites:

596 transaction.copy_binary(

597 pkg, s, component, extra_archives=extra_archives

598 )

599 Logger.log(

600 [

601 "added",

602 package,

603 version,

604 architecture,

605 suite.suite_name,

606 pkid,

607 ]

608 )

609 elif action == "remove": 609 ↛ 495line 609 didn't jump to line 495 because the condition on line 609 was always true

610 if association_id is None: 610 ↛ 611line 610 didn't jump to line 611 because the condition on line 610 was never true

611 utils.warn(

612 "'%s_%s_%s' doesn't exist in suite %s."

613 % (package, version, architecture, suite)

614 )

615 continue

616 else:

617 session.execute(

618 sql.text("""DELETE FROM bin_associations WHERE id = :pkid"""),

619 {"pkid": association_id},

620 )

621 Logger.log(

622 [

623 "removed",

624 package,

625 version,

626 architecture,

627 suite.suite_name,

628 pkid,

629 ]

630 )

631

632 session.commit()

633

634

635#######################################################################################

636

637

638def get_list(suite: Suite, session: "Session") -> None:

639 suite_id = suite.suite_id

640 # List binaries

641 q = session.execute(

642 sql.text(

643 """SELECT b.package, b.version, a.arch_string

644 FROM binaries b, bin_associations ba, architecture a

645 WHERE ba.suite = :suiteid

646 AND ba.bin = b.id AND b.architecture = a.id"""

647 ),

648 {"suiteid": suite_id},

649 )

650 for i in q.fetchall():

651 print(" ".join(i))

652

653 # List source

654 q = session.execute(

655 sql.text(

656 """SELECT s.source, s.version

657 FROM source s, src_associations sa

658 WHERE sa.suite = :suiteid

659 AND sa.source = s.id"""

660 ),

661 {"suiteid": suite_id},

662 )

663 for i in q.fetchall():

664 print(" ".join(i) + " source")

665

666

667#######################################################################################

668

669

670def main() -> None:

671 global Logger

672

673 cnf = Config()

674

675 Arguments = [

676 ("a", "add", "Control-Suite::Options::Add", "HasArg"),

677 ("b", "britney", "Control-Suite::Options::Britney"),

678 ("f", "force", "Control-Suite::Options::Force"),

679 ("h", "help", "Control-Suite::Options::Help"),

680 ("l", "list", "Control-Suite::Options::List", "HasArg"),

681 ("r", "remove", "Control-Suite::Options::Remove", "HasArg"),

682 ("s", "set", "Control-Suite::Options::Set", "HasArg"),

683 ]

684

685 for i in ["add", "britney", "help", "list", "remove", "set", "version"]:

686 key = "Control-Suite::Options::%s" % i

687 if key not in cnf: 687 ↛ 685line 687 didn't jump to line 685 because the condition on line 687 was always true

688 cnf[key] = ""

689

690 try:

691 file_list = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) # type: ignore[attr-defined]

692 except SystemError as e:

693 print("%s\n" % e)

694 usage(1)

695 Options = cnf.subtree("Control-Suite::Options")

696

697 if Options["Help"]:

698 usage()

699

700 force = "Force" in Options and Options["Force"]

701

702 action = None

703

704 for i in ("add", "list", "remove", "set"):

705 if cnf["Control-Suite::Options::%s" % (i)] != "":

706 suite_name = cnf["Control-Suite::Options::%s" % (i)]

707

708 if action: 708 ↛ 709line 708 didn't jump to line 709 because the condition on line 708 was never true

709 utils.fubar("Can only perform one action at a time.")

710

711 action = i

712

713 # Need an action...

714 if action is None: 714 ↛ 715line 714 didn't jump to line 715 because the condition on line 714 was never true

715 utils.fubar("No action specified.")

716

717 britney = False

718 if action == "set" and cnf["Control-Suite::Options::Britney"]:

719 britney = True

720

721 if action == "list":

722 session = DBConn().session()

723 suite = get_suite(suite_name, session)

724 if suite is None: 724 ↛ 725line 724 didn't jump to line 725 because the condition on line 724 was never true

725 utils.fubar("Unknown suite.")

726 get_list(suite, session)

727 else:

728 Logger = daklog.Logger("control-suite")

729

730 with ArchiveTransaction() as transaction:

731 session = transaction.session

732 suite = get_suite(suite_name, session)

733 if suite is None: 733 ↛ 734line 733 didn't jump to line 734 because the condition on line 733 was never true

734 utils.fubar("Unknown suite.")

735

736 if action == "set" and not suite.allowcsset:

737 if force: 737 ↛ 744line 737 didn't jump to line 744 because the condition on line 737 was always true

738 utils.warn(

739 "Would not normally allow setting suite {0} (allowcsset is FALSE), but --force used".format(

740 suite_name

741 )

742 )

743 else:

744 utils.fubar(

745 "Will not reset suite {0} due to its database configuration (allowcsset is FALSE)".format(

746 suite_name

747 )

748 )

749

750 if file_list: 750 ↛ 751line 750 didn't jump to line 751 because the condition on line 750 was never true

751 for f in file_list:

752 process_file(open(f), suite, action, transaction, britney, force)

753 else:

754 process_file(sys.stdin, suite, action, transaction, britney, force)

755

756 Logger.close()

Coverage for dak/control_suite.py: 87%

321 statements