Coverage for dak/control_suite.py: 86%
320 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-01-04 16:18 +0000
1#! /usr/bin/env python3
3"""Manipulate suite tags"""
4# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
6# This program is free software; you can redistribute it and/or modify
7# it under the terms of the GNU General Public License as published by
8# the Free Software Foundation; either version 2 of the License, or
9# (at your option) any later version.
11# This program is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14# GNU General Public License for more details.
16# You should have received a copy of the GNU General Public License
17# along with this program; if not, write to the Free Software
18# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20#######################################################################################
22# 8to6Guy: "Wow, Bob, You look rough!"
23# BTAF: "Mbblpmn..."
24# BTAF <.oO>: "You moron! This is what you get for staying up all night drinking vodka and salad dressing!"
25# BTAF <.oO>: "This coffee I.V. drip is barely even keeping me awake! I need something with more kick! But what?"
26# BTAF: "OMIGOD! I OVERDOSED ON HEROIN"
27# CoWorker#n: "Give him air!!"
28# CoWorker#n+1: "We need a syringe full of adrenaline!"
29# CoWorker#n+2: "Stab him in the heart!"
30# BTAF: "*YES!*"
31# CoWorker#n+3: "Bob's been overdosing quite a bit lately..."
32# CoWorker#n+4: "Third time this week."
34# -- http://www.angryflower.com/8to6.gif
36#######################################################################################
38# Adds or removes packages from a suite. Takes the list of files
39# either from stdin or as a command line argument. Special action
40# "set", will reset the suite (!) and add all packages from scratch.
42#######################################################################################
44import functools
45import os
46import sys
47from collections.abc import Iterable
48from typing import TYPE_CHECKING, NoReturn, cast
50import apt_pkg
51from sqlalchemy import sql
52from sqlalchemy.engine import CursorResult
54from daklib import daklog, utils
55from daklib.archive import ArchiveTransaction
56from daklib.config import Config
57from daklib.dbconn import (
58 Architecture,
59 DBBinary,
60 DBConn,
61 DBSource,
62 Suite,
63 get_suite,
64 get_version_checks,
65)
66from daklib.queue import get_suite_version_by_package, get_suite_version_by_source
68if TYPE_CHECKING:
69 from sqlalchemy.orm import Query, Session
71#######################################################################################
73Logger: daklog.Logger
75################################################################################
78def usage(exit_code=0) -> NoReturn:
79 print(
80 """Usage: dak control-suite [OPTIONS] [FILE]
81Display or alter the contents of a suite using FILE(s), or stdin.
83 -a, --add=SUITE add to SUITE
84 -h, --help show this help and exit
85 -l, --list=SUITE list the contents of SUITE
86 -r, --remove=SUITE remove from SUITE
87 -s, --set=SUITE set SUITE
88 -b, --britney generate changelog entry for britney runs"""
89 )
91 sys.exit(exit_code)
94#######################################################################################
97def get_pkg(
98 package: str, version: str, architecture: str, session: "Session"
99) -> DBBinary | DBSource | None:
100 q: "Query[DBBinary] | Query[DBSource]"
101 if architecture == "source":
102 q = (
103 session.query(DBSource)
104 .filter_by(source=package, version=version)
105 .join(DBSource.poolfile)
106 )
107 else:
108 q = (
109 session.query(DBBinary)
110 .filter_by(package=package, version=version)
111 .join(DBBinary.architecture)
112 .filter(Architecture.arch_string.in_([architecture, "all"]))
113 .join(DBBinary.poolfile)
114 )
116 pkg = q.first()
117 if pkg is None: 117 ↛ 118line 117 didn't jump to line 118 because the condition on line 117 was never true
118 utils.warn("Could not find {0}_{1}_{2}.".format(package, version, architecture))
119 return pkg
122#######################################################################################
125def britney_changelog(
126 packages: Iterable[tuple[str, str, str]], suite: Suite, session: "Session"
127) -> None:
129 old: dict[str, str] = {}
130 current: dict[str, str] = {}
131 Cnf = utils.get_conf()
133 try:
134 q = session.execute(
135 sql.text("SELECT changelog FROM suite WHERE id = :suiteid"),
136 {"suiteid": suite.suite_id},
137 )
138 brit_file = q.scalar_one_or_none()
139 except:
140 brit_file = None
142 if brit_file: 142 ↛ 145line 142 didn't jump to line 145 because the condition on line 142 was always true
143 brit_file = os.path.join(Cnf["Dir::Root"], brit_file)
144 else:
145 return
147 q = session.execute(
148 sql.text(
149 """SELECT s.source, s.version, sa.id
150 FROM source s, src_associations sa
151 WHERE sa.suite = :suiteid
152 AND sa.source = s.id"""
153 ),
154 {"suiteid": suite.suite_id},
155 )
157 for p1 in q.fetchall():
158 current[p1[0]] = p1[1]
159 for p2 in packages: 159 ↛ 160line 159 didn't jump to line 160 because the loop on line 159 never started
160 if p2[2] == "source":
161 old[p2[0]] = p2[1]
163 new: dict[str, tuple[str, str | None]] = {}
164 for p3 in current.keys():
165 if p3 in old: 165 ↛ 166line 165 didn't jump to line 166 because the condition on line 165 was never true
166 if apt_pkg.version_compare(current[p3], old[p3]) > 0:
167 new[p3] = (current[p3], old[p3])
168 else:
169 new[p3] = (current[p3], None)
171 params: dict[str, str | None] = {}
172 query = "SELECT source, changelog FROM changelogs WHERE"
173 for n, p3 in enumerate(new.keys()):
174 query += f" source = :source_{n} AND (:version1_{n} IS NULL OR version > :version1_{n}) AND version <= :version2_{n}"
175 query += " AND architecture LIKE '%source%' AND distribution in \
176 ('unstable', 'experimental', 'testing-proposed-updates') OR"
177 params[f"source_{n}"] = p3
178 params[f"version1_{n}"] = new[p3][1]
179 params[f"version2_{n}"] = new[p3][0]
180 query += " False ORDER BY source, version DESC"
181 q = cast(CursorResult, session.execute(sql.text(query), params))
183 pu = None
184 with open(brit_file, "w") as brit:
186 for u in q:
187 if pu and pu != u[0]: 187 ↛ 188line 187 didn't jump to line 188 because the condition on line 187 was never true
188 brit.write("\n")
189 brit.write("%s\n" % u[1])
190 pu = u[0]
191 if q.rowcount: 191 ↛ 194line 191 didn't jump to line 194 because the condition on line 191 was always true
192 brit.write("\n\n\n")
194 for p in list(set(old.keys()).difference(current.keys())): 194 ↛ 195line 194 didn't jump to line 195 because the loop on line 194 never started
195 brit.write("REMOVED: %s %s\n" % (p, old[p]))
197 brit.flush()
200#######################################################################################
203class VersionCheck:
204 def __init__(self, target_suite: str, force: bool, session: "Session") -> None:
205 self.target_suite = target_suite
206 self.force = force
207 self.session = session
209 self.must_be_newer_than = [
210 vc.reference.suite_name
211 for vc in get_version_checks(target_suite, "MustBeNewerThan", session)
212 ]
213 self.must_be_older_than = [
214 vc.reference.suite_name
215 for vc in get_version_checks(target_suite, "MustBeOlderThan", session)
216 ]
218 # Must be newer than an existing version in target_suite
219 if target_suite not in self.must_be_newer_than: 219 ↛ exitline 219 didn't return from function '__init__' because the condition on line 219 was always true
220 self.must_be_newer_than.append(target_suite)
222 def __call__(self, package: str, architecture: str, new_version: str) -> None:
223 if architecture == "source":
224 suite_version_list = get_suite_version_by_source(package, self.session)
225 else:
226 suite_version_list = get_suite_version_by_package(
227 package, architecture, self.session
228 )
230 violations = False
232 for suite, version in suite_version_list:
233 cmp = apt_pkg.version_compare(new_version, version)
234 # for control-suite we allow equal version (for uploads, we don't)
235 if suite in self.must_be_newer_than and cmp < 0:
236 utils.warn(
237 "%s (%s): version check violated: %s targeted at %s is *not* newer than %s in %s"
238 % (
239 package,
240 architecture,
241 new_version,
242 self.target_suite,
243 version,
244 suite,
245 )
246 )
247 violations = True
248 if suite in self.must_be_older_than and cmp > 0: 248 ↛ 249line 248 didn't jump to line 249 because the condition on line 248 was never true
249 utils.warn(
250 "%s (%s): version check violated: %s targeted at %s is *not* older than %s in %s"
251 % (
252 package,
253 architecture,
254 new_version,
255 self.target_suite,
256 version,
257 suite,
258 )
259 )
260 violations = True
262 if violations:
263 if self.force:
264 utils.warn("Continuing anyway (forced)...")
265 else:
266 utils.fubar("Aborting. Version checks violated and not forced.")
269#######################################################################################
272def cmp_package_version(a: tuple[str, str, str], b: tuple[str, str, str]) -> int:
273 """
274 comparison function for tuples of the form (package-name, version, arch, ...)
275 """
276 res = 0
277 if a[2] == "source" and b[2] != "source":
278 res = -1
279 elif a[2] != "source" and b[2] == "source":
280 res = 1
281 if res == 0:
282 res = (a[0] > b[0]) - (a[0] < b[0])
283 if res == 0:
284 res = apt_pkg.version_compare(a[1], b[1])
285 return res
288#######################################################################################
291def copy_to_suites(
292 transaction: ArchiveTransaction, pkg: DBBinary | DBSource, suites: Iterable[Suite]
293) -> None:
294 component = pkg.poolfile.component
295 if pkg.arch_string == "source":
296 for s in suites:
297 transaction.copy_source(cast(DBSource, pkg), s, component)
298 else:
299 for s in suites:
300 transaction.copy_binary(cast(DBBinary, pkg), s, component)
303def check_propups(
304 pkg: DBBinary | DBSource,
305 psuites_current: dict[int, dict[tuple[str, str], str]],
306 propups: dict[int, set[DBBinary | DBSource]],
307) -> None:
308 key = (pkg.name, pkg.arch_string)
309 for suite_id in psuites_current:
310 if key in psuites_current[suite_id]:
311 old_version = psuites_current[suite_id][key]
312 if apt_pkg.version_compare(pkg.version, old_version) > 0:
313 propups[suite_id].add(pkg)
314 if pkg.arch_string != "source":
315 source = cast(DBBinary, pkg).source
316 propups[suite_id].add(source)
319def get_propup_suites(suite: Suite, session: "Session") -> list[Suite]:
320 propup_suites: list[Suite] = []
321 for rule in Config().value_list("SuiteMappings"):
322 fields = rule.split()
323 if fields[0] == "propup-version" and fields[1] == suite.suite_name:
324 propup_suites.append(
325 session.query(Suite).filter_by(suite_name=fields[2]).one()
326 )
327 return propup_suites
330def set_suite(
331 file: Iterable[str],
332 suite: Suite,
333 transaction: ArchiveTransaction,
334 britney=False,
335 force=False,
336) -> None:
337 session = transaction.session
338 suite_id = suite.suite_id
339 suites = [suite] + [q.suite for q in suite.copy_queues]
340 propup_suites = get_propup_suites(suite, session)
342 # Our session is already in a transaction
344 def get_binary_q(suite_id):
345 return session.execute(
346 sql.text(
347 """SELECT b.package, b.version, a.arch_string, ba.id
348 FROM binaries b, bin_associations ba, architecture a
349 WHERE ba.suite = :suiteid
350 AND ba.bin = b.id AND b.architecture = a.id
351 ORDER BY b.version ASC"""
352 ),
353 {"suiteid": suite_id},
354 )
356 def get_source_q(suite_id):
357 return session.execute(
358 sql.text(
359 """SELECT s.source, s.version, 'source', sa.id
360 FROM source s, src_associations sa
361 WHERE sa.suite = :suiteid
362 AND sa.source = s.id
363 ORDER BY s.version ASC"""
364 ),
365 {"suiteid": suite_id},
366 )
368 # Build up a dictionary of what is currently in the suite
369 current: dict[tuple[str, str, str], int] = {}
371 q = get_binary_q(suite_id)
372 for i in q:
373 key = i[:3]
374 current[key] = i[3]
376 q = get_source_q(suite_id)
377 for i in q:
378 key = i[:3]
379 current[key] = i[3]
381 # Build a dictionary of what's currently in the propup suites
382 psuites_current: dict[int, dict[tuple[str, str], str]] = {}
383 propups_needed: dict[int, set[DBBinary | DBSource]] = {}
384 for p_s in propup_suites:
385 propups_needed[p_s.suite_id] = set()
386 psuites_current[p_s.suite_id] = {}
387 q = get_binary_q(p_s.suite_id)
388 for i in q:
389 key = (i[0], i[2])
390 # the query is sorted, so we only keep the newest version
391 psuites_current[p_s.suite_id][key] = i[1]
393 q = get_source_q(p_s.suite_id)
394 for i in q:
395 key = (i[0], i[2])
396 # the query is sorted, so we only keep the newest version
397 psuites_current[p_s.suite_id][key] = i[1]
399 # Build up a dictionary of what should be in the suite
400 desired: set[tuple[str, str, str]] = set()
401 for line in file:
402 split_line = line.strip().split()
403 if len(split_line) != 3: 403 ↛ 404line 403 didn't jump to line 404 because the condition on line 403 was never true
404 utils.warn(
405 "'%s' does not break into 'package version architecture'." % (line[:-1])
406 )
407 continue
408 desired.add(tuple(split_line)) # type: ignore[arg-type]
410 version_check = VersionCheck(suite.suite_name, force, session)
412 # Check to see which packages need added and add them
413 for key in sorted(desired, key=functools.cmp_to_key(cmp_package_version)):
414 if key not in current:
415 (package, version, architecture) = key
416 version_check(package, architecture, version)
417 pkg = get_pkg(package, version, architecture, session)
418 if pkg is None: 418 ↛ 419line 418 didn't jump to line 419 because the condition on line 418 was never true
419 continue
421 copy_to_suites(transaction, pkg, suites)
422 Logger.log(["added", suite.suite_name, " ".join(key)])
424 check_propups(pkg, psuites_current, propups_needed)
426 # Check to see which packages need removed and remove them
427 for key, pkid in current.items():
428 if key not in desired:
429 (package, version, architecture) = key
430 if architecture == "source":
431 session.execute(
432 sql.text("""DELETE FROM src_associations WHERE id = :pkid"""),
433 {"pkid": pkid},
434 )
435 else:
436 session.execute(
437 sql.text("""DELETE FROM bin_associations WHERE id = :pkid"""),
438 {"pkid": pkid},
439 )
440 Logger.log(["removed", suite.suite_name, " ".join(key), pkid])
442 for p_s in propup_suites:
443 for p in propups_needed[p_s.suite_id]:
444 copy_to_suites(transaction, p, [p_s])
445 info = (p.name, p.version, p.arch_string)
446 Logger.log(["propup", p_s.suite_name, " ".join(info)])
448 session.commit()
450 if britney:
451 britney_changelog(current.keys(), suite, session)
454#######################################################################################
457def process_file(
458 file: Iterable[str],
459 suite: Suite,
460 action: str,
461 transaction: ArchiveTransaction,
462 britney=False,
463 force=False,
464) -> None:
465 session = transaction.session
467 if action == "set":
468 set_suite(file, suite, transaction, britney, force)
469 return
471 suite_id = suite.suite_id
472 suites = [suite] + [q.suite for q in suite.copy_queues]
473 extra_archives = [suite.archive]
475 request: list[tuple[str, str, str]] = []
477 # Our session is already in a transaction
478 for line in file:
479 split_line = line.strip().split()
480 if len(split_line) != 3: 480 ↛ 481line 480 didn't jump to line 481 because the condition on line 480 was never true
481 utils.warn(
482 "'%s' does not break into 'package version architecture'." % (line[:-1])
483 )
484 continue
485 request.append(tuple(split_line)) # type: ignore[arg-type]
487 request.sort(key=functools.cmp_to_key(cmp_package_version))
489 version_check = VersionCheck(suite.suite_name, force, session)
491 for package, version, architecture in request:
492 pkg = get_pkg(package, version, architecture, session)
493 if pkg is None: 493 ↛ 494line 493 didn't jump to line 494 because the condition on line 493 was never true
494 continue
495 if architecture == "source":
496 pkid = cast(DBSource, pkg).source_id
497 else:
498 pkid = cast(DBBinary, pkg).binary_id
500 component = pkg.poolfile.component
502 # Do version checks when adding packages
503 if action == "add":
504 version_check(package, architecture, version)
506 if architecture == "source":
507 assert isinstance(pkg, DBSource)
508 # Find the existing association ID, if any
509 q = session.execute(
510 sql.text(
511 """SELECT id FROM src_associations
512 WHERE suite = :suiteid and source = :pkid"""
513 ),
514 {"suiteid": suite_id, "pkid": pkid},
515 )
516 ql = q.fetchall()
517 if len(ql) < 1:
518 association_id = None
519 else:
520 association_id = ql[0][0]
522 # Take action
523 if action == "add":
524 if association_id: 524 ↛ 525line 524 didn't jump to line 525 because the condition on line 524 was never true
525 utils.warn(
526 "'%s_%s_%s' already exists in suite %s."
527 % (package, version, architecture, suite.suite_name)
528 )
529 continue
530 else:
531 for s in suites:
532 transaction.copy_source(pkg, s, component)
533 Logger.log(
534 [
535 "added",
536 package,
537 version,
538 architecture,
539 suite.suite_name,
540 pkid,
541 ]
542 )
544 elif action == "remove": 544 ↛ 491line 544 didn't jump to line 491 because the condition on line 544 was always true
545 if association_id is None: 545 ↛ 546line 545 didn't jump to line 546 because the condition on line 545 was never true
546 utils.warn(
547 "'%s_%s_%s' doesn't exist in suite %s."
548 % (package, version, architecture, suite)
549 )
550 continue
551 else:
552 session.execute(
553 sql.text("""DELETE FROM src_associations WHERE id = :pkid"""),
554 {"pkid": association_id},
555 )
556 Logger.log(
557 [
558 "removed",
559 package,
560 version,
561 architecture,
562 suite.suite_name,
563 pkid,
564 ]
565 )
566 else:
567 assert isinstance(pkg, DBBinary)
568 # Find the existing associations ID, if any
569 q = session.execute(
570 sql.text(
571 """SELECT id FROM bin_associations
572 WHERE suite = :suiteid and bin = :pkid"""
573 ),
574 {"suiteid": suite_id, "pkid": pkid},
575 )
576 ql = q.fetchall()
577 if len(ql) < 1:
578 association_id = None
579 else:
580 association_id = ql[0][0]
582 # Take action
583 if action == "add":
584 if association_id: 584 ↛ 585line 584 didn't jump to line 585 because the condition on line 584 was never true
585 utils.warn(
586 "'%s_%s_%s' already exists in suite %s."
587 % (package, version, architecture, suite)
588 )
589 continue
590 else:
591 for s in suites:
592 transaction.copy_binary(
593 pkg, s, component, extra_archives=extra_archives
594 )
595 Logger.log(
596 [
597 "added",
598 package,
599 version,
600 architecture,
601 suite.suite_name,
602 pkid,
603 ]
604 )
605 elif action == "remove": 605 ↛ 491line 605 didn't jump to line 491 because the condition on line 605 was always true
606 if association_id is None: 606 ↛ 607line 606 didn't jump to line 607 because the condition on line 606 was never true
607 utils.warn(
608 "'%s_%s_%s' doesn't exist in suite %s."
609 % (package, version, architecture, suite)
610 )
611 continue
612 else:
613 session.execute(
614 sql.text("""DELETE FROM bin_associations WHERE id = :pkid"""),
615 {"pkid": association_id},
616 )
617 Logger.log(
618 [
619 "removed",
620 package,
621 version,
622 architecture,
623 suite.suite_name,
624 pkid,
625 ]
626 )
628 session.commit()
631#######################################################################################
634def get_list(suite: Suite, session: "Session") -> None:
635 suite_id = suite.suite_id
636 # List binaries
637 q = session.execute(
638 sql.text(
639 """SELECT b.package, b.version, a.arch_string
640 FROM binaries b, bin_associations ba, architecture a
641 WHERE ba.suite = :suiteid
642 AND ba.bin = b.id AND b.architecture = a.id"""
643 ),
644 {"suiteid": suite_id},
645 )
646 for i in q.fetchall():
647 print(" ".join(i))
649 # List source
650 q = session.execute(
651 sql.text(
652 """SELECT s.source, s.version
653 FROM source s, src_associations sa
654 WHERE sa.suite = :suiteid
655 AND sa.source = s.id"""
656 ),
657 {"suiteid": suite_id},
658 )
659 for i in q.fetchall():
660 print(" ".join(i) + " source")
663#######################################################################################
666def main() -> None:
667 global Logger
669 cnf = Config()
671 Arguments = [
672 ("a", "add", "Control-Suite::Options::Add", "HasArg"),
673 ("b", "britney", "Control-Suite::Options::Britney"),
674 ("f", "force", "Control-Suite::Options::Force"),
675 ("h", "help", "Control-Suite::Options::Help"),
676 ("l", "list", "Control-Suite::Options::List", "HasArg"),
677 ("r", "remove", "Control-Suite::Options::Remove", "HasArg"),
678 ("s", "set", "Control-Suite::Options::Set", "HasArg"),
679 ]
681 for i in ["add", "britney", "help", "list", "remove", "set", "version"]:
682 key = "Control-Suite::Options::%s" % i
683 if key not in cnf: 683 ↛ 681line 683 didn't jump to line 681 because the condition on line 683 was always true
684 cnf[key] = ""
686 try:
687 file_list = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) # type: ignore[attr-defined]
688 except SystemError as e:
689 print("%s\n" % e)
690 usage(1)
691 Options = cnf.subtree("Control-Suite::Options")
693 if Options["Help"]:
694 usage()
696 force = "Force" in Options and Options["Force"]
698 action = None
700 for i in ("add", "list", "remove", "set"):
701 if cnf["Control-Suite::Options::%s" % (i)] != "":
702 suite_name = cnf["Control-Suite::Options::%s" % (i)]
704 if action: 704 ↛ 705line 704 didn't jump to line 705 because the condition on line 704 was never true
705 utils.fubar("Can only perform one action at a time.")
707 action = i
709 # Need an action...
710 if action is None: 710 ↛ 711line 710 didn't jump to line 711 because the condition on line 710 was never true
711 utils.fubar("No action specified.")
713 britney = False
714 if action == "set" and cnf["Control-Suite::Options::Britney"]:
715 britney = True
717 if action == "list":
718 session = DBConn().session()
719 suite = get_suite(suite_name, session)
720 if suite is None: 720 ↛ 721line 720 didn't jump to line 721 because the condition on line 720 was never true
721 utils.fubar("Unknown suite.")
722 get_list(suite, session)
723 else:
724 Logger = daklog.Logger("control-suite")
726 with ArchiveTransaction() as transaction:
727 session = transaction.session
728 suite = get_suite(suite_name, session)
729 if suite is None: 729 ↛ 730line 729 didn't jump to line 730 because the condition on line 729 was never true
730 utils.fubar("Unknown suite.")
732 if action == "set" and not suite.allowcsset:
733 if force: 733 ↛ 740line 733 didn't jump to line 740 because the condition on line 733 was always true
734 utils.warn(
735 "Would not normally allow setting suite {0} (allowcsset is FALSE), but --force used".format(
736 suite_name
737 )
738 )
739 else:
740 utils.fubar(
741 "Will not reset suite {0} due to its database configuration (allowcsset is FALSE)".format(
742 suite_name
743 )
744 )
746 if file_list: 746 ↛ 747line 746 didn't jump to line 747 because the condition on line 746 was never true
747 for f in file_list:
748 process_file(open(f), suite, action, transaction, britney, force)
749 else:
750 process_file(sys.stdin, suite, action, transaction, britney, force)
752 Logger.close()
755#######################################################################################
758if __name__ == "__main__":
759 main()