Coverage for dak/update_suite.py: 78%
125 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-08-03 16:46 +0000
1# Copyright (C) 2015, Ansgar Burchardt <ansgar@debian.org>
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU General Public License as published by
5# the Free Software Foundation; either version 2 of the License, or
6# (at your option) any later version.
7#
8# This program is distributed in the hope that it will be useful,
9# but WITHOUT ANY WARRANTY; without even the implied warranty of
10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11# GNU General Public License for more details.
12#
13# You should have received a copy of the GNU General Public License along
14# with this program; if not, write to the Free Software Foundation, Inc.,
15# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17import sys
18from collections.abc import Collection, Sequence
19from typing import TYPE_CHECKING, NoReturn
21from sqlalchemy import select, sql
23import daklib.daklog
24import daklib.utils
25from daklib.archive import ArchiveTransaction
26from daklib.dbconn import ArchiveFile, Component, DBBinary, DBSource, PoolFile, Suite
28if TYPE_CHECKING:
29 from sqlalchemy.engine import ScalarResult
31"""
32Idea:
34dak update-suite testing testing-kfreebsd
35 -> grab all source & binary packages from testing with a higher version
36 than in testing-kfreebsd (or not in -kfreebsd) and copy them
37 -> limited to architectures in testing-kfreebsd
38 -> obeys policy queues
39 -> copies to build queues
41dak update-suite --create-in=ftp-master stable testing
42 -> create suite "testing" based on "stable" in archive "ftp-master"
44Additional switches:
45 --skip-policy-queue: skip target suite's policy queue
46 --skip-build-queues: do not copy to build queue
47 --no-new-packages: do not copy new packages
48 -> source-based, new binaries from existing sources will be added
49 --only-new-packages: do not update existing packages
50 -> source-based, do not copy new binaries w/o source!
51 --also-policy-queue: also copy pending packages from policy queue
52 --update-overrides: update overrides as well (if different overrides are used)
53 --no-act
54"""
57def usage() -> NoReturn:
58 print("dak update-suite [-n|--no-act] <origin> <target>")
59 sys.exit(0)
62class SuiteUpdater:
63 def __init__(
64 self,
65 transaction: ArchiveTransaction,
66 origin: Suite,
67 target: Suite,
68 new_packages=True,
69 also_from_policy_queue=False,
70 obey_policy_queue=True,
71 obey_build_queues=True,
72 update_overrides=False,
73 dry_run=False,
74 ):
75 self.transaction = transaction
76 self.origin = origin
77 self.target = target
78 self.new_packages = new_packages
79 self.also_from_policy_queue = also_from_policy_queue
80 self.obey_policy_queue = obey_policy_queue
81 self.obey_build_queues = obey_build_queues
82 self.update_overrides = update_overrides
83 self.dry_run = dry_run
85 if obey_policy_queue and target.policy_queue_id is not None: 85 ↛ 86line 85 didn't jump to line 86 because the condition on line 85 was never true
86 raise Exception("Not implemented...")
87 self.logger = None if dry_run else daklib.daklog.Logger("update-suite")
89 def query_new_binaries(
90 self, additional_sources: Collection[int]
91 ) -> "Sequence[DBBinary]":
92 # Candidates are binaries in the origin suite, and optionally in its policy queue.
93 query = """
94 SELECT b.*
95 FROM binaries b
96 JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :origin
97 """
98 if self.also_from_policy_queue: 98 ↛ 99line 98 didn't jump to line 99 because the condition on line 98 was never true
99 query += """
100 UNION
101 SELECT b.*
102 FROM binaries b
103 JOIN policy_queue_upload_binaries_map pqubm ON pqubm.binary_id = b.id
104 JOIN policy_queue_upload pqu ON pqu.id = pqubm.policy_queue_upload_id
105 WHERE pqu.target_suite_id = :origin
106 AND pqu.policy_queue_id = (SELECT policy_queue_id FROM suite WHERE id = :origin)
107 """
109 # Only take binaries that are for a architecture part of the target suite,
110 # and whose source was just added to the target suite (i.e. listed in additional_sources)
111 # or that have the source already available in the target suite
112 # or in the target suite's policy queue if we obey policy queues,
113 # and filter out binaries with a lower version than already in the target suite.
114 if self.obey_policy_queue: 114 ↛ 123line 114 didn't jump to line 123 because the condition on line 114 was always true
115 cond_source_in_policy_queue = """
116 EXISTS (SELECT 1
117 FROM policy_queue_upload pqu
118 WHERE tmp.source = pqu.source_id
119 AND pqu.target_suite_id = :target
120 AND pqu.policy_queue_id = (SELECT policy_queue_id FROM suite WHERE id = :target))
121 """
122 else:
123 cond_source_in_policy_queue = "FALSE"
124 query = """
125 WITH tmp AS ({0})
126 SELECT DISTINCT *
127 FROM tmp
128 WHERE tmp.architecture IN (SELECT architecture FROM suite_architectures WHERE suite = :target)
129 AND (tmp.source IN :additional_sources
130 OR EXISTS (SELECT 1
131 FROM src_associations sa
132 WHERE tmp.source = sa.source AND sa.suite = :target)
133 OR {1})
134 AND NOT EXISTS (SELECT 1
135 FROM binaries b2
136 JOIN bin_associations ba2 ON b2.id = ba2.bin AND ba2.suite = :target
137 WHERE tmp.package = b2.package AND tmp.architecture = b2.architecture AND b2.version >= tmp.version)
138 ORDER BY package, version, architecture
139 """.format(
140 query, cond_source_in_policy_queue
141 )
143 # An empty tuple generates a SQL statement with "tmp.source IN ()"
144 # which is not valid. Inject an invalid value in this case:
145 # "tmp.source IN (NULL)" is always false.
147 params = {
148 "origin": self.origin.suite_id,
149 "target": self.target.suite_id,
150 "additional_sources": additional_sources or (None,),
151 }
153 return self.transaction.session.scalars(
154 select(DBBinary).from_statement(sql.text(query)), params
155 ).all()
157 def query_new_sources(self) -> "Sequence[DBSource]":
158 # Candidates are source packages in the origin suite, and optionally in its policy queue.
159 query = """
160 SELECT s.*
161 FROM source s
162 JOIN src_associations sa ON s.id = sa.source AND sa.suite = :origin
163 """
164 if self.also_from_policy_queue: 164 ↛ 165line 164 didn't jump to line 165 because the condition on line 164 was never true
165 query += """
166 UNION
167 SELECT s.*
168 FROM source s
169 JOIN policy_queue_upload pqu ON pqu.source_id = s.id
170 WHERE pqu.target_suite_id = :origin
171 AND pqu.policy_queue_id = (SELECT policy_queue_id FROM suite WHERE id = :origin)
172 """
174 # Filter out source packages with a lower version than already in the target suite.
175 query = """
176 WITH tmp AS ({0})
177 SELECT DISTINCT *
178 FROM tmp
179 WHERE NOT EXISTS (SELECT 1
180 FROM source s2
181 JOIN src_associations sa2 ON s2.id = sa2.source AND sa2.suite = :target
182 WHERE s2.source = tmp.source AND s2.version >= tmp.version)
183 """.format(
184 query
185 )
187 # Optionally filter out source packages that are not already in the target suite.
188 if not self.new_packages: 188 ↛ 189line 188 didn't jump to line 189 because the condition on line 188 was never true
189 query += """
190 AND EXISTS (SELECT 1
191 FROM source s2
192 JOIN src_associations sa2 ON s2.id = sa2.source AND sa2.suite = :target
193 WHERE s2.source = tmp.source)
194 """
196 query += "ORDER BY source, version"
198 params = {"origin": self.origin.suite_id, "target": self.target.suite_id}
200 return self.transaction.session.scalars(
201 select(DBSource).from_statement(sql.text(query)), params
202 ).all()
204 def _components_for_binary(
205 self, binary: DBBinary, suite: Suite
206 ) -> "ScalarResult[Component]":
207 session = self.transaction.session
208 return session.scalars(
209 select(Component)
210 .join(ArchiveFile, Component.component_id == ArchiveFile.component_id)
211 .join(ArchiveFile.file)
212 .where(PoolFile.file_id == binary.poolfile_id)
213 .where(ArchiveFile.archive_id == suite.archive_id)
214 )
216 def install_binaries(self, binaries: Collection[DBBinary], suite: Suite) -> None:
217 if len(binaries) == 0: 217 ↛ 218line 217 didn't jump to line 218 because the condition on line 217 was never true
218 return
219 # If origin and target suites are in the same archive, we can skip the
220 # overhead from ArchiveTransaction.copy_binary()
221 if self.origin.archive_id == suite.archive_id: 221 ↛ 227line 221 didn't jump to line 227 because the condition on line 221 was always true
222 query = "INSERT INTO bin_associations (bin, suite) VALUES (:bin, :suite)"
223 target_id = suite.suite_id
224 params = [{"bin": b.binary_id, "suite": target_id} for b in binaries]
225 self.transaction.session.execute(sql.text(query), params)
226 else:
227 for b in binaries:
228 for c in self._components_for_binary(b, suite):
229 self.transaction.copy_binary(b, suite, c)
231 def _components_for_source(
232 self, source: DBSource, suite: Suite
233 ) -> "ScalarResult[Component]":
234 session = self.transaction.session
235 return session.scalars(
236 select(Component)
237 .join(ArchiveFile, Component.component_id == ArchiveFile.component_id)
238 .join(ArchiveFile.file)
239 .where(PoolFile.file_id == source.poolfile_id)
240 .where(ArchiveFile.archive_id == suite.archive_id)
241 )
243 def install_sources(self, sources: Collection[DBSource], suite: Suite) -> None:
244 if len(sources) == 0: 244 ↛ 245line 244 didn't jump to line 245 because the condition on line 244 was never true
245 return
246 # If origin and target suites are in the same archive, we can skip the
247 # overhead from ArchiveTransaction.copy_source()
248 if self.origin.archive_id == suite.archive_id: 248 ↛ 256line 248 didn't jump to line 256 because the condition on line 248 was always true
249 query = (
250 "INSERT INTO src_associations (source, suite) VALUES (:source, :suite)"
251 )
252 target_id = suite.suite_id
253 params = [{"source": s.source_id, "suite": target_id} for s in sources]
254 self.transaction.session.execute(sql.text(query), params)
255 else:
256 for s in sources:
257 for c in self._components_for_source(s, suite):
258 self.transaction.copy_source(s, suite, c)
260 def update_suite(self) -> None:
261 targets = {self.target}
262 if self.obey_build_queues: 262 ↛ 264line 262 didn't jump to line 264 because the condition on line 262 was always true
263 targets.update([bq.suite for bq in self.target.copy_queues])
264 target_names = sorted(s.suite_name for s in targets)
265 target_name = ",".join(target_names)
267 new_sources = self.query_new_sources()
268 additional_sources = tuple(s.source_id for s in new_sources)
269 for s in new_sources:
270 self.log(["add-source", target_name, s.source, s.version])
271 if not self.dry_run:
272 for target in targets:
273 self.install_sources(new_sources, target)
275 new_binaries = self.query_new_binaries(additional_sources)
276 for b in new_binaries:
277 self.log(
278 [
279 "add-binary",
280 target_name,
281 b.package,
282 b.version,
283 b.architecture.arch_string,
284 ]
285 )
286 if not self.dry_run:
287 for target in targets:
288 self.install_binaries(new_binaries, target)
290 def log(self, args: list[object]) -> None:
291 if self.logger:
292 self.logger.log(args)
293 else:
294 print(*args, sep="|")
297def main() -> None:
298 from daklib.config import Config
300 config = Config()
302 import apt_pkg
304 arguments = [
305 ("h", "help", "Update-Suite::Options::Help"),
306 ("n", "no-act", "Update-Suite::options::NoAct"),
307 ]
308 argv = apt_pkg.parse_commandline(config.Cnf, arguments, sys.argv) # type: ignore[attr-defined]
309 try:
310 options = config.subtree("Update-Suite::Options")
311 except KeyError:
312 options = {}
314 if "Help" in options or len(argv) != 2:
315 usage()
317 origin_name = argv[0]
318 target_name = argv[1]
319 dry_run = True if "NoAct" in options else False
321 with ArchiveTransaction() as transaction:
322 session = transaction.session
324 origin = session.execute(
325 select(Suite).filter_by(suite_name=origin_name)
326 ).scalar_one_or_none()
327 if origin is None: 327 ↛ 328line 327 didn't jump to line 328 because the condition on line 327 was never true
328 daklib.utils.fubar("Origin suite '{0}' is unknown.".format(origin_name))
329 target = session.execute(
330 select(Suite).filter_by(suite_name=target_name)
331 ).scalar_one_or_none()
332 if target is None: 332 ↛ 333line 332 didn't jump to line 333 because the condition on line 332 was never true
333 daklib.utils.fubar("Target suite '{0}' is unknown.".format(target_name))
335 su = SuiteUpdater(transaction, origin, target, dry_run=dry_run)
336 su.update_suite()
338 if dry_run:
339 transaction.rollback()
340 else:
341 transaction.commit()