Coverage for dak/update_suite.py: 78%
128 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-01-04 16:18 +0000
1#! /usr/bin/env python3
2#
3# Copyright (C) 2015, Ansgar Burchardt <ansgar@debian.org>
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License as published by
7# the Free Software Foundation; either version 2 of the License, or
8# (at your option) any later version.
9#
10# This program is distributed in the hope that it will be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13# GNU General Public License for more details.
14#
15# You should have received a copy of the GNU General Public License along
16# with this program; if not, write to the Free Software Foundation, Inc.,
17# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19import sys
20from collections.abc import Collection
21from typing import TYPE_CHECKING, NoReturn
23import sqlalchemy.sql as sql
24from sqlalchemy.orm.exc import NoResultFound
26import daklib.daklog
27import daklib.utils
28from daklib.archive import ArchiveTransaction
29from daklib.dbconn import ArchiveFile, Component, DBBinary, DBSource, PoolFile, Suite
31if TYPE_CHECKING:
32 from sqlalchemy.orm import Query
34"""
35Idea:
37dak update-suite testing testing-kfreebsd
38 -> grab all source & binary packages from testing with a higher version
39 than in testing-kfreebsd (or not in -kfreebsd) and copy them
40 -> limited to architectures in testing-kfreebsd
41 -> obeys policy queues
42 -> copies to build queues
44dak update-suite --create-in=ftp-master stable testing
45 -> create suite "testing" based on "stable" in archive "ftp-master"
47Additional switches:
48 --skip-policy-queue: skip target suite's policy queue
49 --skip-build-queues: do not copy to build queue
50 --no-new-packages: do not copy new packages
51 -> source-based, new binaries from existing sources will be added
52 --only-new-packages: do not update existing packages
53 -> source-based, do not copy new binaries w/o source!
54 --also-policy-queue: also copy pending packages from policy queue
55 --update-overrides: update overrides as well (if different overrides are used)
56 --no-act
57"""
60def usage() -> NoReturn:
61 print("dak update-suite [-n|--no-act] <origin> <target>")
62 sys.exit(0)
65class SuiteUpdater:
66 def __init__(
67 self,
68 transaction: ArchiveTransaction,
69 origin: Suite,
70 target: Suite,
71 new_packages=True,
72 also_from_policy_queue=False,
73 obey_policy_queue=True,
74 obey_build_queues=True,
75 update_overrides=False,
76 dry_run=False,
77 ):
78 self.transaction = transaction
79 self.origin = origin
80 self.target = target
81 self.new_packages = new_packages
82 self.also_from_policy_queue = also_from_policy_queue
83 self.obey_policy_queue = obey_policy_queue
84 self.obey_build_queues = obey_build_queues
85 self.update_overrides = update_overrides
86 self.dry_run = dry_run
88 if obey_policy_queue and target.policy_queue_id is not None: 88 ↛ 89line 88 didn't jump to line 89 because the condition on line 88 was never true
89 raise Exception("Not implemented...")
90 self.logger = None if dry_run else daklib.daklog.Logger("update-suite")
92 def query_new_binaries(
93 self, additional_sources: Collection[int]
94 ) -> "Query[DBBinary]":
95 # Candidates are binaries in the origin suite, and optionally in its policy queue.
96 query = """
97 SELECT b.*
98 FROM binaries b
99 JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :origin
100 """
101 if self.also_from_policy_queue: 101 ↛ 102line 101 didn't jump to line 102 because the condition on line 101 was never true
102 query += """
103 UNION
104 SELECT b.*
105 FROM binaries b
106 JOIN policy_queue_upload_binaries_map pqubm ON pqubm.binary_id = b.id
107 JOIN policy_queue_upload pqu ON pqu.id = pqubm.policy_queue_upload_id
108 WHERE pqu.target_suite_id = :origin
109 AND pqu.policy_queue_id = (SELECT policy_queue_id FROM suite WHERE id = :origin)
110 """
112 # Only take binaries that are for a architecture part of the target suite,
113 # and whose source was just added to the target suite (i.e. listed in additional_sources)
114 # or that have the source already available in the target suite
115 # or in the target suite's policy queue if we obey policy queues,
116 # and filter out binaries with a lower version than already in the target suite.
117 if self.obey_policy_queue: 117 ↛ 126line 117 didn't jump to line 126 because the condition on line 117 was always true
118 cond_source_in_policy_queue = """
119 EXISTS (SELECT 1
120 FROM policy_queue_upload pqu
121 WHERE tmp.source = pqu.source_id
122 AND pqu.target_suite_id = :target
123 AND pqu.policy_queue_id = (SELECT policy_queue_id FROM suite WHERE id = :target))
124 """
125 else:
126 cond_source_in_policy_queue = "FALSE"
127 query = """
128 WITH tmp AS ({0})
129 SELECT DISTINCT *
130 FROM tmp
131 WHERE tmp.architecture IN (SELECT architecture FROM suite_architectures WHERE suite = :target)
132 AND (tmp.source IN :additional_sources
133 OR EXISTS (SELECT 1
134 FROM src_associations sa
135 WHERE tmp.source = sa.source AND sa.suite = :target)
136 OR {1})
137 AND NOT EXISTS (SELECT 1
138 FROM binaries b2
139 JOIN bin_associations ba2 ON b2.id = ba2.bin AND ba2.suite = :target
140 WHERE tmp.package = b2.package AND tmp.architecture = b2.architecture AND b2.version >= tmp.version)
141 ORDER BY package, version, architecture
142 """.format(
143 query, cond_source_in_policy_queue
144 )
146 # An empty tuple generates a SQL statement with "tmp.source IN ()"
147 # which is not valid. Inject an invalid value in this case:
148 # "tmp.source IN (NULL)" is always false.
150 params = {
151 "origin": self.origin.suite_id,
152 "target": self.target.suite_id,
153 "additional_sources": additional_sources or (None,),
154 }
156 return (
157 self.transaction.session.query(DBBinary)
158 .from_statement(sql.text(query)) # type: ignore[arg-type]
159 .params(params)
160 )
162 def query_new_sources(self) -> "Query[DBSource]":
163 # Candidates are source packages in the origin suite, and optionally in its policy queue.
164 query = """
165 SELECT s.*
166 FROM source s
167 JOIN src_associations sa ON s.id = sa.source AND sa.suite = :origin
168 """
169 if self.also_from_policy_queue: 169 ↛ 170line 169 didn't jump to line 170 because the condition on line 169 was never true
170 query += """
171 UNION
172 SELECT s.*
173 FROM source s
174 JOIN policy_queue_upload pqu ON pqu.source_id = s.id
175 WHERE pqu.target_suite_id = :origin
176 AND pqu.policy_queue_id = (SELECT policy_queue_id FROM suite WHERE id = :origin)
177 """
179 # Filter out source packages with a lower version than already in the target suite.
180 query = """
181 WITH tmp AS ({0})
182 SELECT DISTINCT *
183 FROM tmp
184 WHERE NOT EXISTS (SELECT 1
185 FROM source s2
186 JOIN src_associations sa2 ON s2.id = sa2.source AND sa2.suite = :target
187 WHERE s2.source = tmp.source AND s2.version >= tmp.version)
188 """.format(
189 query
190 )
192 # Optionally filter out source packages that are not already in the target suite.
193 if not self.new_packages: 193 ↛ 194line 193 didn't jump to line 194 because the condition on line 193 was never true
194 query += """
195 AND EXISTS (SELECT 1
196 FROM source s2
197 JOIN src_associations sa2 ON s2.id = sa2.source AND sa2.suite = :target
198 WHERE s2.source = tmp.source)
199 """
201 query += "ORDER BY source, version"
203 params = {"origin": self.origin.suite_id, "target": self.target.suite_id}
205 return (
206 self.transaction.session.query(DBSource)
207 .from_statement(sql.text(query)) # type: ignore[arg-type]
208 .params(params)
209 )
211 def _components_for_binary(
212 self, binary: DBBinary, suite: Suite
213 ) -> "Query[Component]":
214 session = self.transaction.session
215 return (
216 session.query(Component)
217 .join(ArchiveFile, Component.component_id == ArchiveFile.component_id)
218 .join(ArchiveFile.file)
219 .filter(PoolFile.file_id == binary.poolfile_id)
220 .filter(ArchiveFile.archive_id == suite.archive_id)
221 )
223 def install_binaries(self, binaries: Collection[DBBinary], suite: Suite) -> None:
224 if len(binaries) == 0: 224 ↛ 225line 224 didn't jump to line 225 because the condition on line 224 was never true
225 return
226 # If origin and target suites are in the same archive, we can skip the
227 # overhead from ArchiveTransaction.copy_binary()
228 if self.origin.archive_id == suite.archive_id: 228 ↛ 234line 228 didn't jump to line 234 because the condition on line 228 was always true
229 query = "INSERT INTO bin_associations (bin, suite) VALUES (:bin, :suite)"
230 target_id = suite.suite_id
231 params = [{"bin": b.binary_id, "suite": target_id} for b in binaries]
232 self.transaction.session.execute(sql.text(query), params)
233 else:
234 for b in binaries:
235 for c in self._components_for_binary(b, suite):
236 self.transaction.copy_binary(b, suite, c)
238 def _components_for_source(
239 self, source: DBSource, suite: Suite
240 ) -> "Query[Component]":
241 session = self.transaction.session
242 return (
243 session.query(Component)
244 .join(ArchiveFile, Component.component_id == ArchiveFile.component_id)
245 .join(ArchiveFile.file)
246 .filter(PoolFile.file_id == source.poolfile_id)
247 .filter(ArchiveFile.archive_id == suite.archive_id)
248 )
250 def install_sources(self, sources: Collection[DBSource], suite: Suite) -> None:
251 if len(sources) == 0: 251 ↛ 252line 251 didn't jump to line 252 because the condition on line 251 was never true
252 return
253 # If origin and target suites are in the same archive, we can skip the
254 # overhead from ArchiveTransaction.copy_source()
255 if self.origin.archive_id == suite.archive_id: 255 ↛ 263line 255 didn't jump to line 263 because the condition on line 255 was always true
256 query = (
257 "INSERT INTO src_associations (source, suite) VALUES (:source, :suite)"
258 )
259 target_id = suite.suite_id
260 params = [{"source": s.source_id, "suite": target_id} for s in sources]
261 self.transaction.session.execute(sql.text(query), params)
262 else:
263 for s in sources:
264 for c in self._components_for_source(s, suite):
265 self.transaction.copy_source(s, suite, c)
267 def update_suite(self) -> None:
268 targets = set([self.target])
269 if self.obey_build_queues: 269 ↛ 271line 269 didn't jump to line 271 because the condition on line 269 was always true
270 targets.update([bq.suite for bq in self.target.copy_queues])
271 target_names = sorted(s.suite_name for s in targets)
272 target_name = ",".join(target_names)
274 new_sources = self.query_new_sources().all()
275 additional_sources = tuple(s.source_id for s in new_sources)
276 for s in new_sources:
277 self.log(["add-source", target_name, s.source, s.version])
278 if not self.dry_run:
279 for target in targets:
280 self.install_sources(new_sources, target)
282 new_binaries = self.query_new_binaries(additional_sources).all()
283 for b in new_binaries:
284 self.log(
285 [
286 "add-binary",
287 target_name,
288 b.package,
289 b.version,
290 b.architecture.arch_string,
291 ]
292 )
293 if not self.dry_run:
294 for target in targets:
295 self.install_binaries(new_binaries, target)
297 def log(self, args: list[object]) -> None:
298 if self.logger:
299 self.logger.log(args)
300 else:
301 print(*args, sep="|")
304def main() -> None:
305 from daklib.config import Config
307 config = Config()
309 import apt_pkg
311 arguments = [
312 ("h", "help", "Update-Suite::Options::Help"),
313 ("n", "no-act", "Update-Suite::options::NoAct"),
314 ]
315 argv = apt_pkg.parse_commandline(config.Cnf, arguments, sys.argv) # type: ignore[attr-defined]
316 try:
317 options = config.subtree("Update-Suite::Options")
318 except KeyError:
319 options = {}
321 if "Help" in options or len(argv) != 2:
322 usage()
324 origin_name = argv[0]
325 target_name = argv[1]
326 dry_run = True if "NoAct" in options else False
328 with ArchiveTransaction() as transaction:
329 session = transaction.session
331 try:
332 origin = session.query(Suite).filter_by(suite_name=origin_name).one()
333 except NoResultFound:
334 daklib.utils.fubar("Origin suite '{0}' is unknown.".format(origin_name))
335 try:
336 target = session.query(Suite).filter_by(suite_name=target_name).one()
337 except NoResultFound:
338 daklib.utils.fubar("Target suite '{0}' is unknown.".format(target_name))
340 su = SuiteUpdater(transaction, origin, target, dry_run=dry_run)
341 su.update_suite()
343 if dry_run:
344 transaction.rollback()
345 else:
346 transaction.commit()
349if __name__ == "__main__":
350 pass