1#! /usr/bin/env python3
2#
3# Copyright (C) 2015, Ansgar Burchardt <ansgar@debian.org>
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License as published by
7# the Free Software Foundation; either version 2 of the License, or
8# (at your option) any later version.
9#
10# This program is distributed in the hope that it will be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13# GNU General Public License for more details.
14#
15# You should have received a copy of the GNU General Public License along
16# with this program; if not, write to the Free Software Foundation, Inc.,
17# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19import sys
21import sqlalchemy.sql as sql
22from sqlalchemy.orm.exc import NoResultFound
24import daklib.daklog
25import daklib.utils
26from daklib.archive import ArchiveTransaction
27from daklib.dbconn import ArchiveFile, Component, DBBinary, DBSource, PoolFile, Suite
29"""
30Idea:
32dak update-suite testing testing-kfreebsd
33 -> grab all source & binary packages from testing with a higher version
34 than in testing-kfreebsd (or not in -kfreebsd) and copy them
35 -> limited to architectures in testing-kfreebsd
36 -> obeys policy queues
37 -> copies to build queues
39dak update-suite --create-in=ftp-master stable testing
40 -> create suite "testing" based on "stable" in archive "ftp-master"
42Additional switches:
43 --skip-policy-queue: skip target suite's policy queue
44 --skip-build-queues: do not copy to build queue
45 --no-new-packages: do not copy new packages
46 -> source-based, new binaries from existing sources will be added
47 --only-new-packages: do not update existing packages
48 -> source-based, do not copy new binaries w/o source!
49 --also-policy-queue: also copy pending packages from policy queue
50 --update-overrides: update overrides as well (if different overrides are used)
51 --no-act
52"""
55def usage():
56 print("dak update-suite [-n|--no-act] <origin> <target>")
57 sys.exit(0)
60class SuiteUpdater:
61 def __init__(
62 self,
63 transaction,
64 origin,
65 target,
66 new_packages=True,
67 also_from_policy_queue=False,
68 obey_policy_queue=True,
69 obey_build_queues=True,
70 update_overrides=False,
71 dry_run=False,
72 ):
73 self.transaction = transaction
74 self.origin = origin
75 self.target = target
76 self.new_packages = new_packages
77 self.also_from_policy_queue = also_from_policy_queue
78 self.obey_policy_queue = obey_policy_queue
79 self.obey_build_queues = obey_build_queues
80 self.update_overrides = update_overrides
81 self.dry_run = dry_run
83 if obey_policy_queue and target.policy_queue_id is not None:
84 raise Exception("Not implemented...")
85 self.logger = None if dry_run else daklib.daklog.Logger("update-suite")
87 def query_new_binaries(self, additional_sources):
88 # Candidates are binaries in the origin suite, and optionally in its policy queue.
89 query = """
90 SELECT b.*
91 FROM binaries b
92 JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :origin
93 """
94 if self.also_from_policy_queue:
95 query += """
96 UNION
97 SELECT b.*
98 FROM binaries b
99 JOIN policy_queue_upload_binaries_map pqubm ON pqubm.binary_id = b.id
100 JOIN policy_queue_upload pqu ON pqu.id = pqubm.policy_queue_upload_id
101 WHERE pqu.target_suite_id = :origin
102 AND pqu.policy_queue_id = (SELECT policy_queue_id FROM suite WHERE id = :origin)
103 """
105 # Only take binaries that are for a architecture part of the target suite,
106 # and whose source was just added to the target suite (i.e. listed in additional_sources)
107 # or that have the source already available in the target suite
108 # or in the target suite's policy queue if we obey policy queues,
109 # and filter out binaries with a lower version than already in the target suite.
110 if self.obey_policy_queue:
111 cond_source_in_policy_queue = """
112 EXISTS (SELECT 1
113 FROM policy_queue_upload pqu
114 WHERE tmp.source = pqu.source_id
115 AND pqu.target_suite_id = :target
116 AND pqu.policy_queue_id = (SELECT policy_queue_id FROM suite WHERE id = :target))
117 """
118 else:
119 cond_source_in_policy_queue = "FALSE"
120 query = """
121 WITH tmp AS ({0})
122 SELECT DISTINCT *
123 FROM tmp
124 WHERE tmp.architecture IN (SELECT architecture FROM suite_architectures WHERE suite = :target)
125 AND (tmp.source IN :additional_sources
126 OR EXISTS (SELECT 1
127 FROM src_associations sa
128 WHERE tmp.source = sa.source AND sa.suite = :target)
129 OR {1})
130 AND NOT EXISTS (SELECT 1
131 FROM binaries b2
132 JOIN bin_associations ba2 ON b2.id = ba2.bin AND ba2.suite = :target
133 WHERE tmp.package = b2.package AND tmp.architecture = b2.architecture AND b2.version >= tmp.version)
134 ORDER BY package, version, architecture
135 """.format(
136 query, cond_source_in_policy_queue
137 )
139 # An empty tuple generates a SQL statement with "tmp.source IN ()"
140 # which is not valid. Inject an invalid value in this case:
141 # "tmp.source IN (NULL)" is always false.
142 if len(additional_sources) == 0:
143 additional_sources = tuple([None])
145 params = {
146 "origin": self.origin.suite_id,
147 "target": self.target.suite_id,
148 "additional_sources": additional_sources,
149 }
151 return (
152 self.transaction.session.query(DBBinary)
153 .from_statement(sql.text(query))
154 .params(params)
155 )
157 def query_new_sources(self):
158 # Candidates are source packages in the origin suite, and optionally in its policy queue.
159 query = """
160 SELECT s.*
161 FROM source s
162 JOIN src_associations sa ON s.id = sa.source AND sa.suite = :origin
163 """
164 if self.also_from_policy_queue:
165 query += """
166 UNION
167 SELECT s.*
168 FROM source s
169 JOIN policy_queue_upload pqu ON pqu.source_id = s.id
170 WHERE pqu.target_suite_id = :origin
171 AND pqu.policy_queue_id = (SELECT policy_queue_id FROM suite WHERE id = :origin)
172 """
174 # Filter out source packages with a lower version than already in the target suite.
175 query = """
176 WITH tmp AS ({0})
177 SELECT DISTINCT *
178 FROM tmp
179 WHERE NOT EXISTS (SELECT 1
180 FROM source s2
181 JOIN src_associations sa2 ON s2.id = sa2.source AND sa2.suite = :target
182 WHERE s2.source = tmp.source AND s2.version >= tmp.version)
183 """.format(
184 query
185 )
187 # Optionally filter out source packages that are not already in the target suite.
188 if not self.new_packages:
189 query += """
190 AND EXISTS (SELECT 1
191 FROM source s2
192 JOIN src_associations sa2 ON s2.id = sa2.source AND sa2.suite = :target
193 WHERE s2.source = tmp.source)
194 """
196 query += "ORDER BY source, version"
198 params = {"origin": self.origin.suite_id, "target": self.target.suite_id}
200 return (
201 self.transaction.session.query(DBSource)
202 .from_statement(sql.text(query))
203 .params(params)
204 )
206 def _components_for_binary(self, binary, suite):
207 session = self.transaction.session
208 return (
209 session.query(Component)
210 .join(ArchiveFile, Component.component_id == ArchiveFile.component_id)
211 .join(ArchiveFile.file)
212 .filter(PoolFile.file_id == binary.poolfile_id)
213 .filter(ArchiveFile.archive_id == suite.archive_id)
214 )
216 def install_binaries(self, binaries, suite):
217 if len(binaries) == 0:
218 return
219 # If origin and target suites are in the same archive, we can skip the
220 # overhead from ArchiveTransaction.copy_binary()
221 if self.origin.archive_id == suite.archive_id:
222 query = "INSERT INTO bin_associations (bin, suite) VALUES (:bin, :suite)"
223 target_id = suite.suite_id
224 params = [{"bin": b.binary_id, "suite": target_id} for b in binaries]
225 self.transaction.session.execute(query, params)
226 else:
227 for b in binaries:
228 for c in self._components_for_binary(b, suite):
229 self.transaction.copy_binary(b, suite, c)
231 def _components_for_source(self, source, suite):
232 session = self.transaction.session
233 return (
234 session.query(Component)
235 .join(ArchiveFile, Component.component_id == ArchiveFile.component_id)
236 .join(ArchiveFile.file)
237 .filter(PoolFile.file_id == source.poolfile_id)
238 .filter(ArchiveFile.archive_id == suite.archive_id)
239 )
241 def install_sources(self, sources, suite):
242 if len(sources) == 0:
243 return
244 # If origin and target suites are in the same archive, we can skip the
245 # overhead from ArchiveTransaction.copy_source()
246 if self.origin.archive_id == suite.archive_id:
247 query = (
248 "INSERT INTO src_associations (source, suite) VALUES (:source, :suite)"
249 )
250 target_id = suite.suite_id
251 params = [{"source": s.source_id, "suite": target_id} for s in sources]
252 self.transaction.session.execute(query, params)
253 else:
254 for s in sources:
255 for c in self._components_for_source(s, suite):
256 self.transaction.copy_source(s, suite, c)
258 def update_suite(self):
259 targets = set([self.target])
260 if self.obey_build_queues:
261 targets.update([bq.suite for bq in self.target.copy_queues])
262 target_names = sorted(s.suite_name for s in targets)
263 target_name = ",".join(target_names)
265 new_sources = self.query_new_sources().all()
266 additional_sources = tuple(s.source_id for s in new_sources)
267 for s in new_sources:
268 self.log(["add-source", target_name, s.source, s.version])
269 if not self.dry_run:
270 for target in targets:
271 self.install_sources(new_sources, target)
273 new_binaries = self.query_new_binaries(additional_sources).all()
274 for b in new_binaries:
275 self.log(
276 [
277 "add-binary",
278 target_name,
279 b.package,
280 b.version,
281 b.architecture.arch_string,
282 ]
283 )
284 if not self.dry_run:
285 for target in targets:
286 self.install_binaries(new_binaries, target)
288 def log(self, args):
289 if self.logger:
290 self.logger.log(args)
291 else:
292 print(args)
295def main():
296 from daklib.config import Config
298 config = Config()
300 import apt_pkg
302 arguments = [
303 ("h", "help", "Update-Suite::Options::Help"),
304 ("n", "no-act", "Update-Suite::options::NoAct"),
305 ]
306 argv = apt_pkg.parse_commandline(config.Cnf, arguments, sys.argv)
307 try:
308 options = config.subtree("Update-Suite::Options")
309 except KeyError:
310 options = {}
312 if "Help" in options or len(argv) != 2: 312 ↛ 315line 312 didn't jump to line 315, because the condition on line 312 was never false
313 usage()
315 origin_name = argv[0]
316 target_name = argv[1]
317 dry_run = True if "NoAct" in options else False
319 with ArchiveTransaction() as transaction:
320 session = transaction.session
322 try:
323 origin = session.query(Suite).filter_by(suite_name=origin_name).one()
324 except NoResultFound:
325 daklib.utils.fubar("Origin suite '{0}' is unknown.".format(origin_name))
326 try:
327 target = session.query(Suite).filter_by(suite_name=target_name).one()
328 except NoResultFound:
329 daklib.utils.fubar("Target suite '{0}' is unknown.".format(target_name))
331 su = SuiteUpdater(transaction, origin, target, dry_run=dry_run)
332 su.update_suite()
334 if dry_run:
335 transaction.rollback()
336 else:
337 transaction.commit()
340if __name__ == "__main__":
341 pass