1#! /usr/bin/env python3
2#
3# Copyright (C) 2015, Ansgar Burchardt <ansgar@debian.org>
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License as published by
7# the Free Software Foundation; either version 2 of the License, or
8# (at your option) any later version.
9#
10# This program is distributed in the hope that it will be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13# GNU General Public License for more details.
14#
15# You should have received a copy of the GNU General Public License along
16# with this program; if not, write to the Free Software Foundation, Inc.,
17# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19from daklib.archive import ArchiveTransaction
20from daklib.dbconn import *
21import daklib.daklog
22import daklib.utils
24from sqlalchemy.orm.exc import NoResultFound
25import sqlalchemy.sql as sql
26import sys
28"""
29Idea:
31dak update-suite testing testing-kfreebsd
32 -> grab all source & binary packages from testing with a higher version
33 than in testing-kfreebsd (or not in -kfreebsd) and copy them
34 -> limited to architectures in testing-kfreebsd
35 -> obeys policy queues
36 -> copies to build queues
38dak update-suite --create-in=ftp-master stable testing
39 -> create suite "testing" based on "stable" in archive "ftp-master"
41Additional switches:
42 --skip-policy-queue: skip target suite's policy queue
43 --skip-build-queues: do not copy to build queue
44 --no-new-packages: do not copy new packages
45 -> source-based, new binaries from existing sources will be added
46 --only-new-packages: do not update existing packages
47 -> source-based, do not copy new binaries w/o source!
48 --also-policy-queue: also copy pending packages from policy queue
49 --update-overrides: update overrides as well (if different overrides are used)
50 --no-act
51"""
54def usage():
55 print("dak update-suite [-n|--no-act] <origin> <target>")
56 sys.exit(0)
59class SuiteUpdater:
60 def __init__(self, transaction, origin, target,
61 new_packages=True, also_from_policy_queue=False,
62 obey_policy_queue=True, obey_build_queues=True,
63 update_overrides=False, dry_run=False):
64 self.transaction = transaction
65 self.origin = origin
66 self.target = target
67 self.new_packages = new_packages
68 self.also_from_policy_queue = also_from_policy_queue
69 self.obey_policy_queue = obey_policy_queue
70 self.obey_build_queues = obey_build_queues
71 self.update_overrides = update_overrides
72 self.dry_run = dry_run
74 if obey_policy_queue and target.policy_queue_id is not None:
75 raise Exception('Not implemented...')
76 self.logger = None if dry_run else daklib.daklog.Logger("update-suite")
78 def query_new_binaries(self, additional_sources):
79 # Candidates are binaries in the origin suite, and optionally in its policy queue.
80 query = """
81 SELECT b.*
82 FROM binaries b
83 JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :origin
84 """
85 if self.also_from_policy_queue:
86 query += """
87 UNION
88 SELECT b.*
89 FROM binaries b
90 JOIN policy_queue_upload_binaries_map pqubm ON pqubm.binary_id = b.id
91 JOIN policy_queue_upload pqu ON pqu.id = pqubm.policy_queue_upload_id
92 WHERE pqu.target_suite_id = :origin
93 AND pqu.policy_queue_id = (SELECT policy_queue_id FROM suite WHERE id = :origin)
94 """
96 # Only take binaries that are for a architecture part of the target suite,
97 # and whose source was just added to the target suite (i.e. listed in additional_sources)
98 # or that have the source already available in the target suite
99 # or in the target suite's policy queue if we obey policy queues,
100 # and filter out binaries with a lower version than already in the target suite.
101 if self.obey_policy_queue:
102 cond_source_in_policy_queue = """
103 EXISTS (SELECT 1
104 FROM policy_queue_upload pqu
105 WHERE tmp.source = pqu.source_id
106 AND pqu.target_suite_id = :target
107 AND pqu.policy_queue_id = (SELECT policy_queue_id FROM suite WHERE id = :target))
108 """
109 else:
110 cond_source_in_policy_queue = "FALSE"
111 query = """
112 WITH tmp AS ({0})
113 SELECT DISTINCT *
114 FROM tmp
115 WHERE tmp.architecture IN (SELECT architecture FROM suite_architectures WHERE suite = :target)
116 AND (tmp.source IN :additional_sources
117 OR EXISTS (SELECT 1
118 FROM src_associations sa
119 WHERE tmp.source = sa.source AND sa.suite = :target)
120 OR {1})
121 AND NOT EXISTS (SELECT 1
122 FROM binaries b2
123 JOIN bin_associations ba2 ON b2.id = ba2.bin AND ba2.suite = :target
124 WHERE tmp.package = b2.package AND tmp.architecture = b2.architecture AND b2.version >= tmp.version)
125 ORDER BY package, version, architecture
126 """.format(query, cond_source_in_policy_queue)
128 # An empty tuple generates a SQL statement with "tmp.source IN ()"
129 # which is not valid. Inject an invalid value in this case:
130 # "tmp.source IN (NULL)" is always false.
131 if len(additional_sources) == 0:
132 additional_sources = tuple([None])
134 params = {
135 'origin': self.origin.suite_id,
136 'target': self.target.suite_id,
137 'additional_sources': additional_sources,
138 }
140 return self.transaction.session.query(DBBinary).from_statement(sql.text(query)).params(params)
142 def query_new_sources(self):
143 # Candidates are source packages in the origin suite, and optionally in its policy queue.
144 query = """
145 SELECT s.*
146 FROM source s
147 JOIN src_associations sa ON s.id = sa.source AND sa.suite = :origin
148 """
149 if self.also_from_policy_queue:
150 query += """
151 UNION
152 SELECT s.*
153 FROM source s
154 JOIN policy_queue_upload pqu ON pqu.source_id = s.id
155 WHERE pqu.target_suite_id = :origin
156 AND pqu.policy_queue_id = (SELECT policy_queue_id FROM suite WHERE id = :origin)
157 """
159 # Filter out source packages with a lower version than already in the target suite.
160 query = """
161 WITH tmp AS ({0})
162 SELECT DISTINCT *
163 FROM tmp
164 WHERE NOT EXISTS (SELECT 1
165 FROM source s2
166 JOIN src_associations sa2 ON s2.id = sa2.source AND sa2.suite = :target
167 WHERE s2.source = tmp.source AND s2.version >= tmp.version)
168 """.format(query)
170 # Optionally filter out source packages that are not already in the target suite.
171 if not self.new_packages:
172 query += """
173 AND EXISTS (SELECT 1
174 FROM source s2
175 JOIN src_associations sa2 ON s2.id = sa2.source AND sa2.suite = :target
176 WHERE s2.source = tmp.source)
177 """
179 query += "ORDER BY source, version"
181 params = {'origin': self.origin.suite_id, 'target': self.target.suite_id}
183 return self.transaction.session.query(DBSource).from_statement(sql.text(query)).params(params)
185 def _components_for_binary(self, binary, suite):
186 session = self.transaction.session
187 return session.query(Component) \
188 .join(ArchiveFile, Component.component_id == ArchiveFile.component_id) \
189 .join(ArchiveFile.file).filter(PoolFile.file_id == binary.poolfile_id) \
190 .filter(ArchiveFile.archive_id == suite.archive_id)
192 def install_binaries(self, binaries, suite):
193 if len(binaries) == 0:
194 return
195 # If origin and target suites are in the same archive, we can skip the
196 # overhead from ArchiveTransaction.copy_binary()
197 if self.origin.archive_id == suite.archive_id:
198 query = "INSERT INTO bin_associations (bin, suite) VALUES (:bin, :suite)"
199 target_id = suite.suite_id
200 params = [{'bin': b.binary_id, 'suite': target_id} for b in binaries]
201 self.transaction.session.execute(query, params)
202 else:
203 for b in binaries:
204 for c in self._components_for_binary(b, suite):
205 self.transaction.copy_binary(b, suite, c)
207 def _components_for_source(self, source, suite):
208 session = self.transaction.session
209 return session.query(Component) \
210 .join(ArchiveFile, Component.component_id == ArchiveFile.component_id) \
211 .join(ArchiveFile.file).filter(PoolFile.file_id == source.poolfile_id) \
212 .filter(ArchiveFile.archive_id == suite.archive_id)
214 def install_sources(self, sources, suite):
215 if len(sources) == 0:
216 return
217 # If origin and target suites are in the same archive, we can skip the
218 # overhead from ArchiveTransaction.copy_source()
219 if self.origin.archive_id == suite.archive_id:
220 query = "INSERT INTO src_associations (source, suite) VALUES (:source, :suite)"
221 target_id = suite.suite_id
222 params = [{'source': s.source_id, 'suite': target_id} for s in sources]
223 self.transaction.session.execute(query, params)
224 else:
225 for s in sources:
226 for c in self._components_for_source(s, suite):
227 self.transaction.copy_source(s, suite, c)
229 def update_suite(self):
230 targets = set([self.target])
231 if self.obey_build_queues:
232 targets.update([bq.suite for bq in self.target.copy_queues])
233 target_names = sorted(s.suite_name for s in targets)
234 target_name = ",".join(target_names)
236 new_sources = self.query_new_sources().all()
237 additional_sources = tuple(s.source_id for s in new_sources)
238 for s in new_sources:
239 self.log(["add-source", target_name, s.source, s.version])
240 if not self.dry_run:
241 for target in targets:
242 self.install_sources(new_sources, target)
244 new_binaries = self.query_new_binaries(additional_sources).all()
245 for b in new_binaries:
246 self.log(["add-binary", target_name, b.package, b.version, b.architecture.arch_string])
247 if not self.dry_run:
248 for target in targets:
249 self.install_binaries(new_binaries, target)
251 def log(self, args):
252 if self.logger:
253 self.logger.log(args)
254 else:
255 print(args)
258def main():
259 from daklib.config import Config
260 config = Config()
262 import apt_pkg
263 arguments = [
264 ('h', 'help', 'Update-Suite::Options::Help'),
265 ('n', 'no-act', 'Update-Suite::options::NoAct'),
266 ]
267 argv = apt_pkg.parse_commandline(config.Cnf, arguments, sys.argv)
268 try:
269 options = config.subtree("Update-Suite::Options")
270 except KeyError:
271 options = {}
273 if 'Help' in options or len(argv) != 2: 273 ↛ 276line 273 didn't jump to line 276, because the condition on line 273 was never false
274 usage()
276 origin_name = argv[0]
277 target_name = argv[1]
278 dry_run = True if 'NoAct' in options else False
280 with ArchiveTransaction() as transaction:
281 session = transaction.session
283 try:
284 origin = session.query(Suite).filter_by(suite_name=origin_name).one()
285 except NoResultFound:
286 daklib.utils.fubar("Origin suite '{0}' is unknown.".format(origin_name))
287 try:
288 target = session.query(Suite).filter_by(suite_name=target_name).one()
289 except NoResultFound:
290 daklib.utils.fubar("Target suite '{0}' is unknown.".format(target_name))
292 su = SuiteUpdater(transaction, origin, target, dry_run=dry_run)
293 su.update_suite()
295 if dry_run:
296 transaction.rollback()
297 else:
298 transaction.commit()
301if __name__ == '__main__':
302 pass