Coverage for dak/dominate.py: 87%
55 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-08-03 16:46 +0000
1"""
2Remove obsolete source and binary associations from suites.
4@contact: Debian FTP Master <ftpmaster@debian.org>
5@copyright: 2009 Torsten Werner <twerner@debian.org>
6@license: GNU General Public License version 2 or later
7"""
9# This program is free software; you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation; either version 2 of the License, or
12# (at your option) any later version.
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
19# You should have received a copy of the GNU General Public License
20# along with this program; if not, write to the Free Software
21# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23import sys
25import apt_pkg
26from sqlalchemy.sql import exists, select, text
27from tabulate import tabulate
29from daklib import daklog, utils
30from daklib.config import Config
31from daklib.dbconn import DBConn, PolicyQueue, Suite
33Options: apt_pkg.Configuration
34Logger: daklog.Logger
37def retrieve_associations(suites, session):
38 return session.execute(
39 text(
40 """
41WITH
42 -- Provide (source, suite) tuple of all source packages to remain
43 remain_source AS (
44 SELECT
45 *
46 FROM (
47 SELECT
48 source.id AS source_id,
49 src_associations.suite AS suite_id,
50 -- generate rank over versions of a source package in one suite
51 -- "1" being the newest
52 dense_rank() OVER (
53 PARTITION BY source.source, src_associations.suite
54 ORDER BY source.version DESC
55 ) AS version_rank,
56 src_associations.created AS created,
57 suite.stayofexecution as stayofexecution
58 FROM
59 source
60 INNER JOIN src_associations ON
61 src_associations.source = source.id
62 AND src_associations.suite = ANY(:suite_ids)
63 INNER JOIN suite ON
64 src_associations.suite = suite.id
65 ) AS source_ranked
66 WHERE
67 -- we only want to retain the newest of each, but we want to publish
68 -- all sources in the archive for at least a day
69 CASE
70 WHEN now() BETWEEN created AND created + stayofexecution THEN TRUE
71 ELSE version_rank = 1
72 END
73 ),
74 -- Provide (source, arch, suite) tuple of all binary packages to remain
75 remain_binaries AS (
76 SELECT
77 *
78 FROM (
79 SELECT
80 binaries.id,
81 binaries.architecture AS arch_id,
82 bin_associations.suite AS suite_id,
83 source.id AS source_id,
84 architecture.arch_string AS arch,
85 -- arch of newest version
86 first_value(architecture.arch_string) OVER (
87 PARTITION BY binaries.package, bin_associations.suite
88 ORDER BY binaries.version DESC
89 ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
90 ) as arch_first,
91 -- generate rank over versions of a source package in one suite
92 -- "1" being the newest
93 -- if newest package is arch-any, we use the rank only over current arch
94 dense_rank() OVER (
95 PARTITION BY binaries.package, binaries.architecture, bin_associations.suite
96 ORDER BY binaries.version DESC
97 ) AS version_rank_any,
98 -- if newest package is arch-all, we use the rank over all arches
99 -- this makes it possible to replace all by any and any by all
100 dense_rank() OVER (
101 PARTITION BY binaries.package, bin_associations.suite
102 ORDER BY binaries.version DESC
103 ) AS version_rank_all,
104 bin_associations.created AS created,
105 suite.stayofexecution as stayofexecution
106 FROM
107 binaries
108 INNER JOIN source ON source.id = binaries.source
109 INNER JOIN bin_associations ON
110 bin_associations.bin = binaries.id
111 AND bin_associations.suite = ANY(:suite_ids)
112 INNER JOIN architecture ON architecture.id = binaries.architecture
113 INNER JOIN suite ON
114 bin_associations.suite = suite.id
115 ) AS source_rank
116 WHERE
117 -- we only want to retain the newest of each, but we want to publish
118 -- all binaries in the archive for at least a day
119 CASE
120 WHEN now() BETWEEN created AND created + stayofexecution THEN TRUE
121 WHEN arch != 'all' AND arch_first != 'all' THEN version_rank_any = 1
122 ELSE version_rank_all = 1
123 END
124 ),
125 -- Figure out which source we should remove
126 -- A binary forces the corresponding source to remain
127 dominate_source AS (
128 SELECT
129 source.source AS source_package,
130 source.version AS source_version,
131 source.source AS package,
132 source.version,
133 'source'::text AS arch,
134 suite.suite_name AS suite,
135 src_associations.id AS assoc_id
136 FROM
137 source
138 INNER JOIN src_associations ON
139 src_associations.source = source.id
140 AND src_associations.suite = ANY(:suite_ids)
141 INNER join suite ON suite.id = src_associations.suite
142 LEFT JOIN remain_binaries ON
143 remain_binaries.source_id = source.id
144 AND remain_binaries.suite_id = src_associations.suite
145 LEFT JOIN remain_source ON
146 remain_source.source_id = source.id
147 AND remain_source.suite_id = src_associations.suite
148 WHERE
149 remain_binaries.source_id IS NULL
150 AND remain_source.source_id IS NULL
151 ),
152 -- Figure out which arch-any binaries we should remove
153 dominate_binaries AS (
154 SELECT
155 source.source AS source_package,
156 source.version AS source_version,
157 binaries.package AS package,
158 binaries.version,
159 architecture.arch_string AS arch,
160 suite.suite_name AS suite,
161 bin_associations.id AS assoc_id
162 FROM
163 binaries
164 INNER JOIN source ON source.id = binaries.source
165 INNER JOIN bin_associations ON
166 bin_associations.bin = binaries.id
167 AND bin_associations.suite = ANY(:suite_ids)
168 INNER JOIN architecture ON architecture.id = binaries.architecture
169 INNER join suite ON suite.id = bin_associations.suite
170 LEFT JOIN remain_binaries ON
171 remain_binaries.id = binaries.id
172 AND remain_binaries.arch_id = binaries.architecture
173 AND remain_binaries.suite_id = bin_associations.suite
174 WHERE
175 remain_binaries.source_id IS NULL
176 AND binaries.architecture != (SELECT id from architecture WHERE arch_string = 'all')
177 ),
178 -- Figure out which arch-all binaries we should remove
179 -- A arch-any binary forces the related arch-all binaries to remain
180 dominate_binaries_all AS (
181 SELECT
182 source.source AS source_package,
183 source.version AS source_version,
184 binaries.package AS package,
185 binaries.version,
186 architecture.arch_string AS arch,
187 suite.suite_name AS suite,
188 bin_associations.id AS assoc_id
189 FROM
190 binaries
191 INNER JOIN source ON source.id = binaries.source
192 INNER JOIN bin_associations ON
193 bin_associations.bin = binaries.id
194 AND bin_associations.suite = ANY(:suite_ids)
195 INNER JOIN architecture ON architecture.id = binaries.architecture
196 INNER join suite ON suite.id = bin_associations.suite
197 LEFT JOIN remain_binaries ON
198 remain_binaries.id = binaries.id
199 AND remain_binaries.arch_id = binaries.architecture
200 AND remain_binaries.suite_id = bin_associations.suite
201 LEFT JOIN remain_binaries AS remain_binaries_any ON
202 remain_binaries_any.source_id = source.id
203 AND remain_binaries_any.suite_id = bin_associations.suite
204 AND remain_binaries_any.arch_id != (SELECT id from architecture WHERE arch_string = 'all')
205 WHERE
206 remain_binaries.source_id IS NULL
207 AND remain_binaries_any.source_id IS NULL
208 AND binaries.architecture = (SELECT id from architecture WHERE arch_string = 'all')
209 )
210SELECT
211 *
212 FROM
213 dominate_source
214 UNION SELECT
215 *
216 FROM
217 dominate_binaries
218 UNION SELECT
219 *
220 FROM
221 dominate_binaries_all
222 ORDER BY
223 source_package, source_version, package, version, arch, suite
224"""
225 ).params(
226 suite_ids=[s.suite_id for s in suites],
227 )
228 )
231def delete_associations_table(table, ids, session):
232 result = session.execute(
233 text(
234 """
235 DELETE
236 FROM {}
237 WHERE id = ANY(:assoc_ids)
238 """.format(
239 table
240 )
241 ).params(
242 assoc_ids=list(ids),
243 )
244 )
246 assert result.rowcount == len(
247 ids
248 ), "Rows deleted are not equal to deletion requests"
251def delete_associations(assocs, session):
252 ids_bin = set()
253 ids_src = set()
255 for e in assocs:
256 Logger.log(["newer", e.package, e.version, e.suite, e.arch, e.assoc_id])
258 if e.arch == "source":
259 ids_src.add(e.assoc_id)
260 else:
261 ids_bin.add(e.assoc_id)
263 delete_associations_table("bin_associations", ids_bin, session)
264 delete_associations_table("src_associations", ids_src, session)
267def usage():
268 print(
269 """Usage: dak dominate [OPTIONS]
270Remove obsolete source and binary associations from suites.
272 -s, --suite=SUITE act on this suite
273 -h, --help show this help and exit
274 -n, --no-action don't commit changes
275 -f, --force also clean up untouchable suites
277SUITE can be comma (or space) separated list, e.g.
278 --suite=testing,unstable"""
279 )
280 sys.exit()
283def main():
284 global Options, Logger
285 cnf = Config()
286 Arguments = [
287 ("h", "help", "Obsolete::Options::Help"),
288 ("s", "suite", "Obsolete::Options::Suite", "HasArg"),
289 ("n", "no-action", "Obsolete::Options::No-Action"),
290 ("f", "force", "Obsolete::Options::Force"),
291 ]
292 cnf["Obsolete::Options::Help"] = ""
293 cnf["Obsolete::Options::No-Action"] = ""
294 cnf["Obsolete::Options::Force"] = ""
295 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) # type: ignore[attr-defined]
296 Options = cnf.subtree("Obsolete::Options")
297 if Options["Help"]:
298 usage()
300 if not Options["No-Action"]: 300 ↛ 302line 300 didn't jump to line 302 because the condition on line 300 was always true
301 Logger = daklog.Logger("dominate")
302 session = DBConn().session()
304 suites_query = (
305 select(Suite)
306 .order_by(Suite.suite_name)
307 .where(~exists().where(Suite.suite_id == PolicyQueue.suite_id))
308 )
309 if "Suite" in Options: 309 ↛ 310line 309 didn't jump to line 310 because the condition on line 309 was never true
310 suites_query = suites_query.where(
311 Suite.suite_name.in_(utils.split_args(Options["Suite"]))
312 )
313 if not Options["Force"]: 313 ↛ 315line 313 didn't jump to line 315 because the condition on line 313 was always true
314 suites_query = suites_query.filter_by(untouchable=False)
315 suites = list(session.scalars(suites_query))
317 assocs = list(retrieve_associations(suites, session))
319 if Options["No-Action"]: 319 ↛ 320line 319 didn't jump to line 320 because the condition on line 319 was never true
320 headers = (
321 "source package",
322 "source version",
323 "package",
324 "version",
325 "arch",
326 "suite",
327 "id",
328 )
329 print(tabulate(assocs, headers, tablefmt="orgtbl"))
330 session.rollback()
332 else:
333 delete_associations(assocs, session)
334 session.commit()
336 if Logger: 336 ↛ exitline 336 didn't return from function 'main' because the condition on line 336 was always true
337 Logger.close()