Coverage for dak/dominate.py: 87%
55 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-01-04 16:18 +0000
1#! /usr/bin/env python3
3"""
4Remove obsolete source and binary associations from suites.
6@contact: Debian FTP Master <ftpmaster@debian.org>
7@copyright: 2009 Torsten Werner <twerner@debian.org>
8@license: GNU General Public License version 2 or later
9"""
11# This program is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation; either version 2 of the License, or
14# (at your option) any later version.
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
21# You should have received a copy of the GNU General Public License
22# along with this program; if not, write to the Free Software
23# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25import sys
27import apt_pkg
28from sqlalchemy.sql import exists, text
29from tabulate import tabulate
31from daklib import daklog, utils
32from daklib.config import Config
33from daklib.dbconn import DBConn, PolicyQueue, Suite
35Options: apt_pkg.Configuration
36Logger: daklog.Logger
39def retrieve_associations(suites, session):
40 return session.execute(
41 text(
42 """
43WITH
44 -- Provide (source, suite) tuple of all source packages to remain
45 remain_source AS (
46 SELECT
47 *
48 FROM (
49 SELECT
50 source.id AS source_id,
51 src_associations.suite AS suite_id,
52 -- generate rank over versions of a source package in one suite
53 -- "1" being the newest
54 dense_rank() OVER (
55 PARTITION BY source.source, src_associations.suite
56 ORDER BY source.version DESC
57 ) AS version_rank,
58 src_associations.created AS created,
59 suite.stayofexecution as stayofexecution
60 FROM
61 source
62 INNER JOIN src_associations ON
63 src_associations.source = source.id
64 AND src_associations.suite = ANY(:suite_ids)
65 INNER JOIN suite ON
66 src_associations.suite = suite.id
67 ) AS source_ranked
68 WHERE
69 -- we only want to retain the newest of each, but we want to publish
70 -- all sources in the archive for at least a day
71 CASE
72 WHEN now() BETWEEN created AND created + stayofexecution THEN TRUE
73 ELSE version_rank = 1
74 END
75 ),
76 -- Provide (source, arch, suite) tuple of all binary packages to remain
77 remain_binaries AS (
78 SELECT
79 *
80 FROM (
81 SELECT
82 binaries.id,
83 binaries.architecture AS arch_id,
84 bin_associations.suite AS suite_id,
85 source.id AS source_id,
86 architecture.arch_string AS arch,
87 -- arch of newest version
88 first_value(architecture.arch_string) OVER (
89 PARTITION BY binaries.package, bin_associations.suite
90 ORDER BY binaries.version DESC
91 ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
92 ) as arch_first,
93 -- generate rank over versions of a source package in one suite
94 -- "1" being the newest
95 -- if newest package is arch-any, we use the rank only over current arch
96 dense_rank() OVER (
97 PARTITION BY binaries.package, binaries.architecture, bin_associations.suite
98 ORDER BY binaries.version DESC
99 ) AS version_rank_any,
100 -- if newest package is arch-all, we use the rank over all arches
101 -- this makes it possible to replace all by any and any by all
102 dense_rank() OVER (
103 PARTITION BY binaries.package, bin_associations.suite
104 ORDER BY binaries.version DESC
105 ) AS version_rank_all,
106 bin_associations.created AS created,
107 suite.stayofexecution as stayofexecution
108 FROM
109 binaries
110 INNER JOIN source ON source.id = binaries.source
111 INNER JOIN bin_associations ON
112 bin_associations.bin = binaries.id
113 AND bin_associations.suite = ANY(:suite_ids)
114 INNER JOIN architecture ON architecture.id = binaries.architecture
115 INNER JOIN suite ON
116 bin_associations.suite = suite.id
117 ) AS source_rank
118 WHERE
119 -- we only want to retain the newest of each, but we want to publish
120 -- all binaries in the archive for at least a day
121 CASE
122 WHEN now() BETWEEN created AND created + stayofexecution THEN TRUE
123 WHEN arch != 'all' AND arch_first != 'all' THEN version_rank_any = 1
124 ELSE version_rank_all = 1
125 END
126 ),
127 -- Figure out which source we should remove
128 -- A binary forces the corresponding source to remain
129 dominate_source AS (
130 SELECT
131 source.source AS source_package,
132 source.version AS source_version,
133 source.source AS package,
134 source.version,
135 'source'::text AS arch,
136 suite.suite_name AS suite,
137 src_associations.id AS assoc_id
138 FROM
139 source
140 INNER JOIN src_associations ON
141 src_associations.source = source.id
142 AND src_associations.suite = ANY(:suite_ids)
143 INNER join suite ON suite.id = src_associations.suite
144 LEFT JOIN remain_binaries ON
145 remain_binaries.source_id = source.id
146 AND remain_binaries.suite_id = src_associations.suite
147 LEFT JOIN remain_source ON
148 remain_source.source_id = source.id
149 AND remain_source.suite_id = src_associations.suite
150 WHERE
151 remain_binaries.source_id IS NULL
152 AND remain_source.source_id IS NULL
153 ),
154 -- Figure out which arch-any binaries we should remove
155 dominate_binaries AS (
156 SELECT
157 source.source AS source_package,
158 source.version AS source_version,
159 binaries.package AS package,
160 binaries.version,
161 architecture.arch_string AS arch,
162 suite.suite_name AS suite,
163 bin_associations.id AS assoc_id
164 FROM
165 binaries
166 INNER JOIN source ON source.id = binaries.source
167 INNER JOIN bin_associations ON
168 bin_associations.bin = binaries.id
169 AND bin_associations.suite = ANY(:suite_ids)
170 INNER JOIN architecture ON architecture.id = binaries.architecture
171 INNER join suite ON suite.id = bin_associations.suite
172 LEFT JOIN remain_binaries ON
173 remain_binaries.id = binaries.id
174 AND remain_binaries.arch_id = binaries.architecture
175 AND remain_binaries.suite_id = bin_associations.suite
176 WHERE
177 remain_binaries.source_id IS NULL
178 AND binaries.architecture != (SELECT id from architecture WHERE arch_string = 'all')
179 ),
180 -- Figure out which arch-all binaries we should remove
181 -- A arch-any binary forces the related arch-all binaries to remain
182 dominate_binaries_all AS (
183 SELECT
184 source.source AS source_package,
185 source.version AS source_version,
186 binaries.package AS package,
187 binaries.version,
188 architecture.arch_string AS arch,
189 suite.suite_name AS suite,
190 bin_associations.id AS assoc_id
191 FROM
192 binaries
193 INNER JOIN source ON source.id = binaries.source
194 INNER JOIN bin_associations ON
195 bin_associations.bin = binaries.id
196 AND bin_associations.suite = ANY(:suite_ids)
197 INNER JOIN architecture ON architecture.id = binaries.architecture
198 INNER join suite ON suite.id = bin_associations.suite
199 LEFT JOIN remain_binaries ON
200 remain_binaries.id = binaries.id
201 AND remain_binaries.arch_id = binaries.architecture
202 AND remain_binaries.suite_id = bin_associations.suite
203 LEFT JOIN remain_binaries AS remain_binaries_any ON
204 remain_binaries_any.source_id = source.id
205 AND remain_binaries_any.suite_id = bin_associations.suite
206 AND remain_binaries_any.arch_id != (SELECT id from architecture WHERE arch_string = 'all')
207 WHERE
208 remain_binaries.source_id IS NULL
209 AND remain_binaries_any.source_id IS NULL
210 AND binaries.architecture = (SELECT id from architecture WHERE arch_string = 'all')
211 )
212SELECT
213 *
214 FROM
215 dominate_source
216 UNION SELECT
217 *
218 FROM
219 dominate_binaries
220 UNION SELECT
221 *
222 FROM
223 dominate_binaries_all
224 ORDER BY
225 source_package, source_version, package, version, arch, suite
226"""
227 ).params(
228 suite_ids=[s.suite_id for s in suites],
229 )
230 )
233def delete_associations_table(table, ids, session):
234 result = session.execute(
235 text(
236 """
237 DELETE
238 FROM {}
239 WHERE id = ANY(:assoc_ids)
240 """.format(
241 table
242 )
243 ).params(
244 assoc_ids=list(ids),
245 )
246 )
248 assert result.rowcount == len(
249 ids
250 ), "Rows deleted are not equal to deletion requests"
253def delete_associations(assocs, session):
254 ids_bin = set()
255 ids_src = set()
257 for e in assocs:
258 Logger.log(["newer", e.package, e.version, e.suite, e.arch, e.assoc_id])
260 if e.arch == "source":
261 ids_src.add(e.assoc_id)
262 else:
263 ids_bin.add(e.assoc_id)
265 delete_associations_table("bin_associations", ids_bin, session)
266 delete_associations_table("src_associations", ids_src, session)
269def usage():
270 print(
271 """Usage: dak dominate [OPTIONS]
272Remove obsolete source and binary associations from suites.
274 -s, --suite=SUITE act on this suite
275 -h, --help show this help and exit
276 -n, --no-action don't commit changes
277 -f, --force also clean up untouchable suites
279SUITE can be comma (or space) separated list, e.g.
280 --suite=testing,unstable"""
281 )
282 sys.exit()
285def main():
286 global Options, Logger
287 cnf = Config()
288 Arguments = [
289 ("h", "help", "Obsolete::Options::Help"),
290 ("s", "suite", "Obsolete::Options::Suite", "HasArg"),
291 ("n", "no-action", "Obsolete::Options::No-Action"),
292 ("f", "force", "Obsolete::Options::Force"),
293 ]
294 cnf["Obsolete::Options::Help"] = ""
295 cnf["Obsolete::Options::No-Action"] = ""
296 cnf["Obsolete::Options::Force"] = ""
297 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) # type: ignore[attr-defined]
298 Options = cnf.subtree("Obsolete::Options")
299 if Options["Help"]:
300 usage()
302 if not Options["No-Action"]: 302 ↛ 304line 302 didn't jump to line 304 because the condition on line 302 was always true
303 Logger = daklog.Logger("dominate")
304 session = DBConn().session()
306 suites_query = (
307 session.query(Suite)
308 .order_by(Suite.suite_name)
309 .filter(~exists().where(Suite.suite_id == PolicyQueue.suite_id))
310 )
311 if "Suite" in Options: 311 ↛ 312line 311 didn't jump to line 312 because the condition on line 311 was never true
312 suites_query = suites_query.filter(
313 Suite.suite_name.in_(utils.split_args(Options["Suite"]))
314 )
315 if not Options["Force"]: 315 ↛ 317line 315 didn't jump to line 317 because the condition on line 315 was always true
316 suites_query = suites_query.filter_by(untouchable=False)
317 suites = suites_query.all()
319 assocs = list(retrieve_associations(suites, session))
321 if Options["No-Action"]: 321 ↛ 322line 321 didn't jump to line 322 because the condition on line 321 was never true
322 headers = (
323 "source package",
324 "source version",
325 "package",
326 "version",
327 "arch",
328 "suite",
329 "id",
330 )
331 print((tabulate(assocs, headers, tablefmt="orgtbl")))
332 session.rollback()
334 else:
335 delete_associations(assocs, session)
336 session.commit()
338 if Logger: 338 ↛ exitline 338 didn't return from function 'main' because the condition on line 338 was always true
339 Logger.close()
342if __name__ == "__main__":
343 main()