Coverage for dak/make_changelog.py: 83%
157 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-08-03 16:46 +0000
1"""
2Generate changelog entry between two suites
4@contact: Debian FTP Master <ftpmaster@debian.org>
5@copyright: 2010 Luca Falavigna <dktrkranz@debian.org>
6@license: GNU General Public License version 2 or later
7"""
9# This program is free software; you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation; either version 2 of the License, or
12# (at your option) any later version.
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
19# You should have received a copy of the GNU General Public License
20# along with this program; if not, write to the Free Software
21# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23################################################################################
25# <bdefreese> !dinstall
26# <dak> bdefreese: I guess the next dinstall will be in 0hr 1min 35sec
27# <bdefreese> Wow I have great timing
28# <DktrKranz> dating with dinstall, part II
29# <bdefreese> heh
30# <Ganneff> dating with that monster? do you have good combat armor?
31# <bdefreese> +5 Plate :)
32# <Ganneff> not a good one then
33# <Ganneff> so you wont even manage to bypass the lesser monster in front, unchecked
34# <DktrKranz> asbesto belt
35# <Ganneff> helps only a step
36# <DktrKranz> the Ultimate Weapon: cron_turned_off
37# <bdefreese> heh
38# <Ganneff> thats debadmin limited
39# <Ganneff> no option for you
40# <DktrKranz> bdefreese: it seems ftp-masters want dinstall to sexual harass us, are you good in running?
41# <Ganneff> you can run but you can not hide
42# <bdefreese> No, I'm old and fat :)
43# <Ganneff> you can roll but you can not hide
44# <Ganneff> :)
45# <bdefreese> haha
46# <DktrKranz> damn dinstall, you racist bastard
48################################################################################
50import os
51import sys
52from glob import glob
53from shutil import rmtree
54from typing import TYPE_CHECKING, NoReturn
56import apt_pkg
57from sqlalchemy import select, sql
58from yaml import safe_dump
60from daklib import utils
61from daklib.contents import UnpackedSource
62from daklib.dbconn import Archive, DBConn, get_suite
63from daklib.regexes import re_no_epoch
65if TYPE_CHECKING:
66 from sqlalchemy.engine import Result
67 from sqlalchemy.orm import Session
69################################################################################
71filelist = "filelist.yaml"
74def usage(exit_code=0) -> NoReturn:
75 print(
76 """Generate changelog between two suites
78 Usage:
79 make-changelog -s <suite> -b <base_suite> [OPTION]...
80 make-changelog -e -a <archive>
82Options:
84 -h, --help show this help and exit
85 -s, --suite suite providing packages to compare
86 -b, --base-suite suite to be taken as reference for comparison
87 -n, --binnmu display binNMUs uploads instead of source ones
89 -e, --export export interesting files from source packages
90 -a, --archive archive to fetch data from
91 -p, --progress display progress status"""
92 )
94 sys.exit(exit_code)
97def get_source_uploads(
98 suite: str, base_suite: str, session: "Session"
99) -> "Result[tuple[str, str, str]]":
100 """
101 Returns changelogs for source uploads where version is newer than base.
102 """
104 query = """WITH base AS (
105 SELECT source, max(version) AS version
106 FROM source_suite
107 WHERE suite_name = :base_suite
108 GROUP BY source
109 UNION (SELECT source, CAST(0 AS debversion) AS version
110 FROM source_suite
111 WHERE suite_name = :suite
112 EXCEPT SELECT source, CAST(0 AS debversion) AS version
113 FROM source_suite
114 WHERE suite_name = :base_suite
115 ORDER BY source)),
116 cur_suite AS (
117 SELECT source, max(version) AS version
118 FROM source_suite
119 WHERE suite_name = :suite
120 GROUP BY source)
121 SELECT DISTINCT c.source, c.version, c.changelog
122 FROM changelogs c
123 JOIN base b ON b.source = c.source
124 JOIN cur_suite cs ON cs.source = c.source
125 WHERE c.version > b.version
126 AND c.version <= cs.version
127 AND c.architecture LIKE '%source%'
128 ORDER BY c.source, c.version DESC"""
130 return session.execute(sql.text(query), {"suite": suite, "base_suite": base_suite})
133def get_binary_uploads(
134 suite: str, base_suite: str, session: "Session"
135) -> "Result[tuple[str, str, str, str]]":
136 """
137 Returns changelogs for binary uploads where version is newer than base.
138 """
140 query = """WITH base as (
141 SELECT s.source, max(b.version) AS version, a.arch_string
142 FROM source s
143 JOIN binaries b ON b.source = s.id
144 JOIN bin_associations ba ON ba.bin = b.id
145 JOIN architecture a ON a.id = b.architecture
146 WHERE ba.suite = (
147 SELECT id
148 FROM suite
149 WHERE suite_name = :base_suite)
150 GROUP BY s.source, a.arch_string),
151 cur_suite as (
152 SELECT s.source, max(b.version) AS version, a.arch_string
153 FROM source s
154 JOIN binaries b ON b.source = s.id
155 JOIN bin_associations ba ON ba.bin = b.id
156 JOIN architecture a ON a.id = b.architecture
157 WHERE ba.suite = (
158 SELECT id
159 FROM suite
160 WHERE suite_name = :suite)
161 GROUP BY s.source, a.arch_string)
162 SELECT DISTINCT c.source, c.version, c.architecture, c.changelog
163 FROM changelogs c
164 JOIN base b on b.source = c.source
165 JOIN cur_suite cs ON cs.source = c.source
166 WHERE c.version > b.version
167 AND c.version <= cs.version
168 AND c.architecture = b.arch_string
169 AND c.architecture = cs.arch_string
170 ORDER BY c.source, c.version DESC, c.architecture"""
172 return session.execute(sql.text(query), {"suite": suite, "base_suite": base_suite})
175def display_changes(uploads, index):
176 prev_upload = None
177 for upload in uploads:
178 if prev_upload and prev_upload != upload[0]:
179 print()
180 print(upload[index])
181 prev_upload = upload[0]
184def export_files(
185 session: "Session", archive: Archive, clpool: str, progress=False
186) -> None:
187 """
188 Export interesting files from source packages.
189 """
190 pool = os.path.join(archive.path, "pool")
192 sources: dict[str, dict[str, tuple[str, str]]] = {}
193 unpack: dict[str, tuple[str, set[str]]] = {}
194 files = ("changelog", "copyright", "NEWS", "NEWS.Debian", "README.Debian")
195 stats = {"unpack": 0, "created": 0, "removed": 0, "errors": 0, "files": 0}
196 query = """SELECT DISTINCT s.source, su.suite_name AS suite, s.version, c.name || '/' || f.filename AS filename
197 FROM source s
198 JOIN newest_source n ON n.source = s.source AND n.version = s.version
199 JOIN src_associations sa ON sa.source = s.id
200 JOIN suite su ON su.id = sa.suite
201 JOIN files f ON f.id = s.file
202 JOIN files_archive_map fam ON f.id = fam.file_id AND fam.archive_id = su.archive_id
203 JOIN component c ON fam.component_id = c.id
204 WHERE su.archive_id = :archive_id
205 ORDER BY s.source, suite"""
207 for row in session.execute(sql.text(query), {"archive_id": archive.archive_id}):
208 if row[0] not in sources:
209 sources[row[0]] = {}
210 sources[row[0]][row[1]] = (re_no_epoch.sub("", row[2]), row[3])
212 for p, sources_value in sources.items():
213 for s, (s_version, s_filename) in sources_value.items():
214 path = os.path.join(clpool, "/".join(s_filename.split("/")[:-1]))
215 if not os.path.exists(path):
216 os.makedirs(path)
217 if not os.path.exists( 217 ↛ 224line 217 didn't jump to line 224 because the condition on line 217 was always true
218 os.path.join(path, "%s_%s_changelog" % (p, s_version))
219 ):
220 if os.path.join(pool, s_filename) not in unpack:
221 unpack[os.path.join(pool, s_filename)] = (path, set())
222 unpack[os.path.join(pool, s_filename)][1].add(s)
223 else:
224 for file in glob("%s/%s_%s_*" % (path, p, s_version)):
225 link = "%s%s" % (s, file.split("%s_%s" % (p, s_version))[1])
226 try:
227 os.unlink(os.path.join(path, link))
228 except OSError:
229 pass
230 os.link(os.path.join(path, file), os.path.join(path, link))
232 for p in unpack.keys():
233 package = os.path.splitext(os.path.basename(p))[0].split("_")
234 try:
235 unpacked = UnpackedSource(p, clpool)
236 tempdir = unpacked.get_root_directory()
237 stats["unpack"] += 1
238 if progress: 238 ↛ 239line 238 didn't jump to line 239 because the condition on line 238 was never true
239 if stats["unpack"] % 100 == 0:
240 print("%d packages unpacked" % stats["unpack"], file=sys.stderr)
241 elif stats["unpack"] % 10 == 0:
242 print(".", end="", file=sys.stderr)
243 for file in files:
244 for f in glob(os.path.join(tempdir, "debian", "*%s" % file)):
245 for s in unpack[p][1]:
246 suite = os.path.join(
247 unpack[p][0], "%s_%s" % (s, os.path.basename(f))
248 )
249 version = os.path.join(
250 unpack[p][0],
251 "%s_%s_%s" % (package[0], package[1], os.path.basename(f)),
252 )
253 if not os.path.exists(version):
254 os.link(f, version)
255 stats["created"] += 1
256 try:
257 os.unlink(suite)
258 except OSError:
259 pass
260 os.link(version, suite)
261 stats["created"] += 1
262 unpacked.cleanup()
263 except Exception as e:
264 print("make-changelog: unable to unpack %s\n%s" % (p, e))
265 stats["errors"] += 1
267 for root, dirs, files2 in os.walk(clpool, topdown=False):
268 files2 = [f for f in files2 if f != filelist]
269 if len(files2):
270 if ( 270 ↛ 275line 270 didn't jump to line 275
271 root != clpool
272 and root.split("/")[-1] not in sources
273 and os.path.exists(root)
274 ):
275 stats["removed"] += len(os.listdir(root))
276 rmtree(root)
277 for file in files2:
278 if ( 278 ↛ 282line 278 didn't jump to line 282
279 os.path.exists(os.path.join(root, file))
280 and os.stat(os.path.join(root, file)).st_nlink == 1
281 ):
282 stats["removed"] += 1
283 os.unlink(os.path.join(root, file))
284 for dir in dirs:
285 try:
286 os.rmdir(os.path.join(root, dir))
287 except OSError:
288 pass
289 stats["files"] += len(files2)
290 stats["files"] -= stats["removed"]
292 print("make-changelog: file exporting finished")
293 print(" * New packages unpacked: %d" % stats["unpack"])
294 print(" * New files created: %d" % stats["created"])
295 print(" * New files removed: %d" % stats["removed"])
296 print(" * Unpack errors: %d" % stats["errors"])
297 print(" * Files available into changelog pool: %d" % stats["files"])
300def generate_export_filelist(clpool: str) -> None:
301 clfiles: dict[str, dict[str, list[str]]] = {}
302 for root, dirs, files in os.walk(clpool):
303 for file in [f for f in files if f != filelist]:
304 clpath = os.path.join(root, file).replace(clpool, "").strip("/")
305 source = clpath.split("/")[2]
306 elements = clpath.split("/")[3].split("_")
307 if source not in clfiles:
308 clfiles[source] = {}
309 if elements[0] == source:
310 if elements[1] not in clfiles[source]: 310 ↛ 312line 310 didn't jump to line 312 because the condition on line 310 was always true
311 clfiles[source][elements[1]] = []
312 clfiles[source][elements[1]].append(clpath)
313 else:
314 if elements[0] not in clfiles[source]: 314 ↛ 316line 314 didn't jump to line 316 because the condition on line 314 was always true
315 clfiles[source][elements[0]] = []
316 clfiles[source][elements[0]].append(clpath)
317 with open(os.path.join(clpool, filelist), "w+") as fd:
318 safe_dump(clfiles, fd, default_flow_style=False)
321def main() -> None:
322 Cnf = utils.get_conf()
323 Arguments = [
324 ("h", "help", "Make-Changelog::Options::Help"),
325 ("a", "archive", "Make-Changelog::Options::Archive", "HasArg"),
326 ("s", "suite", "Make-Changelog::Options::Suite", "HasArg"),
327 ("b", "base-suite", "Make-Changelog::Options::Base-Suite", "HasArg"),
328 ("n", "binnmu", "Make-Changelog::Options::binNMU"),
329 ("e", "export", "Make-Changelog::Options::export"),
330 ("p", "progress", "Make-Changelog::Options::progress"),
331 ]
333 for i in ["help", "suite", "base-suite", "binnmu", "export", "progress"]:
334 key = "Make-Changelog::Options::%s" % i
335 if key not in Cnf: 335 ↛ 333line 335 didn't jump to line 333 because the condition on line 335 was always true
336 Cnf[key] = "" # type: ignore[index]
338 apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) # type: ignore[attr-defined]
339 Options = Cnf.subtree("Make-Changelog::Options") # type: ignore[attr-defined]
340 suite = Cnf["Make-Changelog::Options::Suite"]
341 base_suite = Cnf["Make-Changelog::Options::Base-Suite"]
342 binnmu = Cnf["Make-Changelog::Options::binNMU"]
343 export = Cnf["Make-Changelog::Options::export"]
344 progress = Cnf["Make-Changelog::Options::progress"]
346 if Options["help"] or not (suite and base_suite) and not export:
347 usage()
349 for s in suite, base_suite:
350 if not export and not get_suite(s): 350 ↛ 351line 350 didn't jump to line 351 because the condition on line 350 was never true
351 utils.fubar('Invalid suite "%s"' % s)
353 session = DBConn().session()
355 if export:
356 archive = session.execute(
357 select(Archive).filter_by(archive_name=Options["Archive"])
358 ).scalar_one()
359 exportpath = archive.changelog
360 if exportpath: 360 ↛ 364line 360 didn't jump to line 364 because the condition on line 360 was always true
361 export_files(session, archive, exportpath, progress)
362 generate_export_filelist(exportpath)
363 else:
364 utils.fubar("No changelog export path defined")
365 elif binnmu: 365 ↛ 366line 365 didn't jump to line 366 because the condition on line 365 was never true
366 display_changes(get_binary_uploads(suite, base_suite, session), 3)
367 else:
368 display_changes(get_source_uploads(suite, base_suite, session), 2)
370 session.commit()