1#! /usr/bin/env python3
3"""
4Generate changelog entry between two suites
6@contact: Debian FTP Master <ftpmaster@debian.org>
7@copyright: 2010 Luca Falavigna <dktrkranz@debian.org>
8@license: GNU General Public License version 2 or later
9"""
11# This program is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation; either version 2 of the License, or
14# (at your option) any later version.
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
21# You should have received a copy of the GNU General Public License
22# along with this program; if not, write to the Free Software
23# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25################################################################################
27# <bdefreese> !dinstall
28# <dak> bdefreese: I guess the next dinstall will be in 0hr 1min 35sec
29# <bdefreese> Wow I have great timing
30# <DktrKranz> dating with dinstall, part II
31# <bdefreese> heh
32# <Ganneff> dating with that monster? do you have good combat armor?
33# <bdefreese> +5 Plate :)
34# <Ganneff> not a good one then
35# <Ganneff> so you wont even manage to bypass the lesser monster in front, unchecked
36# <DktrKranz> asbesto belt
37# <Ganneff> helps only a step
38# <DktrKranz> the Ultimate Weapon: cron_turned_off
39# <bdefreese> heh
40# <Ganneff> thats debadmin limited
41# <Ganneff> no option for you
42# <DktrKranz> bdefreese: it seems ftp-masters want dinstall to sexual harass us, are you good in running?
43# <Ganneff> you can run but you can not hide
44# <bdefreese> No, I'm old and fat :)
45# <Ganneff> you can roll but you can not hide
46# <Ganneff> :)
47# <bdefreese> haha
48# <DktrKranz> damn dinstall, you racist bastard
50################################################################################
52import os
53import sys
54from glob import glob
55from shutil import rmtree
57import apt_pkg
58from yaml import safe_dump
60from daklib import utils
61from daklib.contents import UnpackedSource
62from daklib.dbconn import Archive, DBConn, get_suite
63from daklib.regexes import re_no_epoch
65################################################################################
67filelist = "filelist.yaml"
70def usage(exit_code=0):
71 print(
72 """Generate changelog between two suites
74 Usage:
75 make-changelog -s <suite> -b <base_suite> [OPTION]...
76 make-changelog -e -a <archive>
78Options:
80 -h, --help show this help and exit
81 -s, --suite suite providing packages to compare
82 -b, --base-suite suite to be taken as reference for comparison
83 -n, --binnmu display binNMUs uploads instead of source ones
85 -e, --export export interesting files from source packages
86 -a, --archive archive to fetch data from
87 -p, --progress display progress status"""
88 )
90 sys.exit(exit_code)
93def get_source_uploads(suite, base_suite, session):
94 """
95 Returns changelogs for source uploads where version is newer than base.
96 """
98 query = """WITH base AS (
99 SELECT source, max(version) AS version
100 FROM source_suite
101 WHERE suite_name = :base_suite
102 GROUP BY source
103 UNION (SELECT source, CAST(0 AS debversion) AS version
104 FROM source_suite
105 WHERE suite_name = :suite
106 EXCEPT SELECT source, CAST(0 AS debversion) AS version
107 FROM source_suite
108 WHERE suite_name = :base_suite
109 ORDER BY source)),
110 cur_suite AS (
111 SELECT source, max(version) AS version
112 FROM source_suite
113 WHERE suite_name = :suite
114 GROUP BY source)
115 SELECT DISTINCT c.source, c.version, c.changelog
116 FROM changelogs c
117 JOIN base b ON b.source = c.source
118 JOIN cur_suite cs ON cs.source = c.source
119 WHERE c.version > b.version
120 AND c.version <= cs.version
121 AND c.architecture LIKE '%source%'
122 ORDER BY c.source, c.version DESC"""
124 return session.execute(query, {"suite": suite, "base_suite": base_suite})
127def get_binary_uploads(suite, base_suite, session):
128 """
129 Returns changelogs for binary uploads where version is newer than base.
130 """
132 query = """WITH base as (
133 SELECT s.source, max(b.version) AS version, a.arch_string
134 FROM source s
135 JOIN binaries b ON b.source = s.id
136 JOIN bin_associations ba ON ba.bin = b.id
137 JOIN architecture a ON a.id = b.architecture
138 WHERE ba.suite = (
139 SELECT id
140 FROM suite
141 WHERE suite_name = :base_suite)
142 GROUP BY s.source, a.arch_string),
143 cur_suite as (
144 SELECT s.source, max(b.version) AS version, a.arch_string
145 FROM source s
146 JOIN binaries b ON b.source = s.id
147 JOIN bin_associations ba ON ba.bin = b.id
148 JOIN architecture a ON a.id = b.architecture
149 WHERE ba.suite = (
150 SELECT id
151 FROM suite
152 WHERE suite_name = :suite)
153 GROUP BY s.source, a.arch_string)
154 SELECT DISTINCT c.source, c.version, c.architecture, c.changelog
155 FROM changelogs c
156 JOIN base b on b.source = c.source
157 JOIN cur_suite cs ON cs.source = c.source
158 WHERE c.version > b.version
159 AND c.version <= cs.version
160 AND c.architecture = b.arch_string
161 AND c.architecture = cs.arch_string
162 ORDER BY c.source, c.version DESC, c.architecture"""
164 return session.execute(query, {"suite": suite, "base_suite": base_suite})
167def display_changes(uploads, index):
168 prev_upload = None
169 for upload in uploads:
170 if prev_upload and prev_upload != upload[0]:
171 print()
172 print(upload[index])
173 prev_upload = upload[0]
176def export_files(session, archive, clpool, progress=False):
177 """
178 Export interesting files from source packages.
179 """
180 pool = os.path.join(archive.path, "pool")
182 sources = {}
183 unpack = {}
184 files = ("changelog", "copyright", "NEWS", "NEWS.Debian", "README.Debian")
185 stats = {"unpack": 0, "created": 0, "removed": 0, "errors": 0, "files": 0}
186 query = """SELECT DISTINCT s.source, su.suite_name AS suite, s.version, c.name || '/' || f.filename AS filename
187 FROM source s
188 JOIN newest_source n ON n.source = s.source AND n.version = s.version
189 JOIN src_associations sa ON sa.source = s.id
190 JOIN suite su ON su.id = sa.suite
191 JOIN files f ON f.id = s.file
192 JOIN files_archive_map fam ON f.id = fam.file_id AND fam.archive_id = su.archive_id
193 JOIN component c ON fam.component_id = c.id
194 WHERE su.archive_id = :archive_id
195 ORDER BY s.source, suite"""
197 for p in session.execute(query, {"archive_id": archive.archive_id}):
198 if p[0] not in sources:
199 sources[p[0]] = {}
200 sources[p[0]][p[1]] = (re_no_epoch.sub("", p[2]), p[3])
202 for p in sources.keys():
203 for s in sources[p].keys():
204 path = os.path.join(clpool, "/".join(sources[p][s][1].split("/")[:-1]))
205 if not os.path.exists(path):
206 os.makedirs(path)
207 if not os.path.exists( 207 ↛ 214line 207 didn't jump to line 214, because the condition on line 207 was never false
208 os.path.join(path, "%s_%s_changelog" % (p, sources[p][s][0]))
209 ):
210 if os.path.join(pool, sources[p][s][1]) not in unpack:
211 unpack[os.path.join(pool, sources[p][s][1])] = (path, set())
212 unpack[os.path.join(pool, sources[p][s][1])][1].add(s)
213 else:
214 for file in glob("%s/%s_%s_*" % (path, p, sources[p][s][0])):
215 link = "%s%s" % (s, file.split("%s_%s" % (p, sources[p][s][0]))[1])
216 try:
217 os.unlink(os.path.join(path, link))
218 except OSError:
219 pass
220 os.link(os.path.join(path, file), os.path.join(path, link))
222 for p in unpack.keys():
223 package = os.path.splitext(os.path.basename(p))[0].split("_")
224 try:
225 unpacked = UnpackedSource(p, clpool)
226 tempdir = unpacked.get_root_directory()
227 stats["unpack"] += 1
228 if progress: 228 ↛ 229line 228 didn't jump to line 229, because the condition on line 228 was never true
229 if stats["unpack"] % 100 == 0:
230 print("%d packages unpacked" % stats["unpack"], file=sys.stderr)
231 elif stats["unpack"] % 10 == 0:
232 print(".", end="", file=sys.stderr)
233 for file in files:
234 for f in glob(os.path.join(tempdir, "debian", "*%s" % file)):
235 for s in unpack[p][1]:
236 suite = os.path.join(
237 unpack[p][0], "%s_%s" % (s, os.path.basename(f))
238 )
239 version = os.path.join(
240 unpack[p][0],
241 "%s_%s_%s" % (package[0], package[1], os.path.basename(f)),
242 )
243 if not os.path.exists(version):
244 os.link(f, version)
245 stats["created"] += 1
246 try:
247 os.unlink(suite)
248 except OSError:
249 pass
250 os.link(version, suite)
251 stats["created"] += 1
252 unpacked.cleanup()
253 except Exception as e:
254 print("make-changelog: unable to unpack %s\n%s" % (p, e))
255 stats["errors"] += 1
257 for root, dirs, files in os.walk(clpool, topdown=False):
258 files = [f for f in files if f != filelist]
259 if len(files):
260 if root != clpool: 260 ↛ 265line 260 didn't jump to line 265, because the condition on line 260 was never false
261 if root.split("/")[-1] not in sources: 261 ↛ 262line 261 didn't jump to line 262, because the condition on line 261 was never true
262 if os.path.exists(root):
263 stats["removed"] += len(os.listdir(root))
264 rmtree(root)
265 for file in files:
266 if os.path.exists(os.path.join(root, file)): 266 ↛ 265line 266 didn't jump to line 265, because the condition on line 266 was never false
267 if os.stat(os.path.join(root, file)).st_nlink == 1: 267 ↛ 268line 267 didn't jump to line 268, because the condition on line 267 was never true
268 stats["removed"] += 1
269 os.unlink(os.path.join(root, file))
270 for dir in dirs:
271 try:
272 os.rmdir(os.path.join(root, dir))
273 except OSError:
274 pass
275 stats["files"] += len(files)
276 stats["files"] -= stats["removed"]
278 print("make-changelog: file exporting finished")
279 print(" * New packages unpacked: %d" % stats["unpack"])
280 print(" * New files created: %d" % stats["created"])
281 print(" * New files removed: %d" % stats["removed"])
282 print(" * Unpack errors: %d" % stats["errors"])
283 print(" * Files available into changelog pool: %d" % stats["files"])
286def generate_export_filelist(clpool):
287 clfiles = {}
288 for root, dirs, files in os.walk(clpool):
289 for file in [f for f in files if f != filelist]:
290 clpath = os.path.join(root, file).replace(clpool, "").strip("/")
291 source = clpath.split("/")[2]
292 elements = clpath.split("/")[3].split("_")
293 if source not in clfiles:
294 clfiles[source] = {}
295 if elements[0] == source:
296 if elements[1] not in clfiles[source]: 296 ↛ 298line 296 didn't jump to line 298, because the condition on line 296 was never false
297 clfiles[source][elements[1]] = []
298 clfiles[source][elements[1]].append(clpath)
299 else:
300 if elements[0] not in clfiles[source]: 300 ↛ 302line 300 didn't jump to line 302, because the condition on line 300 was never false
301 clfiles[source][elements[0]] = []
302 clfiles[source][elements[0]].append(clpath)
303 with open(os.path.join(clpool, filelist), "w+") as fd:
304 safe_dump(clfiles, fd, default_flow_style=False)
307def main():
308 Cnf = utils.get_conf()
309 Arguments = [
310 ("h", "help", "Make-Changelog::Options::Help"),
311 ("a", "archive", "Make-Changelog::Options::Archive", "HasArg"),
312 ("s", "suite", "Make-Changelog::Options::Suite", "HasArg"),
313 ("b", "base-suite", "Make-Changelog::Options::Base-Suite", "HasArg"),
314 ("n", "binnmu", "Make-Changelog::Options::binNMU"),
315 ("e", "export", "Make-Changelog::Options::export"),
316 ("p", "progress", "Make-Changelog::Options::progress"),
317 ]
319 for i in ["help", "suite", "base-suite", "binnmu", "export", "progress"]:
320 key = "Make-Changelog::Options::%s" % i
321 if key not in Cnf: 321 ↛ 319line 321 didn't jump to line 319, because the condition on line 321 was never false
322 Cnf[key] = ""
324 apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
325 Options = Cnf.subtree("Make-Changelog::Options")
326 suite = Cnf["Make-Changelog::Options::Suite"]
327 base_suite = Cnf["Make-Changelog::Options::Base-Suite"]
328 binnmu = Cnf["Make-Changelog::Options::binNMU"]
329 export = Cnf["Make-Changelog::Options::export"]
330 progress = Cnf["Make-Changelog::Options::progress"]
332 if Options["help"] or not (suite and base_suite) and not export:
333 usage()
335 for s in suite, base_suite:
336 if not export and not get_suite(s): 336 ↛ 337line 336 didn't jump to line 337, because the condition on line 336 was never true
337 utils.fubar('Invalid suite "%s"' % s)
339 session = DBConn().session()
341 if export:
342 archive = (
343 session.query(Archive).filter_by(archive_name=Options["Archive"]).one()
344 )
345 exportpath = archive.changelog
346 if exportpath: 346 ↛ 350line 346 didn't jump to line 350, because the condition on line 346 was never false
347 export_files(session, archive, exportpath, progress)
348 generate_export_filelist(exportpath)
349 else:
350 utils.fubar("No changelog export path defined")
351 elif binnmu: 351 ↛ 352line 351 didn't jump to line 352, because the condition on line 351 was never true
352 display_changes(get_binary_uploads(suite, base_suite, session), 3)
353 else:
354 display_changes(get_source_uploads(suite, base_suite, session), 2)
356 session.commit()
359if __name__ == "__main__":
360 main()