1
2
3 """
4 Generate changelog entry between two suites
5
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2010 Luca Falavigna <dktrkranz@debian.org>
8 @license: GNU General Public License version 2 or later
9 """
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52 import os
53 import sys
54 from glob import glob
55 from shutil import rmtree
56
57 import apt_pkg
58 from yaml import safe_dump
59
60 from daklib import utils
61 from daklib.contents import UnpackedSource
62 from daklib.dbconn import Archive, DBConn, get_suite
63 from daklib.regexes import re_no_epoch
64
65
66
67 filelist = "filelist.yaml"
68
69
71 print(
72 """Generate changelog between two suites
73
74 Usage:
75 make-changelog -s <suite> -b <base_suite> [OPTION]...
76 make-changelog -e -a <archive>
77
78 Options:
79
80 -h, --help show this help and exit
81 -s, --suite suite providing packages to compare
82 -b, --base-suite suite to be taken as reference for comparison
83 -n, --binnmu display binNMUs uploads instead of source ones
84
85 -e, --export export interesting files from source packages
86 -a, --archive archive to fetch data from
87 -p, --progress display progress status"""
88 )
89
90 sys.exit(exit_code)
91
92
94 """
95 Returns changelogs for source uploads where version is newer than base.
96 """
97
98 query = """WITH base AS (
99 SELECT source, max(version) AS version
100 FROM source_suite
101 WHERE suite_name = :base_suite
102 GROUP BY source
103 UNION (SELECT source, CAST(0 AS debversion) AS version
104 FROM source_suite
105 WHERE suite_name = :suite
106 EXCEPT SELECT source, CAST(0 AS debversion) AS version
107 FROM source_suite
108 WHERE suite_name = :base_suite
109 ORDER BY source)),
110 cur_suite AS (
111 SELECT source, max(version) AS version
112 FROM source_suite
113 WHERE suite_name = :suite
114 GROUP BY source)
115 SELECT DISTINCT c.source, c.version, c.changelog
116 FROM changelogs c
117 JOIN base b ON b.source = c.source
118 JOIN cur_suite cs ON cs.source = c.source
119 WHERE c.version > b.version
120 AND c.version <= cs.version
121 AND c.architecture LIKE '%source%'
122 ORDER BY c.source, c.version DESC"""
123
124 return session.execute(query, {"suite": suite, "base_suite": base_suite})
125
126
128 """
129 Returns changelogs for binary uploads where version is newer than base.
130 """
131
132 query = """WITH base as (
133 SELECT s.source, max(b.version) AS version, a.arch_string
134 FROM source s
135 JOIN binaries b ON b.source = s.id
136 JOIN bin_associations ba ON ba.bin = b.id
137 JOIN architecture a ON a.id = b.architecture
138 WHERE ba.suite = (
139 SELECT id
140 FROM suite
141 WHERE suite_name = :base_suite)
142 GROUP BY s.source, a.arch_string),
143 cur_suite as (
144 SELECT s.source, max(b.version) AS version, a.arch_string
145 FROM source s
146 JOIN binaries b ON b.source = s.id
147 JOIN bin_associations ba ON ba.bin = b.id
148 JOIN architecture a ON a.id = b.architecture
149 WHERE ba.suite = (
150 SELECT id
151 FROM suite
152 WHERE suite_name = :suite)
153 GROUP BY s.source, a.arch_string)
154 SELECT DISTINCT c.source, c.version, c.architecture, c.changelog
155 FROM changelogs c
156 JOIN base b on b.source = c.source
157 JOIN cur_suite cs ON cs.source = c.source
158 WHERE c.version > b.version
159 AND c.version <= cs.version
160 AND c.architecture = b.arch_string
161 AND c.architecture = cs.arch_string
162 ORDER BY c.source, c.version DESC, c.architecture"""
163
164 return session.execute(query, {"suite": suite, "base_suite": base_suite})
165
166
168 prev_upload = None
169 for upload in uploads:
170 if prev_upload and prev_upload != upload[0]:
171 print()
172 print(upload[index])
173 prev_upload = upload[0]
174
175
177 """
178 Export interesting files from source packages.
179 """
180 pool = os.path.join(archive.path, "pool")
181
182 sources = {}
183 unpack = {}
184 files = ("changelog", "copyright", "NEWS", "NEWS.Debian", "README.Debian")
185 stats = {"unpack": 0, "created": 0, "removed": 0, "errors": 0, "files": 0}
186 query = """SELECT DISTINCT s.source, su.suite_name AS suite, s.version, c.name || '/' || f.filename AS filename
187 FROM source s
188 JOIN newest_source n ON n.source = s.source AND n.version = s.version
189 JOIN src_associations sa ON sa.source = s.id
190 JOIN suite su ON su.id = sa.suite
191 JOIN files f ON f.id = s.file
192 JOIN files_archive_map fam ON f.id = fam.file_id AND fam.archive_id = su.archive_id
193 JOIN component c ON fam.component_id = c.id
194 WHERE su.archive_id = :archive_id
195 ORDER BY s.source, suite"""
196
197 for p in session.execute(query, {"archive_id": archive.archive_id}):
198 if p[0] not in sources:
199 sources[p[0]] = {}
200 sources[p[0]][p[1]] = (re_no_epoch.sub("", p[2]), p[3])
201
202 for p in sources.keys():
203 for s in sources[p].keys():
204 path = os.path.join(clpool, "/".join(sources[p][s][1].split("/")[:-1]))
205 if not os.path.exists(path):
206 os.makedirs(path)
207 if not os.path.exists(
208 os.path.join(path, "%s_%s_changelog" % (p, sources[p][s][0]))
209 ):
210 if os.path.join(pool, sources[p][s][1]) not in unpack:
211 unpack[os.path.join(pool, sources[p][s][1])] = (path, set())
212 unpack[os.path.join(pool, sources[p][s][1])][1].add(s)
213 else:
214 for file in glob("%s/%s_%s_*" % (path, p, sources[p][s][0])):
215 link = "%s%s" % (s, file.split("%s_%s" % (p, sources[p][s][0]))[1])
216 try:
217 os.unlink(os.path.join(path, link))
218 except OSError:
219 pass
220 os.link(os.path.join(path, file), os.path.join(path, link))
221
222 for p in unpack.keys():
223 package = os.path.splitext(os.path.basename(p))[0].split("_")
224 try:
225 unpacked = UnpackedSource(p, clpool)
226 tempdir = unpacked.get_root_directory()
227 stats["unpack"] += 1
228 if progress:
229 if stats["unpack"] % 100 == 0:
230 print("%d packages unpacked" % stats["unpack"], file=sys.stderr)
231 elif stats["unpack"] % 10 == 0:
232 print(".", end="", file=sys.stderr)
233 for file in files:
234 for f in glob(os.path.join(tempdir, "debian", "*%s" % file)):
235 for s in unpack[p][1]:
236 suite = os.path.join(
237 unpack[p][0], "%s_%s" % (s, os.path.basename(f))
238 )
239 version = os.path.join(
240 unpack[p][0],
241 "%s_%s_%s" % (package[0], package[1], os.path.basename(f)),
242 )
243 if not os.path.exists(version):
244 os.link(f, version)
245 stats["created"] += 1
246 try:
247 os.unlink(suite)
248 except OSError:
249 pass
250 os.link(version, suite)
251 stats["created"] += 1
252 unpacked.cleanup()
253 except Exception as e:
254 print("make-changelog: unable to unpack %s\n%s" % (p, e))
255 stats["errors"] += 1
256
257 for root, dirs, files in os.walk(clpool, topdown=False):
258 files = [f for f in files if f != filelist]
259 if len(files):
260 if root != clpool:
261 if root.split("/")[-1] not in sources:
262 if os.path.exists(root):
263 stats["removed"] += len(os.listdir(root))
264 rmtree(root)
265 for file in files:
266 if os.path.exists(os.path.join(root, file)):
267 if os.stat(os.path.join(root, file)).st_nlink == 1:
268 stats["removed"] += 1
269 os.unlink(os.path.join(root, file))
270 for dir in dirs:
271 try:
272 os.rmdir(os.path.join(root, dir))
273 except OSError:
274 pass
275 stats["files"] += len(files)
276 stats["files"] -= stats["removed"]
277
278 print("make-changelog: file exporting finished")
279 print(" * New packages unpacked: %d" % stats["unpack"])
280 print(" * New files created: %d" % stats["created"])
281 print(" * New files removed: %d" % stats["removed"])
282 print(" * Unpack errors: %d" % stats["errors"])
283 print(" * Files available into changelog pool: %d" % stats["files"])
284
285
287 clfiles = {}
288 for root, dirs, files in os.walk(clpool):
289 for file in [f for f in files if f != filelist]:
290 clpath = os.path.join(root, file).replace(clpool, "").strip("/")
291 source = clpath.split("/")[2]
292 elements = clpath.split("/")[3].split("_")
293 if source not in clfiles:
294 clfiles[source] = {}
295 if elements[0] == source:
296 if elements[1] not in clfiles[source]:
297 clfiles[source][elements[1]] = []
298 clfiles[source][elements[1]].append(clpath)
299 else:
300 if elements[0] not in clfiles[source]:
301 clfiles[source][elements[0]] = []
302 clfiles[source][elements[0]].append(clpath)
303 with open(os.path.join(clpool, filelist), "w+") as fd:
304 safe_dump(clfiles, fd, default_flow_style=False)
305
306
308 Cnf = utils.get_conf()
309 Arguments = [
310 ("h", "help", "Make-Changelog::Options::Help"),
311 ("a", "archive", "Make-Changelog::Options::Archive", "HasArg"),
312 ("s", "suite", "Make-Changelog::Options::Suite", "HasArg"),
313 ("b", "base-suite", "Make-Changelog::Options::Base-Suite", "HasArg"),
314 ("n", "binnmu", "Make-Changelog::Options::binNMU"),
315 ("e", "export", "Make-Changelog::Options::export"),
316 ("p", "progress", "Make-Changelog::Options::progress"),
317 ]
318
319 for i in ["help", "suite", "base-suite", "binnmu", "export", "progress"]:
320 key = "Make-Changelog::Options::%s" % i
321 if key not in Cnf:
322 Cnf[key] = ""
323
324 apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
325 Options = Cnf.subtree("Make-Changelog::Options")
326 suite = Cnf["Make-Changelog::Options::Suite"]
327 base_suite = Cnf["Make-Changelog::Options::Base-Suite"]
328 binnmu = Cnf["Make-Changelog::Options::binNMU"]
329 export = Cnf["Make-Changelog::Options::export"]
330 progress = Cnf["Make-Changelog::Options::progress"]
331
332 if Options["help"] or not (suite and base_suite) and not export:
333 usage()
334
335 for s in suite, base_suite:
336 if not export and not get_suite(s):
337 utils.fubar('Invalid suite "%s"' % s)
338
339 session = DBConn().session()
340
341 if export:
342 archive = (
343 session.query(Archive).filter_by(archive_name=Options["Archive"]).one()
344 )
345 exportpath = archive.changelog
346 if exportpath:
347 export_files(session, archive, exportpath, progress)
348 generate_export_filelist(exportpath)
349 else:
350 utils.fubar("No changelog export path defined")
351 elif binnmu:
352 display_changes(get_binary_uploads(suite, base_suite, session), 3)
353 else:
354 display_changes(get_source_uploads(suite, base_suite, session), 2)
355
356 session.commit()
357
358
359 if __name__ == "__main__":
360 main()
361