1
2
3 """
4 Generate changelog entry between two suites
5
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2010 Luca Falavigna <dktrkranz@debian.org>
8 @license: GNU General Public License version 2 or later
9 """
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52 import os
53 import sys
54 import apt_pkg
55 from glob import glob
56 from shutil import rmtree
57 from yaml import safe_dump
58 from daklib.dbconn import *
59 from daklib import utils
60 from daklib.contents import UnpackedSource
61 from daklib.regexes import re_no_epoch
62
63
64
65 filelist = 'filelist.yaml'
66
67
69 print("""Generate changelog between two suites
70
71 Usage:
72 make-changelog -s <suite> -b <base_suite> [OPTION]...
73 make-changelog -e -a <archive>
74
75 Options:
76
77 -h, --help show this help and exit
78 -s, --suite suite providing packages to compare
79 -b, --base-suite suite to be taken as reference for comparison
80 -n, --binnmu display binNMUs uploads instead of source ones
81
82 -e, --export export interesting files from source packages
83 -a, --archive archive to fetch data from
84 -p, --progress display progress status""")
85
86 sys.exit(exit_code)
87
88
90 """
91 Returns changelogs for source uploads where version is newer than base.
92 """
93
94 query = """WITH base AS (
95 SELECT source, max(version) AS version
96 FROM source_suite
97 WHERE suite_name = :base_suite
98 GROUP BY source
99 UNION (SELECT source, CAST(0 AS debversion) AS version
100 FROM source_suite
101 WHERE suite_name = :suite
102 EXCEPT SELECT source, CAST(0 AS debversion) AS version
103 FROM source_suite
104 WHERE suite_name = :base_suite
105 ORDER BY source)),
106 cur_suite AS (
107 SELECT source, max(version) AS version
108 FROM source_suite
109 WHERE suite_name = :suite
110 GROUP BY source)
111 SELECT DISTINCT c.source, c.version, c.changelog
112 FROM changelogs c
113 JOIN base b ON b.source = c.source
114 JOIN cur_suite cs ON cs.source = c.source
115 WHERE c.version > b.version
116 AND c.version <= cs.version
117 AND c.architecture LIKE '%source%'
118 ORDER BY c.source, c.version DESC"""
119
120 return session.execute(query, {'suite': suite, 'base_suite': base_suite})
121
122
124 """
125 Returns changelogs for binary uploads where version is newer than base.
126 """
127
128 query = """WITH base as (
129 SELECT s.source, max(b.version) AS version, a.arch_string
130 FROM source s
131 JOIN binaries b ON b.source = s.id
132 JOIN bin_associations ba ON ba.bin = b.id
133 JOIN architecture a ON a.id = b.architecture
134 WHERE ba.suite = (
135 SELECT id
136 FROM suite
137 WHERE suite_name = :base_suite)
138 GROUP BY s.source, a.arch_string),
139 cur_suite as (
140 SELECT s.source, max(b.version) AS version, a.arch_string
141 FROM source s
142 JOIN binaries b ON b.source = s.id
143 JOIN bin_associations ba ON ba.bin = b.id
144 JOIN architecture a ON a.id = b.architecture
145 WHERE ba.suite = (
146 SELECT id
147 FROM suite
148 WHERE suite_name = :suite)
149 GROUP BY s.source, a.arch_string)
150 SELECT DISTINCT c.source, c.version, c.architecture, c.changelog
151 FROM changelogs c
152 JOIN base b on b.source = c.source
153 JOIN cur_suite cs ON cs.source = c.source
154 WHERE c.version > b.version
155 AND c.version <= cs.version
156 AND c.architecture = b.arch_string
157 AND c.architecture = cs.arch_string
158 ORDER BY c.source, c.version DESC, c.architecture"""
159
160 return session.execute(query, {'suite': suite, 'base_suite': base_suite})
161
162
164 prev_upload = None
165 for upload in uploads:
166 if prev_upload and prev_upload != upload[0]:
167 print()
168 print(upload[index])
169 prev_upload = upload[0]
170
171
173 """
174 Export interesting files from source packages.
175 """
176 pool = os.path.join(archive.path, 'pool')
177
178 sources = {}
179 unpack = {}
180 files = ('changelog', 'copyright', 'NEWS', 'NEWS.Debian', 'README.Debian')
181 stats = {'unpack': 0, 'created': 0, 'removed': 0, 'errors': 0, 'files': 0}
182 query = """SELECT DISTINCT s.source, su.suite_name AS suite, s.version, c.name || '/' || f.filename AS filename
183 FROM source s
184 JOIN newest_source n ON n.source = s.source AND n.version = s.version
185 JOIN src_associations sa ON sa.source = s.id
186 JOIN suite su ON su.id = sa.suite
187 JOIN files f ON f.id = s.file
188 JOIN files_archive_map fam ON f.id = fam.file_id AND fam.archive_id = su.archive_id
189 JOIN component c ON fam.component_id = c.id
190 WHERE su.archive_id = :archive_id
191 ORDER BY s.source, suite"""
192
193 for p in session.execute(query, {'archive_id': archive.archive_id}):
194 if p[0] not in sources:
195 sources[p[0]] = {}
196 sources[p[0]][p[1]] = (re_no_epoch.sub('', p[2]), p[3])
197
198 for p in sources.keys():
199 for s in sources[p].keys():
200 path = os.path.join(clpool, '/'.join(sources[p][s][1].split('/')[:-1]))
201 if not os.path.exists(path):
202 os.makedirs(path)
203 if not os.path.exists(os.path.join(path,
204 '%s_%s_changelog' % (p, sources[p][s][0]))):
205 if os.path.join(pool, sources[p][s][1]) not in unpack:
206 unpack[os.path.join(pool, sources[p][s][1])] = (path, set())
207 unpack[os.path.join(pool, sources[p][s][1])][1].add(s)
208 else:
209 for file in glob('%s/%s_%s_*' % (path, p, sources[p][s][0])):
210 link = '%s%s' % (s, file.split('%s_%s'
211 % (p, sources[p][s][0]))[1])
212 try:
213 os.unlink(os.path.join(path, link))
214 except OSError:
215 pass
216 os.link(os.path.join(path, file), os.path.join(path, link))
217
218 for p in unpack.keys():
219 package = os.path.splitext(os.path.basename(p))[0].split('_')
220 try:
221 unpacked = UnpackedSource(p, clpool)
222 tempdir = unpacked.get_root_directory()
223 stats['unpack'] += 1
224 if progress:
225 if stats['unpack'] % 100 == 0:
226 print('%d packages unpacked' % stats['unpack'], file=sys.stderr)
227 elif stats['unpack'] % 10 == 0:
228 print('.', end='', file=sys.stderr)
229 for file in files:
230 for f in glob(os.path.join(tempdir, 'debian', '*%s' % file)):
231 for s in unpack[p][1]:
232 suite = os.path.join(unpack[p][0], '%s_%s'
233 % (s, os.path.basename(f)))
234 version = os.path.join(unpack[p][0], '%s_%s_%s' %
235 (package[0], package[1], os.path.basename(f)))
236 if not os.path.exists(version):
237 os.link(f, version)
238 stats['created'] += 1
239 try:
240 os.unlink(suite)
241 except OSError:
242 pass
243 os.link(version, suite)
244 stats['created'] += 1
245 unpacked.cleanup()
246 except Exception as e:
247 print('make-changelog: unable to unpack %s\n%s' % (p, e))
248 stats['errors'] += 1
249
250 for root, dirs, files in os.walk(clpool, topdown=False):
251 files = [f for f in files if f != filelist]
252 if len(files):
253 if root != clpool:
254 if root.split('/')[-1] not in sources:
255 if os.path.exists(root):
256 stats['removed'] += len(os.listdir(root))
257 rmtree(root)
258 for file in files:
259 if os.path.exists(os.path.join(root, file)):
260 if os.stat(os.path.join(root, file)).st_nlink == 1:
261 stats['removed'] += 1
262 os.unlink(os.path.join(root, file))
263 for dir in dirs:
264 try:
265 os.rmdir(os.path.join(root, dir))
266 except OSError:
267 pass
268 stats['files'] += len(files)
269 stats['files'] -= stats['removed']
270
271 print('make-changelog: file exporting finished')
272 print(' * New packages unpacked: %d' % stats['unpack'])
273 print(' * New files created: %d' % stats['created'])
274 print(' * New files removed: %d' % stats['removed'])
275 print(' * Unpack errors: %d' % stats['errors'])
276 print(' * Files available into changelog pool: %d' % stats['files'])
277
278
280 clfiles = {}
281 for root, dirs, files in os.walk(clpool):
282 for file in [f for f in files if f != filelist]:
283 clpath = os.path.join(root, file).replace(clpool, '').strip('/')
284 source = clpath.split('/')[2]
285 elements = clpath.split('/')[3].split('_')
286 if source not in clfiles:
287 clfiles[source] = {}
288 if elements[0] == source:
289 if elements[1] not in clfiles[source]:
290 clfiles[source][elements[1]] = []
291 clfiles[source][elements[1]].append(clpath)
292 else:
293 if elements[0] not in clfiles[source]:
294 clfiles[source][elements[0]] = []
295 clfiles[source][elements[0]].append(clpath)
296 with open(os.path.join(clpool, filelist), 'w+') as fd:
297 safe_dump(clfiles, fd, default_flow_style=False)
298
299
301 Cnf = utils.get_conf()
302 Arguments = [('h', 'help', 'Make-Changelog::Options::Help'),
303 ('a', 'archive', 'Make-Changelog::Options::Archive', 'HasArg'),
304 ('s', 'suite', 'Make-Changelog::Options::Suite', 'HasArg'),
305 ('b', 'base-suite', 'Make-Changelog::Options::Base-Suite', 'HasArg'),
306 ('n', 'binnmu', 'Make-Changelog::Options::binNMU'),
307 ('e', 'export', 'Make-Changelog::Options::export'),
308 ('p', 'progress', 'Make-Changelog::Options::progress')]
309
310 for i in ['help', 'suite', 'base-suite', 'binnmu', 'export', 'progress']:
311 key = 'Make-Changelog::Options::%s' % i
312 if key not in Cnf:
313 Cnf[key] = ''
314
315 apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
316 Options = Cnf.subtree('Make-Changelog::Options')
317 suite = Cnf['Make-Changelog::Options::Suite']
318 base_suite = Cnf['Make-Changelog::Options::Base-Suite']
319 binnmu = Cnf['Make-Changelog::Options::binNMU']
320 export = Cnf['Make-Changelog::Options::export']
321 progress = Cnf['Make-Changelog::Options::progress']
322
323 if Options['help'] or not (suite and base_suite) and not export:
324 usage()
325
326 for s in suite, base_suite:
327 if not export and not get_suite(s):
328 utils.fubar('Invalid suite "%s"' % s)
329
330 session = DBConn().session()
331
332 if export:
333 archive = session.query(Archive).filter_by(archive_name=Options['Archive']).one()
334 exportpath = archive.changelog
335 if exportpath:
336 export_files(session, archive, exportpath, progress)
337 generate_export_filelist(exportpath)
338 else:
339 utils.fubar('No changelog export path defined')
340 elif binnmu:
341 display_changes(get_binary_uploads(suite, base_suite, session), 3)
342 else:
343 display_changes(get_source_uploads(suite, base_suite, session), 2)
344
345 session.commit()
346
347
348 if __name__ == '__main__':
349 main()
350