1#! /usr/bin/env python3
3"""
4Generate changelog entry between two suites
6@contact: Debian FTP Master <ftpmaster@debian.org>
7@copyright: 2010 Luca Falavigna <dktrkranz@debian.org>
8@license: GNU General Public License version 2 or later
9"""
11# This program is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation; either version 2 of the License, or
14# (at your option) any later version.
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
21# You should have received a copy of the GNU General Public License
22# along with this program; if not, write to the Free Software
23# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25################################################################################
27# <bdefreese> !dinstall
28# <dak> bdefreese: I guess the next dinstall will be in 0hr 1min 35sec
29# <bdefreese> Wow I have great timing
30# <DktrKranz> dating with dinstall, part II
31# <bdefreese> heh
32# <Ganneff> dating with that monster? do you have good combat armor?
33# <bdefreese> +5 Plate :)
34# <Ganneff> not a good one then
35# <Ganneff> so you wont even manage to bypass the lesser monster in front, unchecked
36# <DktrKranz> asbesto belt
37# <Ganneff> helps only a step
38# <DktrKranz> the Ultimate Weapon: cron_turned_off
39# <bdefreese> heh
40# <Ganneff> thats debadmin limited
41# <Ganneff> no option for you
42# <DktrKranz> bdefreese: it seems ftp-masters want dinstall to sexual harass us, are you good in running?
43# <Ganneff> you can run but you can not hide
44# <bdefreese> No, I'm old and fat :)
45# <Ganneff> you can roll but you can not hide
46# <Ganneff> :)
47# <bdefreese> haha
48# <DktrKranz> damn dinstall, you racist bastard
50################################################################################
52import os
53import sys
54import apt_pkg
55from glob import glob
56from shutil import rmtree
57from yaml import safe_dump
58from daklib.dbconn import *
59from daklib import utils
60from daklib.contents import UnpackedSource
61from daklib.regexes import re_no_epoch
63################################################################################
65filelist = 'filelist.yaml'
68def usage(exit_code=0):
69 print("""Generate changelog between two suites
71 Usage:
72 make-changelog -s <suite> -b <base_suite> [OPTION]...
73 make-changelog -e -a <archive>
75Options:
77 -h, --help show this help and exit
78 -s, --suite suite providing packages to compare
79 -b, --base-suite suite to be taken as reference for comparison
80 -n, --binnmu display binNMUs uploads instead of source ones
82 -e, --export export interesting files from source packages
83 -a, --archive archive to fetch data from
84 -p, --progress display progress status""")
86 sys.exit(exit_code)
89def get_source_uploads(suite, base_suite, session):
90 """
91 Returns changelogs for source uploads where version is newer than base.
92 """
94 query = """WITH base AS (
95 SELECT source, max(version) AS version
96 FROM source_suite
97 WHERE suite_name = :base_suite
98 GROUP BY source
99 UNION (SELECT source, CAST(0 AS debversion) AS version
100 FROM source_suite
101 WHERE suite_name = :suite
102 EXCEPT SELECT source, CAST(0 AS debversion) AS version
103 FROM source_suite
104 WHERE suite_name = :base_suite
105 ORDER BY source)),
106 cur_suite AS (
107 SELECT source, max(version) AS version
108 FROM source_suite
109 WHERE suite_name = :suite
110 GROUP BY source)
111 SELECT DISTINCT c.source, c.version, c.changelog
112 FROM changelogs c
113 JOIN base b ON b.source = c.source
114 JOIN cur_suite cs ON cs.source = c.source
115 WHERE c.version > b.version
116 AND c.version <= cs.version
117 AND c.architecture LIKE '%source%'
118 ORDER BY c.source, c.version DESC"""
120 return session.execute(query, {'suite': suite, 'base_suite': base_suite})
123def get_binary_uploads(suite, base_suite, session):
124 """
125 Returns changelogs for binary uploads where version is newer than base.
126 """
128 query = """WITH base as (
129 SELECT s.source, max(b.version) AS version, a.arch_string
130 FROM source s
131 JOIN binaries b ON b.source = s.id
132 JOIN bin_associations ba ON ba.bin = b.id
133 JOIN architecture a ON a.id = b.architecture
134 WHERE ba.suite = (
135 SELECT id
136 FROM suite
137 WHERE suite_name = :base_suite)
138 GROUP BY s.source, a.arch_string),
139 cur_suite as (
140 SELECT s.source, max(b.version) AS version, a.arch_string
141 FROM source s
142 JOIN binaries b ON b.source = s.id
143 JOIN bin_associations ba ON ba.bin = b.id
144 JOIN architecture a ON a.id = b.architecture
145 WHERE ba.suite = (
146 SELECT id
147 FROM suite
148 WHERE suite_name = :suite)
149 GROUP BY s.source, a.arch_string)
150 SELECT DISTINCT c.source, c.version, c.architecture, c.changelog
151 FROM changelogs c
152 JOIN base b on b.source = c.source
153 JOIN cur_suite cs ON cs.source = c.source
154 WHERE c.version > b.version
155 AND c.version <= cs.version
156 AND c.architecture = b.arch_string
157 AND c.architecture = cs.arch_string
158 ORDER BY c.source, c.version DESC, c.architecture"""
160 return session.execute(query, {'suite': suite, 'base_suite': base_suite})
163def display_changes(uploads, index):
164 prev_upload = None
165 for upload in uploads:
166 if prev_upload and prev_upload != upload[0]:
167 print()
168 print(upload[index])
169 prev_upload = upload[0]
172def export_files(session, archive, clpool, progress=False):
173 """
174 Export interesting files from source packages.
175 """
176 pool = os.path.join(archive.path, 'pool')
178 sources = {}
179 unpack = {}
180 files = ('changelog', 'copyright', 'NEWS', 'NEWS.Debian', 'README.Debian')
181 stats = {'unpack': 0, 'created': 0, 'removed': 0, 'errors': 0, 'files': 0}
182 query = """SELECT DISTINCT s.source, su.suite_name AS suite, s.version, c.name || '/' || f.filename AS filename
183 FROM source s
184 JOIN newest_source n ON n.source = s.source AND n.version = s.version
185 JOIN src_associations sa ON sa.source = s.id
186 JOIN suite su ON su.id = sa.suite
187 JOIN files f ON f.id = s.file
188 JOIN files_archive_map fam ON f.id = fam.file_id AND fam.archive_id = su.archive_id
189 JOIN component c ON fam.component_id = c.id
190 WHERE su.archive_id = :archive_id
191 ORDER BY s.source, suite"""
193 for p in session.execute(query, {'archive_id': archive.archive_id}):
194 if p[0] not in sources:
195 sources[p[0]] = {}
196 sources[p[0]][p[1]] = (re_no_epoch.sub('', p[2]), p[3])
198 for p in sources.keys():
199 for s in sources[p].keys():
200 path = os.path.join(clpool, '/'.join(sources[p][s][1].split('/')[:-1]))
201 if not os.path.exists(path):
202 os.makedirs(path)
203 if not os.path.exists(os.path.join(path, 203 ↛ 209line 203 didn't jump to line 209, because the condition on line 203 was never false
204 '%s_%s_changelog' % (p, sources[p][s][0]))):
205 if os.path.join(pool, sources[p][s][1]) not in unpack:
206 unpack[os.path.join(pool, sources[p][s][1])] = (path, set())
207 unpack[os.path.join(pool, sources[p][s][1])][1].add(s)
208 else:
209 for file in glob('%s/%s_%s_*' % (path, p, sources[p][s][0])):
210 link = '%s%s' % (s, file.split('%s_%s'
211 % (p, sources[p][s][0]))[1])
212 try:
213 os.unlink(os.path.join(path, link))
214 except OSError:
215 pass
216 os.link(os.path.join(path, file), os.path.join(path, link))
218 for p in unpack.keys():
219 package = os.path.splitext(os.path.basename(p))[0].split('_')
220 try:
221 unpacked = UnpackedSource(p, clpool)
222 tempdir = unpacked.get_root_directory()
223 stats['unpack'] += 1
224 if progress: 224 ↛ 225line 224 didn't jump to line 225, because the condition on line 224 was never true
225 if stats['unpack'] % 100 == 0:
226 print('%d packages unpacked' % stats['unpack'], file=sys.stderr)
227 elif stats['unpack'] % 10 == 0:
228 print('.', end='', file=sys.stderr)
229 for file in files:
230 for f in glob(os.path.join(tempdir, 'debian', '*%s' % file)):
231 for s in unpack[p][1]:
232 suite = os.path.join(unpack[p][0], '%s_%s'
233 % (s, os.path.basename(f)))
234 version = os.path.join(unpack[p][0], '%s_%s_%s' %
235 (package[0], package[1], os.path.basename(f)))
236 if not os.path.exists(version):
237 os.link(f, version)
238 stats['created'] += 1
239 try:
240 os.unlink(suite)
241 except OSError:
242 pass
243 os.link(version, suite)
244 stats['created'] += 1
245 unpacked.cleanup()
246 except Exception as e:
247 print('make-changelog: unable to unpack %s\n%s' % (p, e))
248 stats['errors'] += 1
250 for root, dirs, files in os.walk(clpool, topdown=False):
251 files = [f for f in files if f != filelist]
252 if len(files):
253 if root != clpool: 253 ↛ 258line 253 didn't jump to line 258, because the condition on line 253 was never false
254 if root.split('/')[-1] not in sources: 254 ↛ 255line 254 didn't jump to line 255, because the condition on line 254 was never true
255 if os.path.exists(root):
256 stats['removed'] += len(os.listdir(root))
257 rmtree(root)
258 for file in files:
259 if os.path.exists(os.path.join(root, file)): 259 ↛ 258line 259 didn't jump to line 258, because the condition on line 259 was never false
260 if os.stat(os.path.join(root, file)).st_nlink == 1: 260 ↛ 261line 260 didn't jump to line 261, because the condition on line 260 was never true
261 stats['removed'] += 1
262 os.unlink(os.path.join(root, file))
263 for dir in dirs:
264 try:
265 os.rmdir(os.path.join(root, dir))
266 except OSError:
267 pass
268 stats['files'] += len(files)
269 stats['files'] -= stats['removed']
271 print('make-changelog: file exporting finished')
272 print(' * New packages unpacked: %d' % stats['unpack'])
273 print(' * New files created: %d' % stats['created'])
274 print(' * New files removed: %d' % stats['removed'])
275 print(' * Unpack errors: %d' % stats['errors'])
276 print(' * Files available into changelog pool: %d' % stats['files'])
279def generate_export_filelist(clpool):
280 clfiles = {}
281 for root, dirs, files in os.walk(clpool):
282 for file in [f for f in files if f != filelist]:
283 clpath = os.path.join(root, file).replace(clpool, '').strip('/')
284 source = clpath.split('/')[2]
285 elements = clpath.split('/')[3].split('_')
286 if source not in clfiles:
287 clfiles[source] = {}
288 if elements[0] == source:
289 if elements[1] not in clfiles[source]: 289 ↛ 291line 289 didn't jump to line 291, because the condition on line 289 was never false
290 clfiles[source][elements[1]] = []
291 clfiles[source][elements[1]].append(clpath)
292 else:
293 if elements[0] not in clfiles[source]: 293 ↛ 295line 293 didn't jump to line 295, because the condition on line 293 was never false
294 clfiles[source][elements[0]] = []
295 clfiles[source][elements[0]].append(clpath)
296 with open(os.path.join(clpool, filelist), 'w+') as fd:
297 safe_dump(clfiles, fd, default_flow_style=False)
300def main():
301 Cnf = utils.get_conf()
302 Arguments = [('h', 'help', 'Make-Changelog::Options::Help'),
303 ('a', 'archive', 'Make-Changelog::Options::Archive', 'HasArg'),
304 ('s', 'suite', 'Make-Changelog::Options::Suite', 'HasArg'),
305 ('b', 'base-suite', 'Make-Changelog::Options::Base-Suite', 'HasArg'),
306 ('n', 'binnmu', 'Make-Changelog::Options::binNMU'),
307 ('e', 'export', 'Make-Changelog::Options::export'),
308 ('p', 'progress', 'Make-Changelog::Options::progress')]
310 for i in ['help', 'suite', 'base-suite', 'binnmu', 'export', 'progress']:
311 key = 'Make-Changelog::Options::%s' % i
312 if key not in Cnf: 312 ↛ 310line 312 didn't jump to line 310, because the condition on line 312 was never false
313 Cnf[key] = ''
315 apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
316 Options = Cnf.subtree('Make-Changelog::Options')
317 suite = Cnf['Make-Changelog::Options::Suite']
318 base_suite = Cnf['Make-Changelog::Options::Base-Suite']
319 binnmu = Cnf['Make-Changelog::Options::binNMU']
320 export = Cnf['Make-Changelog::Options::export']
321 progress = Cnf['Make-Changelog::Options::progress']
323 if Options['help'] or not (suite and base_suite) and not export:
324 usage()
326 for s in suite, base_suite:
327 if not export and not get_suite(s): 327 ↛ 328line 327 didn't jump to line 328, because the condition on line 327 was never true
328 utils.fubar('Invalid suite "%s"' % s)
330 session = DBConn().session()
332 if export:
333 archive = session.query(Archive).filter_by(archive_name=Options['Archive']).one()
334 exportpath = archive.changelog
335 if exportpath: 335 ↛ 339line 335 didn't jump to line 339, because the condition on line 335 was never false
336 export_files(session, archive, exportpath, progress)
337 generate_export_filelist(exportpath)
338 else:
339 utils.fubar('No changelog export path defined')
340 elif binnmu: 340 ↛ 341line 340 didn't jump to line 341, because the condition on line 340 was never true
341 display_changes(get_binary_uploads(suite, base_suite, session), 3)
342 else:
343 display_changes(get_source_uploads(suite, base_suite, session), 2)
345 session.commit()
348if __name__ == '__main__':
349 main()