1#! /usr/bin/env python3
3"""
4Create all the Release files
6@contact: Debian FTPMaster <ftpmaster@debian.org>
7@copyright: 2011 Joerg Jaspert <joerg@debian.org>
8@copyright: 2011 Mark Hymers <mhy@debian.org>
9@license: GNU General Public License version 2 or later
11"""
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27################################################################################
29# <mhy> I wish they wouldnt leave biscuits out, thats just tempting. Damnit.
31################################################################################
33import sys
34import os
35import os.path
36import time
37import gzip
38import bz2
39import errno
40import apt_pkg
41import subprocess
42from sqlalchemy.orm import object_session
44import daklib.gpg
45from daklib import utils, daklog
46from daklib.regexes import re_gensubrelease, re_includeinrelease_byhash, re_includeinrelease_plain
47from daklib.dbconn import *
48from daklib.config import Config
49from daklib.dakmultiprocessing import DakProcessPool, PROC_STATUS_SUCCESS
51################################################################################
52Logger = None #: Our logging object
54################################################################################
57def usage(exit_code=0):
58 """ Usage information"""
60 print("""Usage: dak generate-releases [OPTIONS]
61Generate the Release files
63 -a, --archive=ARCHIVE process suites in ARCHIVE
64 -s, --suite=SUITE(s) process this suite
65 Default: All suites not marked 'untouchable'
66 -f, --force Allow processing of untouchable suites
67 CAREFUL: Only to be used at (point) release time!
68 -h, --help show this help and exit
69 -q, --quiet Don't output progress
71SUITE can be a space separated list, e.g.
72 --suite=unstable testing
73 """)
74 sys.exit(exit_code)
76########################################################################
79def sign_release_dir(suite, dirname):
80 cnf = Config()
82 if 'Dinstall::SigningKeyring' in cnf or 'Dinstall::SigningHomedir' in cnf: 82 ↛ exitline 82 didn't return from function 'sign_release_dir', because the condition on line 82 was never false
83 args = {
84 'keyids': suite.signingkeys or [],
85 'pubring': cnf.get('Dinstall::SigningPubKeyring') or None,
86 'secring': cnf.get('Dinstall::SigningKeyring') or None,
87 'homedir': cnf.get('Dinstall::SigningHomedir') or None,
88 'passphrase_file': cnf.get('Dinstall::SigningPassphraseFile') or None,
89 }
91 relname = os.path.join(dirname, 'Release')
93 dest = os.path.join(dirname, 'Release.gpg')
94 if os.path.exists(dest):
95 os.unlink(dest)
97 inlinedest = os.path.join(dirname, 'InRelease')
98 if os.path.exists(inlinedest):
99 os.unlink(inlinedest)
101 with open(relname, 'r') as stdin:
102 with open(dest, 'w') as stdout:
103 daklib.gpg.sign(stdin, stdout, inline=False, **args)
104 stdin.seek(0)
105 with open(inlinedest, 'w') as stdout:
106 daklib.gpg.sign(stdin, stdout, inline=True, **args)
109class XzFile:
110 def __init__(self, filename, mode='r'):
111 self.filename = filename
113 def read(self):
114 with open(self.filename, 'rb') as stdin:
115 return subprocess.check_output(['xz', '-d'], stdin=stdin)
118class ZstdFile:
119 def __init__(self, filename, mode='r'):
120 self.filename = filename
122 def read(self):
123 with open(self.filename, 'rb') as stdin:
124 return subprocess.check_output(['zstd', '--decompress'], stdin=stdin)
127class HashFunc:
128 def __init__(self, release_field, func, db_name):
129 self.release_field = release_field
130 self.func = func
131 self.db_name = db_name
134RELEASE_HASHES = [
135 HashFunc('MD5Sum', apt_pkg.md5sum, 'md5sum'),
136 HashFunc('SHA1', apt_pkg.sha1sum, 'sha1'),
137 HashFunc('SHA256', apt_pkg.sha256sum, 'sha256'),
138]
141class ReleaseWriter:
142 def __init__(self, suite):
143 self.suite = suite
145 def suite_path(self):
146 """
147 Absolute path to the suite-specific files.
148 """
149 suite_suffix = utils.suite_suffix(self.suite.suite_name)
151 return os.path.join(self.suite.archive.path, 'dists',
152 self.suite.suite_name, suite_suffix)
154 def suite_release_path(self):
155 """
156 Absolute path where Release files are physically stored.
157 This should be a path that sorts after the dists/ directory.
158 """
159 cnf = Config()
160 suite_suffix = utils.suite_suffix(self.suite.suite_name)
162 return os.path.join(self.suite.archive.path, 'zzz-dists',
163 self.suite.codename or self.suite.suite_name, suite_suffix)
165 def create_release_symlinks(self):
166 """
167 Create symlinks for Release files.
168 This creates the symlinks for Release files in the `suite_path`
169 to the actual files in `suite_release_path`.
170 """
171 relpath = os.path.relpath(self.suite_release_path(), self.suite_path())
172 for f in ("Release", "Release.gpg", "InRelease"):
173 source = os.path.join(relpath, f)
174 dest = os.path.join(self.suite_path(), f)
175 if os.path.lexists(dest):
176 if not os.path.islink(dest): 176 ↛ 177line 176 didn't jump to line 177, because the condition on line 176 was never true
177 os.unlink(dest)
178 elif os.readlink(dest) == source: 178 ↛ 181line 178 didn't jump to line 181, because the condition on line 178 was never false
179 continue
180 else:
181 os.unlink(dest)
182 os.symlink(source, dest)
184 def create_output_directories(self):
185 for path in (self.suite_path(), self.suite_release_path()):
186 try:
187 os.makedirs(path)
188 except OSError as e:
189 if e.errno != errno.EEXIST: 189 ↛ 190line 189 didn't jump to line 190, because the condition on line 189 was never true
190 raise
192 def _update_hashfile_table(self, session, fileinfo, hashes):
193 # Mark all by-hash files as obsolete. We will undo that for the ones
194 # we still reference later.
195 query = """
196 UPDATE hashfile SET unreferenced = CURRENT_TIMESTAMP
197 WHERE suite_id = :id AND unreferenced IS NULL"""
198 session.execute(query, {'id': self.suite.suite_id})
200 query = "SELECT path FROM hashfile WHERE suite_id = :id"
201 q = session.execute(query, {'id': self.suite.suite_id})
202 known_hashfiles = set(row[0] for row in q)
203 updated = set()
204 new = set()
206 # Update the hashfile table with new or updated files
207 for filename in fileinfo:
208 if not os.path.lexists(filename): 208 ↛ 210line 208 didn't jump to line 210, because the condition on line 208 was never true
209 # probably an uncompressed index we didn't generate
210 continue
211 byhashdir = os.path.join(os.path.dirname(filename), 'by-hash')
212 for h in hashes:
213 field = h.release_field
214 hashfile = os.path.join(byhashdir, field, fileinfo[filename][field])
215 if hashfile in known_hashfiles:
216 updated.add(hashfile)
217 else:
218 new.add(hashfile)
220 if updated:
221 session.execute("""
222 UPDATE hashfile SET unreferenced = NULL
223 WHERE path = ANY(:p) AND suite_id = :id""",
224 {'p': list(updated), 'id': self.suite.suite_id})
225 if new:
226 session.execute("""
227 INSERT INTO hashfile (path, suite_id)
228 VALUES (:p, :id)""",
229 [{'p': hashfile, 'id': self.suite.suite_id} for hashfile in new])
231 session.commit()
233 def _make_byhash_links(self, fileinfo, hashes):
234 # Create hardlinks in by-hash directories
235 for filename in fileinfo:
236 if not os.path.lexists(filename): 236 ↛ 238line 236 didn't jump to line 238, because the condition on line 236 was never true
237 # probably an uncompressed index we didn't generate
238 continue
240 for h in hashes:
241 field = h.release_field
242 hashfile = os.path.join(os.path.dirname(filename), 'by-hash', field, fileinfo[filename][field])
243 try:
244 os.makedirs(os.path.dirname(hashfile))
245 except OSError as exc:
246 if exc.errno != errno.EEXIST: 246 ↛ 247line 246 didn't jump to line 247, because the condition on line 246 was never true
247 raise
248 try:
249 os.link(filename, hashfile)
250 except OSError as exc:
251 if exc.errno != errno.EEXIST: 251 ↛ 252line 251 didn't jump to line 252, because the condition on line 251 was never true
252 raise
254 def _make_byhash_base_symlink(self, fileinfo, hashes):
255 # Create symlinks to files in by-hash directories
256 for filename in fileinfo:
257 if not os.path.lexists(filename): 257 ↛ 259line 257 didn't jump to line 259, because the condition on line 257 was never true
258 # probably an uncompressed index we didn't generate
259 continue
261 besthash = hashes[-1]
262 field = besthash.release_field
263 hashfilebase = os.path.join('by-hash', field, fileinfo[filename][field])
264 hashfile = os.path.join(os.path.dirname(filename), hashfilebase)
266 assert os.path.exists(hashfile), 'by-hash file {} is missing'.format(hashfile)
268 os.unlink(filename)
269 os.symlink(hashfilebase, filename)
271 def generate_release_files(self):
272 """
273 Generate Release files for the given suite
274 """
276 suite = self.suite
277 session = object_session(suite)
279 # Attribs contains a tuple of field names and the database names to use to
280 # fill them in
281 attribs = (('Origin', 'origin'),
282 ('Label', 'label'),
283 ('Suite', 'release_suite_output'),
284 ('Version', 'version'),
285 ('Codename', 'codename'),
286 ('Changelogs', 'changelog_url'),
287 )
289 # A "Sub" Release file has slightly different fields
290 subattribs = (('Archive', 'suite_name'),
291 ('Origin', 'origin'),
292 ('Label', 'label'),
293 ('Version', 'version'))
295 # Boolean stuff. If we find it true in database, write out "yes" into the release file
296 boolattrs = (('NotAutomatic', 'notautomatic'),
297 ('ButAutomaticUpgrades', 'butautomaticupgrades'),
298 ('Acquire-By-Hash', 'byhash'),
299 )
301 cnf = Config()
302 cnf_suite_suffix = cnf.get("Dinstall::SuiteSuffix", "").rstrip("/")
304 suite_suffix = utils.suite_suffix(suite.suite_name)
306 self.create_output_directories()
307 self.create_release_symlinks()
309 outfile = os.path.join(self.suite_release_path(), "Release")
310 out = open(outfile + ".new", "w")
312 for key, dbfield in attribs:
313 # Hack to skip NULL Version fields as we used to do this
314 # We should probably just always ignore anything which is None
315 if key in ("Version", "Changelogs") and getattr(suite, dbfield) is None:
316 continue
318 out.write("%s: %s\n" % (key, getattr(suite, dbfield)))
320 out.write("Date: %s\n" % (time.strftime("%a, %d %b %Y %H:%M:%S UTC", time.gmtime(time.time()))))
322 if suite.validtime: 322 ↛ 326line 322 didn't jump to line 326, because the condition on line 322 was never false
323 validtime = float(suite.validtime)
324 out.write("Valid-Until: %s\n" % (time.strftime("%a, %d %b %Y %H:%M:%S UTC", time.gmtime(time.time() + validtime))))
326 for key, dbfield in boolattrs:
327 if getattr(suite, dbfield, False):
328 out.write("%s: yes\n" % (key))
330 skip_arch_all = True
331 if suite.separate_contents_architecture_all or suite.separate_packages_architecture_all:
332 # According to the Repository format specification:
333 # https://wiki.debian.org/DebianRepository/Format#No-Support-for-Architecture-all
334 #
335 # Clients are not expected to support Packages-all without Contents-all. At the
336 # time of writing, it is not possible to set separate_packages_architecture_all.
337 # However, we add this little assert to stop the bug early.
338 #
339 # If you are here because the assert failed, you probably want to see "update123.py"
340 # and its advice on updating the CHECK constraint.
341 assert suite.separate_contents_architecture_all
342 skip_arch_all = False
344 if not suite.separate_packages_architecture_all: 344 ↛ 347line 344 didn't jump to line 347, because the condition on line 344 was never false
345 out.write("No-Support-for-Architecture-all: Packages\n")
347 architectures = get_suite_architectures(suite.suite_name, skipall=skip_arch_all, skipsrc=True, session=session)
349 out.write("Architectures: %s\n" % (" ".join(a.arch_string for a in architectures)))
351 components = [c.component_name for c in suite.components]
353 out.write("Components: %s\n" % (" ".join(components)))
355 # For exact compatibility with old g-r, write out Description here instead
356 # of with the rest of the DB fields above
357 if getattr(suite, 'description') is not None: 357 ↛ 358line 357 didn't jump to line 358, because the condition on line 357 was never true
358 out.write("Description: %s\n" % suite.description)
360 for comp in components:
361 for dirpath, dirnames, filenames in os.walk(os.path.join(self.suite_path(), comp), topdown=True):
362 if not re_gensubrelease.match(dirpath):
363 continue
365 subfile = os.path.join(dirpath, "Release")
366 subrel = open(subfile + '.new', "w")
368 for key, dbfield in subattribs:
369 if getattr(suite, dbfield) is not None:
370 subrel.write("%s: %s\n" % (key, getattr(suite, dbfield)))
372 for key, dbfield in boolattrs:
373 if getattr(suite, dbfield, False):
374 subrel.write("%s: yes\n" % (key))
376 subrel.write("Component: %s%s\n" % (suite_suffix, comp))
378 # Urgh, but until we have all the suite/component/arch stuff in the DB,
379 # this'll have to do
380 arch = os.path.split(dirpath)[-1]
381 if arch.startswith('binary-'):
382 arch = arch[7:]
384 subrel.write("Architecture: %s\n" % (arch))
385 subrel.close()
387 os.rename(subfile + '.new', subfile)
389 # Now that we have done the groundwork, we want to get off and add the files with
390 # their checksums to the main Release file
391 oldcwd = os.getcwd()
393 os.chdir(self.suite_path())
395 hashes = [x for x in RELEASE_HASHES if x.db_name in suite.checksums]
397 fileinfo = {}
398 fileinfo_byhash = {}
400 uncompnotseen = {}
402 for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True):
403 # SuiteSuffix deprecation:
404 # components on security-master are updates/{main,contrib,non-free}, but
405 # we want dists/${suite}/main. Until we can rename the components,
406 # we cheat by having an updates -> . symlink. This should not be visited.
407 if cnf_suite_suffix: 407 ↛ 408line 407 didn't jump to line 408, because the condition on line 407 was never true
408 path = os.path.join(dirpath, cnf_suite_suffix)
409 try:
410 target = os.readlink(path)
411 if target == ".":
412 dirnames.remove(cnf_suite_suffix)
413 except (OSError, ValueError):
414 pass
415 for entry in filenames:
416 if dirpath == '.' and entry in ["Release", "Release.gpg", "InRelease"]:
417 continue
419 filename = os.path.join(dirpath.lstrip('./'), entry)
421 if re_includeinrelease_byhash.match(entry):
422 fileinfo[filename] = fileinfo_byhash[filename] = {}
423 elif re_includeinrelease_plain.match(entry): 423 ↛ 424, 423 ↛ 4272 missed branches: 1) line 423 didn't jump to line 424, because the condition on line 423 was never true, 2) line 423 didn't jump to line 427, because the condition on line 423 was never false
424 fileinfo[filename] = {}
425 # Skip things we don't want to include
426 else:
427 continue
429 with open(filename, 'rb') as fd:
430 contents = fd.read()
432 # If we find a file for which we have a compressed version and
433 # haven't yet seen the uncompressed one, store the possibility
434 # for future use
435 if entry.endswith(".gz") and filename[:-3] not in uncompnotseen:
436 uncompnotseen[filename[:-3]] = (gzip.GzipFile, filename)
437 elif entry.endswith(".bz2") and filename[:-4] not in uncompnotseen: 437 ↛ 438line 437 didn't jump to line 438, because the condition on line 437 was never true
438 uncompnotseen[filename[:-4]] = (bz2.BZ2File, filename)
439 elif entry.endswith(".xz") and filename[:-3] not in uncompnotseen:
440 uncompnotseen[filename[:-3]] = (XzFile, filename)
441 elif entry.endswith(".zst") and filename[:-3] not in uncompnotseen: 441 ↛ 442line 441 didn't jump to line 442, because the condition on line 441 was never true
442 uncompnotseen[filename[:-3]] = (ZstdFile, filename)
444 fileinfo[filename]['len'] = len(contents)
446 for hf in hashes:
447 fileinfo[filename][hf.release_field] = hf.func(contents)
449 for filename, comp in uncompnotseen.items():
450 # If we've already seen the uncompressed file, we don't
451 # need to do anything again
452 if filename in fileinfo: 452 ↛ 453line 452 didn't jump to line 453, because the condition on line 452 was never true
453 continue
455 fileinfo[filename] = {}
457 # File handler is comp[0], filename of compressed file is comp[1]
458 contents = comp[0](comp[1], 'r').read()
460 fileinfo[filename]['len'] = len(contents)
462 for hf in hashes:
463 fileinfo[filename][hf.release_field] = hf.func(contents)
465 for field in sorted(h.release_field for h in hashes):
466 out.write('%s:\n' % field)
467 for filename in sorted(fileinfo.keys()):
468 out.write(" %s %8d %s\n" % (fileinfo[filename][field], fileinfo[filename]['len'], filename))
470 out.close()
471 os.rename(outfile + '.new', outfile)
473 self._update_hashfile_table(session, fileinfo_byhash, hashes)
474 self._make_byhash_links(fileinfo_byhash, hashes)
475 self._make_byhash_base_symlink(fileinfo_byhash, hashes)
477 sign_release_dir(suite, os.path.dirname(outfile))
479 os.chdir(oldcwd)
481 return
484def main():
485 global Logger
487 cnf = Config()
489 for i in ["Help", "Suite", "Force", "Quiet"]:
490 key = "Generate-Releases::Options::%s" % i
491 if key not in cnf: 491 ↛ 489line 491 didn't jump to line 489, because the condition on line 491 was never false
492 cnf[key] = ""
494 Arguments = [('h', "help", "Generate-Releases::Options::Help"),
495 ('a', 'archive', 'Generate-Releases::Options::Archive', 'HasArg'),
496 ('s', "suite", "Generate-Releases::Options::Suite"),
497 ('f', "force", "Generate-Releases::Options::Force"),
498 ('q', "quiet", "Generate-Releases::Options::Quiet"),
499 ('o', 'option', '', 'ArbItem')]
501 suite_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
502 Options = cnf.subtree("Generate-Releases::Options")
504 if Options["Help"]:
505 usage()
507 Logger = daklog.Logger('generate-releases')
508 pool = DakProcessPool()
510 session = DBConn().session()
512 if Options["Suite"]:
513 suites = []
514 for s in suite_names:
515 suite = get_suite(s.lower(), session)
516 if suite: 516 ↛ 519line 516 didn't jump to line 519, because the condition on line 516 was never false
517 suites.append(suite)
518 else:
519 print("cannot find suite %s" % s)
520 Logger.log(['cannot find suite %s' % s])
521 else:
522 query = session.query(Suite).filter(Suite.untouchable == False) # noqa:E712
523 if 'Archive' in Options: 523 ↛ 526line 523 didn't jump to line 526, because the condition on line 523 was never false
524 archive_names = utils.split_args(Options['Archive'])
525 query = query.join(Suite.archive).filter(Archive.archive_name.in_(archive_names))
526 suites = query.all()
528 for s in suites:
529 # Setup a multiprocessing Pool. As many workers as we have CPU cores.
530 if s.untouchable and not Options["Force"]: 530 ↛ 531line 530 didn't jump to line 531, because the condition on line 530 was never true
531 print("Skipping %s (untouchable)" % s.suite_name)
532 continue
534 if not Options["Quiet"]: 534 ↛ 536line 534 didn't jump to line 536, because the condition on line 534 was never false
535 print("Processing %s" % s.suite_name)
536 Logger.log(['Processing release file for Suite: %s' % (s.suite_name)])
537 pool.apply_async(generate_helper, (s.suite_id, ))
539 # No more work will be added to our pool, close it and then wait for all to finish
540 pool.close()
541 pool.join()
543 retcode = pool.overall_status()
545 if retcode > 0: 545 ↛ 547line 545 didn't jump to line 547, because the condition on line 545 was never true
546 # TODO: CENTRAL FUNCTION FOR THIS / IMPROVE LOGGING
547 Logger.log(['Release file generation broken: %s' % (','.join([str(x[1]) for x in pool.results]))])
549 Logger.close()
551 sys.exit(retcode)
554def generate_helper(suite_id):
555 '''
556 This function is called in a new subprocess.
557 '''
558 session = DBConn().session()
559 suite = Suite.get(suite_id, session)
561 # We allow the process handler to catch and deal with any exceptions
562 rw = ReleaseWriter(suite)
563 rw.generate_release_files()
565 return (PROC_STATUS_SUCCESS, 'Release file written for %s' % suite.suite_name)
567#######################################################################################
570if __name__ == '__main__':
571 main()