1#! /usr/bin/env python3
3"""
4Create all the Release files
6@contact: Debian FTPMaster <ftpmaster@debian.org>
7@copyright: 2011 Joerg Jaspert <joerg@debian.org>
8@copyright: 2011 Mark Hymers <mhy@debian.org>
9@license: GNU General Public License version 2 or later
11"""
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27################################################################################
29# <mhy> I wish they wouldnt leave biscuits out, thats just tempting. Damnit.
31################################################################################
33import bz2
34import errno
35import gzip
36import os
37import os.path
38import subprocess
39import sys
40import time
42import apt_pkg
43from sqlalchemy.orm import object_session
45import daklib.gpg
46from daklib import daklog, utils
47from daklib.config import Config
48from daklib.dakmultiprocessing import PROC_STATUS_SUCCESS, DakProcessPool
49from daklib.dbconn import Archive, DBConn, Suite, get_suite, get_suite_architectures
50from daklib.regexes import (
51 re_gensubrelease,
52 re_includeinrelease_byhash,
53 re_includeinrelease_plain,
54)
56################################################################################
57Logger = None #: Our logging object
59################################################################################
62def usage(exit_code=0):
63 """Usage information"""
65 print(
66 """Usage: dak generate-releases [OPTIONS]
67Generate the Release files
69 -a, --archive=ARCHIVE process suites in ARCHIVE
70 -s, --suite=SUITE(s) process this suite
71 Default: All suites not marked 'untouchable'
72 -f, --force Allow processing of untouchable suites
73 CAREFUL: Only to be used at (point) release time!
74 -h, --help show this help and exit
75 -q, --quiet Don't output progress
77SUITE can be a space separated list, e.g.
78 --suite=unstable testing
79 """
80 )
81 sys.exit(exit_code)
84########################################################################
87def sign_release_dir(suite, dirname):
88 cnf = Config()
90 if "Dinstall::SigningKeyring" in cnf or "Dinstall::SigningHomedir" in cnf: 90 ↛ exitline 90 didn't return from function 'sign_release_dir', because the condition on line 90 was never false
91 args = {
92 "keyids": suite.signingkeys or [],
93 "pubring": cnf.get("Dinstall::SigningPubKeyring") or None,
94 "secring": cnf.get("Dinstall::SigningKeyring") or None,
95 "homedir": cnf.get("Dinstall::SigningHomedir") or None,
96 "passphrase_file": cnf.get("Dinstall::SigningPassphraseFile") or None,
97 }
99 relname = os.path.join(dirname, "Release")
101 dest = os.path.join(dirname, "Release.gpg")
102 if os.path.exists(dest):
103 os.unlink(dest)
105 inlinedest = os.path.join(dirname, "InRelease")
106 if os.path.exists(inlinedest):
107 os.unlink(inlinedest)
109 with open(relname, "r") as stdin:
110 with open(dest, "w") as stdout:
111 daklib.gpg.sign(stdin, stdout, inline=False, **args)
112 stdin.seek(0)
113 with open(inlinedest, "w") as stdout:
114 daklib.gpg.sign(stdin, stdout, inline=True, **args)
117class XzFile:
118 def __init__(self, filename, mode="r"):
119 self.filename = filename
121 def read(self):
122 with open(self.filename, "rb") as stdin:
123 return subprocess.check_output(["xz", "-d"], stdin=stdin)
126class ZstdFile:
127 def __init__(self, filename, mode="r"):
128 self.filename = filename
130 def read(self):
131 with open(self.filename, "rb") as stdin:
132 return subprocess.check_output(["zstd", "--decompress"], stdin=stdin)
135class HashFunc:
136 def __init__(self, release_field, func, db_name):
137 self.release_field = release_field
138 self.func = func
139 self.db_name = db_name
142RELEASE_HASHES = [
143 HashFunc("MD5Sum", apt_pkg.md5sum, "md5sum"),
144 HashFunc("SHA1", apt_pkg.sha1sum, "sha1"),
145 HashFunc("SHA256", apt_pkg.sha256sum, "sha256"),
146]
149class ReleaseWriter:
150 def __init__(self, suite):
151 self.suite = suite
153 def suite_path(self):
154 """
155 Absolute path to the suite-specific files.
156 """
157 suite_suffix = utils.suite_suffix(self.suite.suite_name)
159 return os.path.join(
160 self.suite.archive.path, "dists", self.suite.suite_name, suite_suffix
161 )
163 def suite_release_path(self):
164 """
165 Absolute path where Release files are physically stored.
166 This should be a path that sorts after the dists/ directory.
167 """
168 suite_suffix = utils.suite_suffix(self.suite.suite_name)
170 return os.path.join(
171 self.suite.archive.path,
172 "zzz-dists",
173 self.suite.codename or self.suite.suite_name,
174 suite_suffix,
175 )
177 def create_release_symlinks(self):
178 """
179 Create symlinks for Release files.
180 This creates the symlinks for Release files in the `suite_path`
181 to the actual files in `suite_release_path`.
182 """
183 relpath = os.path.relpath(self.suite_release_path(), self.suite_path())
184 for f in ("Release", "Release.gpg", "InRelease"):
185 source = os.path.join(relpath, f)
186 dest = os.path.join(self.suite_path(), f)
187 if os.path.lexists(dest):
188 if not os.path.islink(dest): 188 ↛ 189line 188 didn't jump to line 189, because the condition on line 188 was never true
189 os.unlink(dest)
190 elif os.readlink(dest) == source: 190 ↛ 193line 190 didn't jump to line 193, because the condition on line 190 was never false
191 continue
192 else:
193 os.unlink(dest)
194 os.symlink(source, dest)
196 def create_output_directories(self):
197 for path in (self.suite_path(), self.suite_release_path()):
198 try:
199 os.makedirs(path)
200 except OSError as e:
201 if e.errno != errno.EEXIST: 201 ↛ 202line 201 didn't jump to line 202, because the condition on line 201 was never true
202 raise
204 def _update_hashfile_table(self, session, fileinfo, hashes):
205 # Mark all by-hash files as obsolete. We will undo that for the ones
206 # we still reference later.
207 query = """
208 UPDATE hashfile SET unreferenced = CURRENT_TIMESTAMP
209 WHERE suite_id = :id AND unreferenced IS NULL"""
210 session.execute(query, {"id": self.suite.suite_id})
212 query = "SELECT path FROM hashfile WHERE suite_id = :id"
213 q = session.execute(query, {"id": self.suite.suite_id})
214 known_hashfiles = set(row[0] for row in q)
215 updated = set()
216 new = set()
218 # Update the hashfile table with new or updated files
219 for filename in fileinfo:
220 if not os.path.lexists(filename): 220 ↛ 222line 220 didn't jump to line 222, because the condition on line 220 was never true
221 # probably an uncompressed index we didn't generate
222 continue
223 byhashdir = os.path.join(os.path.dirname(filename), "by-hash")
224 for h in hashes:
225 field = h.release_field
226 hashfile = os.path.join(byhashdir, field, fileinfo[filename][field])
227 if hashfile in known_hashfiles:
228 updated.add(hashfile)
229 else:
230 new.add(hashfile)
232 if updated:
233 session.execute(
234 """
235 UPDATE hashfile SET unreferenced = NULL
236 WHERE path = ANY(:p) AND suite_id = :id""",
237 {"p": list(updated), "id": self.suite.suite_id},
238 )
239 if new:
240 session.execute(
241 """
242 INSERT INTO hashfile (path, suite_id)
243 VALUES (:p, :id)""",
244 [{"p": hashfile, "id": self.suite.suite_id} for hashfile in new],
245 )
247 session.commit()
249 def _make_byhash_links(self, fileinfo, hashes):
250 # Create hardlinks in by-hash directories
251 for filename in fileinfo:
252 if not os.path.lexists(filename): 252 ↛ 254line 252 didn't jump to line 254, because the condition on line 252 was never true
253 # probably an uncompressed index we didn't generate
254 continue
256 for h in hashes:
257 field = h.release_field
258 hashfile = os.path.join(
259 os.path.dirname(filename),
260 "by-hash",
261 field,
262 fileinfo[filename][field],
263 )
264 try:
265 os.makedirs(os.path.dirname(hashfile))
266 except OSError as exc:
267 if exc.errno != errno.EEXIST: 267 ↛ 268line 267 didn't jump to line 268, because the condition on line 267 was never true
268 raise
269 try:
270 os.link(filename, hashfile)
271 except OSError as exc:
272 if exc.errno != errno.EEXIST: 272 ↛ 273line 272 didn't jump to line 273, because the condition on line 272 was never true
273 raise
275 def _make_byhash_base_symlink(self, fileinfo, hashes):
276 # Create symlinks to files in by-hash directories
277 for filename in fileinfo:
278 if not os.path.lexists(filename): 278 ↛ 280line 278 didn't jump to line 280, because the condition on line 278 was never true
279 # probably an uncompressed index we didn't generate
280 continue
282 besthash = hashes[-1]
283 field = besthash.release_field
284 hashfilebase = os.path.join("by-hash", field, fileinfo[filename][field])
285 hashfile = os.path.join(os.path.dirname(filename), hashfilebase)
287 assert os.path.exists(hashfile), "by-hash file {} is missing".format(
288 hashfile
289 )
291 os.unlink(filename)
292 os.symlink(hashfilebase, filename)
294 def generate_release_files(self):
295 """
296 Generate Release files for the given suite
297 """
299 suite = self.suite
300 session = object_session(suite)
302 # Attribs contains a tuple of field names and the database names to use to
303 # fill them in
304 attribs = (
305 ("Origin", "origin"),
306 ("Label", "label"),
307 ("Suite", "release_suite_output"),
308 ("Version", "version"),
309 ("Codename", "codename"),
310 ("Changelogs", "changelog_url"),
311 )
313 # A "Sub" Release file has slightly different fields
314 subattribs = (
315 ("Archive", "suite_name"),
316 ("Origin", "origin"),
317 ("Label", "label"),
318 ("Version", "version"),
319 )
321 # Boolean stuff. If we find it true in database, write out "yes" into the release file
322 boolattrs = (
323 ("NotAutomatic", "notautomatic"),
324 ("ButAutomaticUpgrades", "butautomaticupgrades"),
325 ("Acquire-By-Hash", "byhash"),
326 )
328 cnf = Config()
329 cnf_suite_suffix = cnf.get("Dinstall::SuiteSuffix", "").rstrip("/")
331 suite_suffix = utils.suite_suffix(suite.suite_name)
333 self.create_output_directories()
334 self.create_release_symlinks()
336 outfile = os.path.join(self.suite_release_path(), "Release")
337 out = open(outfile + ".new", "w")
339 for key, dbfield in attribs:
340 # Hack to skip NULL Version fields as we used to do this
341 # We should probably just always ignore anything which is None
342 if key in ("Version", "Changelogs") and getattr(suite, dbfield) is None:
343 continue
345 out.write("%s: %s\n" % (key, getattr(suite, dbfield)))
347 out.write(
348 "Date: %s\n"
349 % (time.strftime("%a, %d %b %Y %H:%M:%S UTC", time.gmtime(time.time())))
350 )
352 if suite.validtime: 352 ↛ 364line 352 didn't jump to line 364, because the condition on line 352 was never false
353 validtime = float(suite.validtime)
354 out.write(
355 "Valid-Until: %s\n"
356 % (
357 time.strftime(
358 "%a, %d %b %Y %H:%M:%S UTC",
359 time.gmtime(time.time() + validtime),
360 )
361 )
362 )
364 for key, dbfield in boolattrs:
365 if getattr(suite, dbfield, False):
366 out.write("%s: yes\n" % (key))
368 skip_arch_all = True
369 if (
370 suite.separate_contents_architecture_all
371 or suite.separate_packages_architecture_all
372 ):
373 # According to the Repository format specification:
374 # https://wiki.debian.org/DebianRepository/Format#No-Support-for-Architecture-all
375 #
376 # Clients are not expected to support Packages-all without Contents-all. At the
377 # time of writing, it is not possible to set separate_packages_architecture_all.
378 # However, we add this little assert to stop the bug early.
379 #
380 # If you are here because the assert failed, you probably want to see "update123.py"
381 # and its advice on updating the CHECK constraint.
382 assert suite.separate_contents_architecture_all
383 skip_arch_all = False
385 if not suite.separate_packages_architecture_all: 385 ↛ 388line 385 didn't jump to line 388, because the condition on line 385 was never false
386 out.write("No-Support-for-Architecture-all: Packages\n")
388 architectures = get_suite_architectures(
389 suite.suite_name, skipall=skip_arch_all, skipsrc=True, session=session
390 )
392 out.write(
393 "Architectures: %s\n" % (" ".join(a.arch_string for a in architectures))
394 )
396 components = [c.component_name for c in suite.components]
398 out.write("Components: %s\n" % (" ".join(components)))
400 # For exact compatibility with old g-r, write out Description here instead
401 # of with the rest of the DB fields above
402 if getattr(suite, "description") is not None: 402 ↛ 403line 402 didn't jump to line 403, because the condition on line 402 was never true
403 out.write("Description: %s\n" % suite.description)
405 for comp in components:
406 for dirpath, dirnames, filenames in os.walk(
407 os.path.join(self.suite_path(), comp), topdown=True
408 ):
409 if not re_gensubrelease.match(dirpath):
410 continue
412 subfile = os.path.join(dirpath, "Release")
413 subrel = open(subfile + ".new", "w")
415 for key, dbfield in subattribs:
416 if getattr(suite, dbfield) is not None:
417 subrel.write("%s: %s\n" % (key, getattr(suite, dbfield)))
419 for key, dbfield in boolattrs:
420 if getattr(suite, dbfield, False):
421 subrel.write("%s: yes\n" % (key))
423 subrel.write("Component: %s%s\n" % (suite_suffix, comp))
425 # Urgh, but until we have all the suite/component/arch stuff in the DB,
426 # this'll have to do
427 arch = os.path.split(dirpath)[-1]
428 if arch.startswith("binary-"):
429 arch = arch[7:]
431 subrel.write("Architecture: %s\n" % (arch))
432 subrel.close()
434 os.rename(subfile + ".new", subfile)
436 # Now that we have done the groundwork, we want to get off and add the files with
437 # their checksums to the main Release file
438 oldcwd = os.getcwd()
440 os.chdir(self.suite_path())
442 hashes = [x for x in RELEASE_HASHES if x.db_name in suite.checksums]
444 fileinfo = {}
445 fileinfo_byhash = {}
447 uncompnotseen = {}
449 for dirpath, dirnames, filenames in os.walk(
450 ".", followlinks=True, topdown=True
451 ):
452 # SuiteSuffix deprecation:
453 # components on security-master are updates/{main,contrib,non-free}, but
454 # we want dists/${suite}/main. Until we can rename the components,
455 # we cheat by having an updates -> . symlink. This should not be visited.
456 if cnf_suite_suffix: 456 ↛ 457line 456 didn't jump to line 457, because the condition on line 456 was never true
457 path = os.path.join(dirpath, cnf_suite_suffix)
458 try:
459 target = os.readlink(path)
460 if target == ".":
461 dirnames.remove(cnf_suite_suffix)
462 except (OSError, ValueError):
463 pass
464 for entry in filenames:
465 if dirpath == "." and entry in ["Release", "Release.gpg", "InRelease"]:
466 continue
468 filename = os.path.join(dirpath.lstrip("./"), entry)
470 if re_includeinrelease_byhash.match(entry):
471 fileinfo[filename] = fileinfo_byhash[filename] = {}
472 elif re_includeinrelease_plain.match(entry): 472 ↛ 473, 472 ↛ 4762 missed branches: 1) line 472 didn't jump to line 473, because the condition on line 472 was never true, 2) line 472 didn't jump to line 476, because the condition on line 472 was never false
473 fileinfo[filename] = {}
474 # Skip things we don't want to include
475 else:
476 continue
478 with open(filename, "rb") as fd:
479 contents = fd.read()
481 # If we find a file for which we have a compressed version and
482 # haven't yet seen the uncompressed one, store the possibility
483 # for future use
484 if entry.endswith(".gz") and filename[:-3] not in uncompnotseen:
485 uncompnotseen[filename[:-3]] = (gzip.GzipFile, filename)
486 elif entry.endswith(".bz2") and filename[:-4] not in uncompnotseen: 486 ↛ 487line 486 didn't jump to line 487, because the condition on line 486 was never true
487 uncompnotseen[filename[:-4]] = (bz2.BZ2File, filename)
488 elif entry.endswith(".xz") and filename[:-3] not in uncompnotseen:
489 uncompnotseen[filename[:-3]] = (XzFile, filename)
490 elif entry.endswith(".zst") and filename[:-3] not in uncompnotseen: 490 ↛ 491line 490 didn't jump to line 491, because the condition on line 490 was never true
491 uncompnotseen[filename[:-3]] = (ZstdFile, filename)
493 fileinfo[filename]["len"] = len(contents)
495 for hf in hashes:
496 fileinfo[filename][hf.release_field] = hf.func(contents)
498 for filename, comp in uncompnotseen.items():
499 # If we've already seen the uncompressed file, we don't
500 # need to do anything again
501 if filename in fileinfo: 501 ↛ 502line 501 didn't jump to line 502, because the condition on line 501 was never true
502 continue
504 fileinfo[filename] = {}
506 # File handler is comp[0], filename of compressed file is comp[1]
507 contents = comp[0](comp[1], "r").read()
509 fileinfo[filename]["len"] = len(contents)
511 for hf in hashes:
512 fileinfo[filename][hf.release_field] = hf.func(contents)
514 for field in sorted(h.release_field for h in hashes):
515 out.write("%s:\n" % field)
516 for filename in sorted(fileinfo.keys()):
517 out.write(
518 " %s %8d %s\n"
519 % (fileinfo[filename][field], fileinfo[filename]["len"], filename)
520 )
522 out.close()
523 os.rename(outfile + ".new", outfile)
525 self._update_hashfile_table(session, fileinfo_byhash, hashes)
526 self._make_byhash_links(fileinfo_byhash, hashes)
527 self._make_byhash_base_symlink(fileinfo_byhash, hashes)
529 sign_release_dir(suite, os.path.dirname(outfile))
531 os.chdir(oldcwd)
533 return
536def main():
537 global Logger
539 cnf = Config()
541 for i in ["Help", "Suite", "Force", "Quiet"]:
542 key = "Generate-Releases::Options::%s" % i
543 if key not in cnf: 543 ↛ 541line 543 didn't jump to line 541, because the condition on line 543 was never false
544 cnf[key] = ""
546 Arguments = [
547 ("h", "help", "Generate-Releases::Options::Help"),
548 ("a", "archive", "Generate-Releases::Options::Archive", "HasArg"),
549 ("s", "suite", "Generate-Releases::Options::Suite"),
550 ("f", "force", "Generate-Releases::Options::Force"),
551 ("q", "quiet", "Generate-Releases::Options::Quiet"),
552 ("o", "option", "", "ArbItem"),
553 ]
555 suite_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
556 Options = cnf.subtree("Generate-Releases::Options")
558 if Options["Help"]:
559 usage()
561 Logger = daklog.Logger("generate-releases")
562 pool = DakProcessPool()
564 session = DBConn().session()
566 if Options["Suite"]:
567 suites = []
568 for s in suite_names:
569 suite = get_suite(s.lower(), session)
570 if suite: 570 ↛ 573line 570 didn't jump to line 573, because the condition on line 570 was never false
571 suites.append(suite)
572 else:
573 print("cannot find suite %s" % s)
574 Logger.log(["cannot find suite %s" % s])
575 else:
576 query = session.query(Suite).filter(Suite.untouchable == False) # noqa:E712
577 if "Archive" in Options: 577 ↛ 582line 577 didn't jump to line 582, because the condition on line 577 was never false
578 archive_names = utils.split_args(Options["Archive"])
579 query = query.join(Suite.archive).filter(
580 Archive.archive_name.in_(archive_names)
581 )
582 suites = query.all()
584 for s in suites:
585 # Setup a multiprocessing Pool. As many workers as we have CPU cores.
586 if s.untouchable and not Options["Force"]: 586 ↛ 587line 586 didn't jump to line 587, because the condition on line 586 was never true
587 print("Skipping %s (untouchable)" % s.suite_name)
588 continue
590 if not Options["Quiet"]: 590 ↛ 592line 590 didn't jump to line 592, because the condition on line 590 was never false
591 print("Processing %s" % s.suite_name)
592 Logger.log(["Processing release file for Suite: %s" % (s.suite_name)])
593 pool.apply_async(generate_helper, (s.suite_id,))
595 # No more work will be added to our pool, close it and then wait for all to finish
596 pool.close()
597 pool.join()
599 retcode = pool.overall_status()
601 if retcode > 0: 601 ↛ 603line 601 didn't jump to line 603, because the condition on line 601 was never true
602 # TODO: CENTRAL FUNCTION FOR THIS / IMPROVE LOGGING
603 Logger.log(
604 [
605 "Release file generation broken: %s"
606 % (",".join([str(x[1]) for x in pool.results]))
607 ]
608 )
610 Logger.close()
612 sys.exit(retcode)
615def generate_helper(suite_id):
616 """
617 This function is called in a new subprocess.
618 """
619 session = DBConn().session()
620 suite = Suite.get(suite_id, session)
622 # We allow the process handler to catch and deal with any exceptions
623 rw = ReleaseWriter(suite)
624 rw.generate_release_files()
626 return (PROC_STATUS_SUCCESS, "Release file written for %s" % suite.suite_name)
629#######################################################################################
632if __name__ == "__main__":
633 main()