Source code for dak.generate_releases

#! /usr/bin/env python3

"""
Create all the Release files

@contact: Debian FTPMaster <ftpmaster@debian.org>
@copyright: 2011  Joerg Jaspert <joerg@debian.org>
@copyright: 2011  Mark Hymers <mhy@debian.org>
@license: GNU General Public License version 2 or later

"""

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

################################################################################

# <mhy> I wish they wouldnt leave biscuits out, thats just tempting. Damnit.

################################################################################

import bz2
import errno
import gzip
import os
import os.path
import subprocess
import sys
import time

import apt_pkg
from sqlalchemy.orm import object_session

import daklib.gpg
from daklib import daklog, utils
from daklib.config import Config
from daklib.dakmultiprocessing import PROC_STATUS_SUCCESS, DakProcessPool
from daklib.dbconn import Archive, DBConn, Suite, get_suite, get_suite_architectures
from daklib.regexes import (
    re_gensubrelease,
    re_includeinrelease_byhash,
    re_includeinrelease_plain,
)

################################################################################
Logger = None  #: Our logging object

################################################################################


[docs]def usage(exit_code=0): """Usage information""" print( """Usage: dak generate-releases [OPTIONS] Generate the Release files -a, --archive=ARCHIVE process suites in ARCHIVE -s, --suite=SUITE(s) process this suite Default: All suites not marked 'untouchable' -f, --force Allow processing of untouchable suites CAREFUL: Only to be used at (point) release time! -h, --help show this help and exit -q, --quiet Don't output progress SUITE can be a space separated list, e.g. --suite=unstable testing """ ) sys.exit(exit_code)
########################################################################
[docs]def sign_release_dir(suite, dirname): cnf = Config() if "Dinstall::SigningKeyring" in cnf or "Dinstall::SigningHomedir" in cnf: args = { "keyids": suite.signingkeys or [], "pubring": cnf.get("Dinstall::SigningPubKeyring") or None, "secring": cnf.get("Dinstall::SigningKeyring") or None, "homedir": cnf.get("Dinstall::SigningHomedir") or None, "passphrase_file": cnf.get("Dinstall::SigningPassphraseFile") or None, } relname = os.path.join(dirname, "Release") dest = os.path.join(dirname, "Release.gpg") if os.path.exists(dest): os.unlink(dest) inlinedest = os.path.join(dirname, "InRelease") if os.path.exists(inlinedest): os.unlink(inlinedest) with open(relname, "r") as stdin: with open(dest, "w") as stdout: daklib.gpg.sign(stdin, stdout, inline=False, **args) stdin.seek(0) with open(inlinedest, "w") as stdout: daklib.gpg.sign(stdin, stdout, inline=True, **args)
[docs]class XzFile: def __init__(self, filename, mode="r"): self.filename = filename
[docs] def read(self): with open(self.filename, "rb") as stdin: return subprocess.check_output(["xz", "-d"], stdin=stdin)
[docs]class ZstdFile: def __init__(self, filename, mode="r"): self.filename = filename
[docs] def read(self): with open(self.filename, "rb") as stdin: return subprocess.check_output(["zstd", "--decompress"], stdin=stdin)
[docs]class HashFunc: def __init__(self, release_field, func, db_name): self.release_field = release_field self.func = func self.db_name = db_name
RELEASE_HASHES = [ HashFunc("MD5Sum", apt_pkg.md5sum, "md5sum"), HashFunc("SHA1", apt_pkg.sha1sum, "sha1"), HashFunc("SHA256", apt_pkg.sha256sum, "sha256"), ]
[docs]class ReleaseWriter: def __init__(self, suite): self.suite = suite
[docs] def suite_path(self): """ Absolute path to the suite-specific files. """ suite_suffix = utils.suite_suffix(self.suite.suite_name) return os.path.join( self.suite.archive.path, "dists", self.suite.suite_name, suite_suffix )
[docs] def suite_release_path(self): """ Absolute path where Release files are physically stored. This should be a path that sorts after the dists/ directory. """ suite_suffix = utils.suite_suffix(self.suite.suite_name) return os.path.join( self.suite.archive.path, "zzz-dists", self.suite.codename or self.suite.suite_name, suite_suffix, )
[docs] def create_output_directories(self): for path in (self.suite_path(), self.suite_release_path()): try: os.makedirs(path) except OSError as e: if e.errno != errno.EEXIST: raise
[docs] def _update_hashfile_table(self, session, fileinfo, hashes): # Mark all by-hash files as obsolete. We will undo that for the ones # we still reference later. query = """ UPDATE hashfile SET unreferenced = CURRENT_TIMESTAMP WHERE suite_id = :id AND unreferenced IS NULL""" session.execute(query, {"id": self.suite.suite_id}) query = "SELECT path FROM hashfile WHERE suite_id = :id" q = session.execute(query, {"id": self.suite.suite_id}) known_hashfiles = set(row[0] for row in q) updated = set() new = set() # Update the hashfile table with new or updated files for filename in fileinfo: if not os.path.lexists(filename): # probably an uncompressed index we didn't generate continue byhashdir = os.path.join(os.path.dirname(filename), "by-hash") for h in hashes: field = h.release_field hashfile = os.path.join(byhashdir, field, fileinfo[filename][field]) if hashfile in known_hashfiles: updated.add(hashfile) else: new.add(hashfile) if updated: session.execute( """ UPDATE hashfile SET unreferenced = NULL WHERE path = ANY(:p) AND suite_id = :id""", {"p": list(updated), "id": self.suite.suite_id}, ) if new: session.execute( """ INSERT INTO hashfile (path, suite_id) VALUES (:p, :id)""", [{"p": hashfile, "id": self.suite.suite_id} for hashfile in new], ) session.commit()
[docs] def generate_release_files(self): """ Generate Release files for the given suite """ suite = self.suite session = object_session(suite) # Attribs contains a tuple of field names and the database names to use to # fill them in attribs = ( ("Origin", "origin"), ("Label", "label"), ("Suite", "release_suite_output"), ("Version", "version"), ("Codename", "codename"), ("Changelogs", "changelog_url"), ) # A "Sub" Release file has slightly different fields subattribs = ( ("Archive", "suite_name"), ("Origin", "origin"), ("Label", "label"), ("Version", "version"), ) # Boolean stuff. If we find it true in database, write out "yes" into the release file boolattrs = ( ("NotAutomatic", "notautomatic"), ("ButAutomaticUpgrades", "butautomaticupgrades"), ("Acquire-By-Hash", "byhash"), ) cnf = Config() cnf_suite_suffix = cnf.get("Dinstall::SuiteSuffix", "").rstrip("/") suite_suffix = utils.suite_suffix(suite.suite_name) self.create_output_directories() self.create_release_symlinks() outfile = os.path.join(self.suite_release_path(), "Release") out = open(outfile + ".new", "w") for key, dbfield in attribs: # Hack to skip NULL Version fields as we used to do this # We should probably just always ignore anything which is None if key in ("Version", "Changelogs") and getattr(suite, dbfield) is None: continue out.write("%s: %s\n" % (key, getattr(suite, dbfield))) out.write( "Date: %s\n" % (time.strftime("%a, %d %b %Y %H:%M:%S UTC", time.gmtime(time.time()))) ) if suite.validtime: validtime = float(suite.validtime) out.write( "Valid-Until: %s\n" % ( time.strftime( "%a, %d %b %Y %H:%M:%S UTC", time.gmtime(time.time() + validtime), ) ) ) for key, dbfield in boolattrs: if getattr(suite, dbfield, False): out.write("%s: yes\n" % (key)) skip_arch_all = True if ( suite.separate_contents_architecture_all or suite.separate_packages_architecture_all ): # According to the Repository format specification: # https://wiki.debian.org/DebianRepository/Format#No-Support-for-Architecture-all # # Clients are not expected to support Packages-all without Contents-all. At the # time of writing, it is not possible to set separate_packages_architecture_all. # However, we add this little assert to stop the bug early. # # If you are here because the assert failed, you probably want to see "update123.py" # and its advice on updating the CHECK constraint. assert suite.separate_contents_architecture_all skip_arch_all = False if not suite.separate_packages_architecture_all: out.write("No-Support-for-Architecture-all: Packages\n") architectures = get_suite_architectures( suite.suite_name, skipall=skip_arch_all, skipsrc=True, session=session ) out.write( "Architectures: %s\n" % (" ".join(a.arch_string for a in architectures)) ) components = [c.component_name for c in suite.components] out.write("Components: %s\n" % (" ".join(components))) # For exact compatibility with old g-r, write out Description here instead # of with the rest of the DB fields above if getattr(suite, "description") is not None: out.write("Description: %s\n" % suite.description) for comp in components: for dirpath, dirnames, filenames in os.walk( os.path.join(self.suite_path(), comp), topdown=True ): if not re_gensubrelease.match(dirpath): continue subfile = os.path.join(dirpath, "Release") subrel = open(subfile + ".new", "w") for key, dbfield in subattribs: if getattr(suite, dbfield) is not None: subrel.write("%s: %s\n" % (key, getattr(suite, dbfield))) for key, dbfield in boolattrs: if getattr(suite, dbfield, False): subrel.write("%s: yes\n" % (key)) subrel.write("Component: %s%s\n" % (suite_suffix, comp)) # Urgh, but until we have all the suite/component/arch stuff in the DB, # this'll have to do arch = os.path.split(dirpath)[-1] if arch.startswith("binary-"): arch = arch[7:] subrel.write("Architecture: %s\n" % (arch)) subrel.close() os.rename(subfile + ".new", subfile) # Now that we have done the groundwork, we want to get off and add the files with # their checksums to the main Release file oldcwd = os.getcwd() os.chdir(self.suite_path()) hashes = [x for x in RELEASE_HASHES if x.db_name in suite.checksums] fileinfo = {} fileinfo_byhash = {} uncompnotseen = {} for dirpath, dirnames, filenames in os.walk( ".", followlinks=True, topdown=True ): # SuiteSuffix deprecation: # components on security-master are updates/{main,contrib,non-free}, but # we want dists/${suite}/main. Until we can rename the components, # we cheat by having an updates -> . symlink. This should not be visited. if cnf_suite_suffix: path = os.path.join(dirpath, cnf_suite_suffix) try: target = os.readlink(path) if target == ".": dirnames.remove(cnf_suite_suffix) except (OSError, ValueError): pass for entry in filenames: if dirpath == "." and entry in ["Release", "Release.gpg", "InRelease"]: continue filename = os.path.join(dirpath.lstrip("./"), entry) if re_includeinrelease_byhash.match(entry): fileinfo[filename] = fileinfo_byhash[filename] = {} elif re_includeinrelease_plain.match(entry): fileinfo[filename] = {} # Skip things we don't want to include else: continue with open(filename, "rb") as fd: contents = fd.read() # If we find a file for which we have a compressed version and # haven't yet seen the uncompressed one, store the possibility # for future use if entry.endswith(".gz") and filename[:-3] not in uncompnotseen: uncompnotseen[filename[:-3]] = (gzip.GzipFile, filename) elif entry.endswith(".bz2") and filename[:-4] not in uncompnotseen: uncompnotseen[filename[:-4]] = (bz2.BZ2File, filename) elif entry.endswith(".xz") and filename[:-3] not in uncompnotseen: uncompnotseen[filename[:-3]] = (XzFile, filename) elif entry.endswith(".zst") and filename[:-3] not in uncompnotseen: uncompnotseen[filename[:-3]] = (ZstdFile, filename) fileinfo[filename]["len"] = len(contents) for hf in hashes: fileinfo[filename][hf.release_field] = hf.func(contents) for filename, comp in uncompnotseen.items(): # If we've already seen the uncompressed file, we don't # need to do anything again if filename in fileinfo: continue fileinfo[filename] = {} # File handler is comp[0], filename of compressed file is comp[1] contents = comp[0](comp[1], "r").read() fileinfo[filename]["len"] = len(contents) for hf in hashes: fileinfo[filename][hf.release_field] = hf.func(contents) for field in sorted(h.release_field for h in hashes): out.write("%s:\n" % field) for filename in sorted(fileinfo.keys()): out.write( " %s %8d %s\n" % (fileinfo[filename][field], fileinfo[filename]["len"], filename) ) out.close() os.rename(outfile + ".new", outfile) self._update_hashfile_table(session, fileinfo_byhash, hashes) self._make_byhash_links(fileinfo_byhash, hashes) self._make_byhash_base_symlink(fileinfo_byhash, hashes) sign_release_dir(suite, os.path.dirname(outfile)) os.chdir(oldcwd) return
[docs]def main(): global Logger cnf = Config() for i in ["Help", "Suite", "Force", "Quiet"]: key = "Generate-Releases::Options::%s" % i if key not in cnf: cnf[key] = "" Arguments = [ ("h", "help", "Generate-Releases::Options::Help"), ("a", "archive", "Generate-Releases::Options::Archive", "HasArg"), ("s", "suite", "Generate-Releases::Options::Suite"), ("f", "force", "Generate-Releases::Options::Force"), ("q", "quiet", "Generate-Releases::Options::Quiet"), ("o", "option", "", "ArbItem"), ] suite_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) Options = cnf.subtree("Generate-Releases::Options") if Options["Help"]: usage() Logger = daklog.Logger("generate-releases") pool = DakProcessPool() session = DBConn().session() if Options["Suite"]: suites = [] for s in suite_names: suite = get_suite(s.lower(), session) if suite: suites.append(suite) else: print("cannot find suite %s" % s) Logger.log(["cannot find suite %s" % s]) else: query = session.query(Suite).filter(Suite.untouchable == False) # noqa:E712 if "Archive" in Options: archive_names = utils.split_args(Options["Archive"]) query = query.join(Suite.archive).filter( Archive.archive_name.in_(archive_names) ) suites = query.all() for s in suites: # Setup a multiprocessing Pool. As many workers as we have CPU cores. if s.untouchable and not Options["Force"]: print("Skipping %s (untouchable)" % s.suite_name) continue if not Options["Quiet"]: print("Processing %s" % s.suite_name) Logger.log(["Processing release file for Suite: %s" % (s.suite_name)]) pool.apply_async(generate_helper, (s.suite_id,)) # No more work will be added to our pool, close it and then wait for all to finish pool.close() pool.join() retcode = pool.overall_status() if retcode > 0: # TODO: CENTRAL FUNCTION FOR THIS / IMPROVE LOGGING Logger.log( [ "Release file generation broken: %s" % (",".join([str(x[1]) for x in pool.results])) ] ) Logger.close() sys.exit(retcode)
[docs]def generate_helper(suite_id): """ This function is called in a new subprocess. """ session = DBConn().session() suite = Suite.get(suite_id, session) # We allow the process handler to catch and deal with any exceptions rw = ReleaseWriter(suite) rw.generate_release_files() return (PROC_STATUS_SUCCESS, "Release file written for %s" % suite.suite_name)
####################################################################################### if __name__ == "__main__": main()