#! /usr/bin/env python3
#
# Copyright (C) 2015, Ansgar Burchardt <ansgar@debian.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import sys
import sqlalchemy.sql as sql
from sqlalchemy.orm.exc import NoResultFound
import daklib.daklog
import daklib.utils
from daklib.archive import ArchiveTransaction
from daklib.dbconn import ArchiveFile, Component, DBBinary, DBSource, PoolFile, Suite
"""
Idea:
dak update-suite testing testing-kfreebsd
-> grab all source & binary packages from testing with a higher version
than in testing-kfreebsd (or not in -kfreebsd) and copy them
-> limited to architectures in testing-kfreebsd
-> obeys policy queues
-> copies to build queues
dak update-suite --create-in=ftp-master stable testing
-> create suite "testing" based on "stable" in archive "ftp-master"
Additional switches:
--skip-policy-queue: skip target suite's policy queue
--skip-build-queues: do not copy to build queue
--no-new-packages: do not copy new packages
-> source-based, new binaries from existing sources will be added
--only-new-packages: do not update existing packages
-> source-based, do not copy new binaries w/o source!
--also-policy-queue: also copy pending packages from policy queue
--update-overrides: update overrides as well (if different overrides are used)
--no-act
"""
[docs]def usage():
print("dak update-suite [-n|--no-act] <origin> <target>")
sys.exit(0)
[docs]class SuiteUpdater:
def __init__(
self,
transaction,
origin,
target,
new_packages=True,
also_from_policy_queue=False,
obey_policy_queue=True,
obey_build_queues=True,
update_overrides=False,
dry_run=False,
):
self.transaction = transaction
self.origin = origin
self.target = target
self.new_packages = new_packages
self.also_from_policy_queue = also_from_policy_queue
self.obey_policy_queue = obey_policy_queue
self.obey_build_queues = obey_build_queues
self.update_overrides = update_overrides
self.dry_run = dry_run
if obey_policy_queue and target.policy_queue_id is not None:
raise Exception("Not implemented...")
self.logger = None if dry_run else daklib.daklog.Logger("update-suite")
[docs] def query_new_binaries(self, additional_sources):
# Candidates are binaries in the origin suite, and optionally in its policy queue.
query = """
SELECT b.*
FROM binaries b
JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :origin
"""
if self.also_from_policy_queue:
query += """
UNION
SELECT b.*
FROM binaries b
JOIN policy_queue_upload_binaries_map pqubm ON pqubm.binary_id = b.id
JOIN policy_queue_upload pqu ON pqu.id = pqubm.policy_queue_upload_id
WHERE pqu.target_suite_id = :origin
AND pqu.policy_queue_id = (SELECT policy_queue_id FROM suite WHERE id = :origin)
"""
# Only take binaries that are for a architecture part of the target suite,
# and whose source was just added to the target suite (i.e. listed in additional_sources)
# or that have the source already available in the target suite
# or in the target suite's policy queue if we obey policy queues,
# and filter out binaries with a lower version than already in the target suite.
if self.obey_policy_queue:
cond_source_in_policy_queue = """
EXISTS (SELECT 1
FROM policy_queue_upload pqu
WHERE tmp.source = pqu.source_id
AND pqu.target_suite_id = :target
AND pqu.policy_queue_id = (SELECT policy_queue_id FROM suite WHERE id = :target))
"""
else:
cond_source_in_policy_queue = "FALSE"
query = """
WITH tmp AS ({0})
SELECT DISTINCT *
FROM tmp
WHERE tmp.architecture IN (SELECT architecture FROM suite_architectures WHERE suite = :target)
AND (tmp.source IN :additional_sources
OR EXISTS (SELECT 1
FROM src_associations sa
WHERE tmp.source = sa.source AND sa.suite = :target)
OR {1})
AND NOT EXISTS (SELECT 1
FROM binaries b2
JOIN bin_associations ba2 ON b2.id = ba2.bin AND ba2.suite = :target
WHERE tmp.package = b2.package AND tmp.architecture = b2.architecture AND b2.version >= tmp.version)
ORDER BY package, version, architecture
""".format(
query, cond_source_in_policy_queue
)
# An empty tuple generates a SQL statement with "tmp.source IN ()"
# which is not valid. Inject an invalid value in this case:
# "tmp.source IN (NULL)" is always false.
if len(additional_sources) == 0:
additional_sources = tuple([None])
params = {
"origin": self.origin.suite_id,
"target": self.target.suite_id,
"additional_sources": additional_sources,
}
return (
self.transaction.session.query(DBBinary)
.from_statement(sql.text(query))
.params(params)
)
[docs] def query_new_sources(self):
# Candidates are source packages in the origin suite, and optionally in its policy queue.
query = """
SELECT s.*
FROM source s
JOIN src_associations sa ON s.id = sa.source AND sa.suite = :origin
"""
if self.also_from_policy_queue:
query += """
UNION
SELECT s.*
FROM source s
JOIN policy_queue_upload pqu ON pqu.source_id = s.id
WHERE pqu.target_suite_id = :origin
AND pqu.policy_queue_id = (SELECT policy_queue_id FROM suite WHERE id = :origin)
"""
# Filter out source packages with a lower version than already in the target suite.
query = """
WITH tmp AS ({0})
SELECT DISTINCT *
FROM tmp
WHERE NOT EXISTS (SELECT 1
FROM source s2
JOIN src_associations sa2 ON s2.id = sa2.source AND sa2.suite = :target
WHERE s2.source = tmp.source AND s2.version >= tmp.version)
""".format(
query
)
# Optionally filter out source packages that are not already in the target suite.
if not self.new_packages:
query += """
AND EXISTS (SELECT 1
FROM source s2
JOIN src_associations sa2 ON s2.id = sa2.source AND sa2.suite = :target
WHERE s2.source = tmp.source)
"""
query += "ORDER BY source, version"
params = {"origin": self.origin.suite_id, "target": self.target.suite_id}
return (
self.transaction.session.query(DBSource)
.from_statement(sql.text(query))
.params(params)
)
[docs] def _components_for_binary(self, binary, suite):
session = self.transaction.session
return (
session.query(Component)
.join(ArchiveFile, Component.component_id == ArchiveFile.component_id)
.join(ArchiveFile.file)
.filter(PoolFile.file_id == binary.poolfile_id)
.filter(ArchiveFile.archive_id == suite.archive_id)
)
[docs] def install_binaries(self, binaries, suite):
if len(binaries) == 0:
return
# If origin and target suites are in the same archive, we can skip the
# overhead from ArchiveTransaction.copy_binary()
if self.origin.archive_id == suite.archive_id:
query = "INSERT INTO bin_associations (bin, suite) VALUES (:bin, :suite)"
target_id = suite.suite_id
params = [{"bin": b.binary_id, "suite": target_id} for b in binaries]
self.transaction.session.execute(query, params)
else:
for b in binaries:
for c in self._components_for_binary(b, suite):
self.transaction.copy_binary(b, suite, c)
[docs] def _components_for_source(self, source, suite):
session = self.transaction.session
return (
session.query(Component)
.join(ArchiveFile, Component.component_id == ArchiveFile.component_id)
.join(ArchiveFile.file)
.filter(PoolFile.file_id == source.poolfile_id)
.filter(ArchiveFile.archive_id == suite.archive_id)
)
[docs] def install_sources(self, sources, suite):
if len(sources) == 0:
return
# If origin and target suites are in the same archive, we can skip the
# overhead from ArchiveTransaction.copy_source()
if self.origin.archive_id == suite.archive_id:
query = (
"INSERT INTO src_associations (source, suite) VALUES (:source, :suite)"
)
target_id = suite.suite_id
params = [{"source": s.source_id, "suite": target_id} for s in sources]
self.transaction.session.execute(query, params)
else:
for s in sources:
for c in self._components_for_source(s, suite):
self.transaction.copy_source(s, suite, c)
[docs] def update_suite(self):
targets = set([self.target])
if self.obey_build_queues:
targets.update([bq.suite for bq in self.target.copy_queues])
target_names = sorted(s.suite_name for s in targets)
target_name = ",".join(target_names)
new_sources = self.query_new_sources().all()
additional_sources = tuple(s.source_id for s in new_sources)
for s in new_sources:
self.log(["add-source", target_name, s.source, s.version])
if not self.dry_run:
for target in targets:
self.install_sources(new_sources, target)
new_binaries = self.query_new_binaries(additional_sources).all()
for b in new_binaries:
self.log(
[
"add-binary",
target_name,
b.package,
b.version,
b.architecture.arch_string,
]
)
if not self.dry_run:
for target in targets:
self.install_binaries(new_binaries, target)
[docs] def log(self, args):
if self.logger:
self.logger.log(args)
else:
print(args)
[docs]def main():
from daklib.config import Config
config = Config()
import apt_pkg
arguments = [
("h", "help", "Update-Suite::Options::Help"),
("n", "no-act", "Update-Suite::options::NoAct"),
]
argv = apt_pkg.parse_commandline(config.Cnf, arguments, sys.argv)
try:
options = config.subtree("Update-Suite::Options")
except KeyError:
options = {}
if "Help" in options or len(argv) != 2:
usage()
origin_name = argv[0]
target_name = argv[1]
dry_run = True if "NoAct" in options else False
with ArchiveTransaction() as transaction:
session = transaction.session
try:
origin = session.query(Suite).filter_by(suite_name=origin_name).one()
except NoResultFound:
daklib.utils.fubar("Origin suite '{0}' is unknown.".format(origin_name))
try:
target = session.query(Suite).filter_by(suite_name=target_name).one()
except NoResultFound:
daklib.utils.fubar("Target suite '{0}' is unknown.".format(target_name))
su = SuiteUpdater(transaction, origin, target, dry_run=dry_run)
su.update_suite()
if dry_run:
transaction.rollback()
else:
transaction.commit()
if __name__ == "__main__":
pass