#! /usr/bin/env python3
"""Cruft checker and hole filler for overrides
@contact: Debian FTPMaster <ftpmaster@debian.org>
@copyright: 2000, 2001, 2002, 2004, 2006 James Troup <james@nocrew.org>
@opyright: 2005 Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
@copyright: 2011 Joerg Jaspert <joerg@debian.org>
@license: GNU General Public License version 2 or later
"""
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
################################################################################
######################################################################
# NB: dak check-overrides is not a good idea with New Incoming as it #
# doesn't take into account accepted. You can minimize the impact #
# of this by running it immediately after dak process-accepted but #
# that's still racy because 'dak process-new' doesn't lock with 'dak #
# process-accepted'. A better long term fix is the evil plan for #
# accepted to be in the DB. #
######################################################################
# dak check-overrides should now work fine being done during
# cron.daily, for example just before 'dak make-overrides' (after 'dak
# process-accepted' and 'dak make-suite-file-list'). At that point,
# queue/accepted should be empty and installed, so... dak
# check-overrides does now take into account suites sharing overrides
# TODO:
# * Only update out-of-sync overrides when corresponding versions are equal to
# some degree
# * consistency checks like:
# - section=debian-installer only for udeb and # dsc
# - priority=optional if dsc
# - (suite, package, 'dsc') is unique,
# - just as (suite, package, (u)deb) (yes, across components!)
# - sections match their component (each component has an own set of sections,
# could probably be reduced...)
################################################################################
import sys
import apt_pkg
from daklib import daklog, utils
from daklib.config import Config
from daklib.dbconn import (
Component,
DBConn,
OverrideType,
Suite,
get_component,
get_override_type,
get_priorities,
get_priority,
get_sections,
get_suite,
)
################################################################################
Options = None #: Commandline arguments parsed into this
Logger = None #: Our logging object
sections = {}
priorities = {}
blacklist = {}
################################################################################
[docs]def usage(exit_code=0):
print(
"""Usage: dak check-overrides
Check for cruft in overrides.
-n, --no-action don't do anything
-h, --help show this help and exit"""
)
sys.exit(exit_code)
################################################################################
[docs]def process(osuite, affected_suites, originosuite, component, otype, session):
global Logger, Options, sections, priorities
o = get_suite(osuite, session)
if o is None:
utils.fubar("Suite '%s' not recognised." % (osuite))
osuite_id = o.suite_id
originosuite_id = None
if originosuite:
oo = get_suite(originosuite, session)
if oo is None:
utils.fubar("Suite '%s' not recognised." % (originosuite))
originosuite_id = oo.suite_id
c = get_component(component, session)
if c is None:
utils.fubar("Component '%s' not recognised." % (component))
component_id = c.component_id
ot = get_override_type(otype, session)
if ot is None:
utils.fubar(
"Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype)
)
type_id = ot.overridetype_id
dsc_type_id = get_override_type("dsc", session).overridetype_id
source_priority_id = get_priority("optional", session).priority_id
if otype == "deb" or otype == "udeb":
packages = {}
# TODO: Fix to use placeholders (check how to with arrays)
q = session.execute(
"""
SELECT b.package
FROM binaries b
JOIN bin_associations ba ON b.id = ba.bin
JOIN suite ON ba.suite = suite.id
JOIN files_archive_map af ON b.file = af.file_id AND suite.archive_id = af.archive_id
WHERE b.type = :otype AND ba.suite IN :affected_suites AND af.component_id = :component_id
""",
{
"otype": otype,
"affected_suites": tuple(affected_suites),
"component_id": component_id,
},
)
for i in q.fetchall():
packages[i[0]] = 0
src_packages = {}
q = session.execute(
"""
SELECT s.source FROM source s
JOIN src_associations sa ON s.id = sa.source
JOIN suite ON sa.suite = suite.id
JOIN files_archive_map af ON s.file = af.file_id AND suite.archive_id = af.archive_id
WHERE sa.suite IN :affected_suites AND af.component_id = :component_id
""",
{"affected_suites": tuple(affected_suites), "component_id": component_id},
)
for i in q.fetchall():
src_packages[i[0]] = 0
# -----------
# Drop unused overrides
q = session.execute(
"""SELECT package, priority, section, maintainer
FROM override WHERE suite = :suite_id
AND component = :component_id AND type = :type_id""",
{"suite_id": osuite_id, "component_id": component_id, "type_id": type_id},
)
# We're already within a transaction
if otype == "dsc":
for i in q.fetchall():
package = i[0]
if package in src_packages:
src_packages[package] = 1
else:
if package in blacklist:
utils.warn("%s in incoming, not touching" % package)
continue
Logger.log(
[
"removing unused override",
osuite,
component,
otype,
package,
priorities[i[1]],
sections[i[2]],
i[3],
]
)
if not Options["No-Action"]:
session.execute(
"""DELETE FROM override WHERE package = :package
AND suite = :suite_id AND component = :component_id
AND type = :type_id
AND created < now() - interval '14 days'""",
{
"package": package,
"suite_id": osuite_id,
"component_id": component_id,
"type_id": type_id,
},
)
# create source overrides based on binary overrides, as source
# overrides not always get created
q = session.execute(
"""SELECT package, priority, section, maintainer
FROM override WHERE suite = :suite_id AND component = :component_id""",
{"suite_id": osuite_id, "component_id": component_id},
)
for i in q.fetchall():
package = i[0]
if package not in src_packages or src_packages[package]:
continue
src_packages[package] = 1
Logger.log(
[
"add missing override",
osuite,
component,
otype,
package,
"source",
sections[i[2]],
i[3],
]
)
if not Options["No-Action"]:
session.execute(
"""INSERT INTO override (package, suite, component,
priority, section, type, maintainer)
VALUES (:package, :suite_id, :component_id,
:priority_id, :section_id, :type_id, :maintainer)""",
{
"package": package,
"suite_id": osuite_id,
"component_id": component_id,
"priority_id": source_priority_id,
"section_id": i[2],
"type_id": dsc_type_id,
"maintainer": i[3],
},
)
# Check whether originosuite has an override for us we can
# copy
if originosuite:
q = session.execute(
"""SELECT origin.package, origin.priority, origin.section,
origin.maintainer, target.priority, target.section,
target.maintainer
FROM override origin
LEFT JOIN override target ON (origin.package = target.package
AND target.suite = :suite_id
AND origin.component = target.component
AND origin.type = target.type)
WHERE origin.suite = :originsuite_id
AND origin.component = :component_id
AND origin.type = :type_id""",
{
"suite_id": osuite_id,
"originsuite_id": originosuite_id,
"component_id": component_id,
"type_id": type_id,
},
)
for i in q.fetchall():
package = i[0]
if package not in src_packages or src_packages[package]:
if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
Logger.log(
[
"syncing override",
osuite,
component,
otype,
package,
"source",
sections[i[5]],
i[6],
"source",
sections[i[2]],
i[3],
]
)
if not Options["No-Action"]:
session.execute(
"""UPDATE override
SET priority = :priority,
section = :section,
maintainer = :maintainer
WHERE package = :package AND suite = :suite_id
AND component = :component_id AND type = :type_id""",
{
"priority": i[1],
"section": i[2],
"maintainer": i[3],
"package": package,
"suite_id": osuite_id,
"component_id": component_id,
"type_id": dsc_type_id,
},
)
continue
# we can copy
src_packages[package] = 1
Logger.log(
[
"copying missing override",
osuite,
component,
otype,
package,
"source",
sections[i[2]],
i[3],
]
)
if not Options["No-Action"]:
session.execute(
"""INSERT INTO override (package, suite, component,
priority, section, type, maintainer)
VALUES (:package, :suite_id, :component_id,
:priority_id, :section_id, :type_id,
:maintainer)""",
{
"package": package,
"suite_id": osuite_id,
"component_id": component_id,
"priority_id": source_priority_id,
"section_id": i[2],
"type_id": dsc_type_id,
"maintainer": i[3],
},
)
for package, hasoverride in list(src_packages.items()):
if not hasoverride:
utils.warn("%s has no override!" % package)
else: # binary override
for i in q.fetchall():
package = i[0]
if package in packages:
packages[package] = 1
else:
if package in blacklist:
utils.warn("%s in incoming, not touching" % package)
continue
Logger.log(
[
"removing unused override",
osuite,
component,
otype,
package,
priorities[i[1]],
sections[i[2]],
i[3],
]
)
if not Options["No-Action"]:
session.execute(
"""DELETE FROM override
WHERE package = :package AND suite = :suite_id
AND component = :component_id AND type = :type_id
AND created < now() - interval '14 days'""",
{
"package": package,
"suite_id": osuite_id,
"component_id": component_id,
"type_id": type_id,
},
)
# Check whether originosuite has an override for us we can
# copy
if originosuite:
q = session.execute(
"""SELECT origin.package, origin.priority, origin.section,
origin.maintainer, target.priority, target.section,
target.maintainer
FROM override origin LEFT JOIN override target
ON (origin.package = target.package
AND target.suite = :suite_id
AND origin.component = target.component
AND origin.type = target.type)
WHERE origin.suite = :originsuite_id
AND origin.component = :component_id
AND origin.type = :type_id""",
{
"suite_id": osuite_id,
"originsuite_id": originosuite_id,
"component_id": component_id,
"type_id": type_id,
},
)
for i in q.fetchall():
package = i[0]
if package not in packages or packages[package]:
if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
Logger.log(
[
"syncing override",
osuite,
component,
otype,
package,
priorities[i[4]],
sections[i[5]],
i[6],
priorities[i[1]],
sections[i[2]],
i[3],
]
)
if not Options["No-Action"]:
session.execute(
"""UPDATE override
SET priority = :priority_id,
section = :section_id,
maintainer = :maintainer
WHERE package = :package
AND suite = :suite_id
AND component = :component_id
AND type = :type_id""",
{
"priority_id": i[1],
"section_id": i[2],
"maintainer": i[3],
"package": package,
"suite_id": osuite_id,
"component_id": component_id,
"type_id": type_id,
},
)
continue
# we can copy
packages[package] = 1
Logger.log(
[
"copying missing override",
osuite,
component,
otype,
package,
priorities[i[1]],
sections[i[2]],
i[3],
]
)
if not Options["No-Action"]:
session.execute(
"""INSERT INTO override (package, suite, component,
priority, section, type, maintainer)
VALUES (:package, :suite_id, :component_id,
:priority_id, :section_id, :type_id, :maintainer)""",
{
"package": package,
"suite_id": osuite_id,
"component_id": component_id,
"priority_id": i[1],
"section_id": i[2],
"type_id": type_id,
"maintainer": i[3],
},
)
for package, hasoverride in list(packages.items()):
if not hasoverride:
utils.warn("%s has no override!" % package)
session.commit()
sys.stdout.flush()
################################################################################
[docs]def main():
global Logger, Options, sections, priorities
cnf = Config()
Arguments = [
("h", "help", "Check-Overrides::Options::Help"),
("n", "no-action", "Check-Overrides::Options::No-Action"),
]
for i in ["help", "no-action"]:
key = "Check-Overrides::Options::%s" % i
if key not in cnf:
cnf[key] = ""
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Check-Overrides::Options")
if Options["Help"]:
usage()
session = DBConn().session()
# init sections, priorities:
# We need forward and reverse
sections = get_sections(session)
for name, entry in list(sections.items()):
sections[entry] = name
priorities = get_priorities(session)
for name, entry in list(priorities.items()):
priorities[entry] = name
if not Options["No-Action"]:
Logger = daklog.Logger("check-overrides")
else:
Logger = daklog.Logger("check-overrides", 1)
for suite in session.query(Suite).filter(
Suite.overrideprocess == True # noqa:E712
):
originosuite = None
originremark = ""
if suite.overrideorigin is not None:
originosuite = get_suite(suite.overrideorigin, session)
if originosuite is None:
utils.fubar(
"%s has an override origin suite of %s but it doesn't exist!"
% (suite.suite_name, suite.overrideorigin)
)
originosuite = originosuite.suite_name
originremark = " taking missing from %s" % originosuite
print("Processing %s%s..." % (suite.suite_name, originremark))
# Get a list of all suites that use the override file of 'suite.suite_name' as
# well as the suite
ocodename = suite.codename
suiteids = [
x.suite_id
for x in session.query(Suite)
.filter(Suite.overridecodename == ocodename)
.all()
]
if suite.suite_id not in suiteids:
suiteids.append(suite.suite_id)
if len(suiteids) < 1:
utils.fubar("Couldn't find id's of all suites: %s" % suiteids)
for component in session.query(Component).all():
# It is crucial for the dsc override creation based on binary
# overrides that 'dsc' goes first
component_name = component.component_name
otypes = ["dsc"]
for ot in session.query(OverrideType):
if ot.overridetype == "dsc":
continue
otypes.append(ot.overridetype)
for otype in otypes:
print(
"Processing %s [%s - %s]"
% (suite.suite_name, component_name, otype)
)
sys.stdout.flush()
process(
suite.suite_name,
suiteids,
originosuite,
component_name,
otype,
session,
)
Logger.close()
################################################################################
if __name__ == "__main__":
main()