#! /usr/bin/env python3
""" Overview of the DEFERRED queue, based on queue-report """
# Copyright (C) 2001, 2002, 2003, 2005, 2006 James Troup <james@nocrew.org>
# Copyright (C) 2008 Thomas Viehmann <tv@beamnet.de>
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
################################################################################
import html
import sys
import os
import re
import time
import apt_pkg
import rrdtool
from debian import deb822
from daklib.dbconn import *
from daklib.gpg import SignedFile
from daklib import utils
################################################################################
[docs]def table_row(changesname, delay, changed_by, closes, fingerprint):
res = '<tr>'
res += (2 * '<td valign="top">%s</td>') % tuple(html.escape(x, quote=False) for x in (changesname, delay))
res += '<td valign="top">%s<br><span class=\"deferredfp\">Fingerprint: %s</span></td>' % (html.escape(changed_by, quote=False), fingerprint)
res += ('<td valign="top">%s</td>' %
''.join(['<a href="https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s">#%s</a><br>' % (close, close) for close in closes]))
res += '</tr>\n'
return res
[docs]def update_graph_database(rrd_dir, *counts):
if not rrd_dir:
return
rrd_file = os.path.join(rrd_dir, 'deferred.rrd')
counts = [str(count) for count in counts]
update = [rrd_file, "N:" + ":".join(counts)]
try:
rrdtool.update(*update)
except rrdtool.error:
create = [rrd_file] + """
--step
300
--start
0
DS:day0:GAUGE:7200:0:1000
DS:day1:GAUGE:7200:0:1000
DS:day2:GAUGE:7200:0:1000
DS:day3:GAUGE:7200:0:1000
DS:day4:GAUGE:7200:0:1000
DS:day5:GAUGE:7200:0:1000
DS:day6:GAUGE:7200:0:1000
DS:day7:GAUGE:7200:0:1000
DS:day8:GAUGE:7200:0:1000
DS:day9:GAUGE:7200:0:1000
DS:day10:GAUGE:7200:0:1000
DS:day11:GAUGE:7200:0:1000
DS:day12:GAUGE:7200:0:1000
DS:day13:GAUGE:7200:0:1000
DS:day14:GAUGE:7200:0:1000
DS:day15:GAUGE:7200:0:1000
RRA:AVERAGE:0.5:1:599
RRA:AVERAGE:0.5:6:700
RRA:AVERAGE:0.5:24:775
RRA:AVERAGE:0.5:288:795
RRA:MIN:0.5:1:600
RRA:MIN:0.5:6:700
RRA:MIN:0.5:24:775
RRA:MIN:0.5:288:795
RRA:MAX:0.5:1:600
RRA:MAX:0.5:6:700
RRA:MAX:0.5:24:775
RRA:MAX:0.5:288:795
""".strip().split("\n")
try:
rc = rrdtool.create(*create)
ru = rrdtool.update(*update)
except rrdtool.error as e:
print(('warning: queue_report: rrdtool error, skipping %s.rrd: %s' % (type, e)))
except NameError:
pass
[docs]def get_upload_data(changesfn):
with open(changesfn) as fd:
achanges = deb822.Changes(fd)
changesname = os.path.basename(changesfn)
delay = os.path.basename(os.path.dirname(changesfn))
m = re.match(r'([0-9]+)-day', delay)
if m:
delaydays = int(m.group(1))
remainingtime = (delaydays > 0) * max(0, 24 * 60 * 60 + os.stat(changesfn).st_mtime - time.time())
delay = "%d days %02d:%02d" % (max(delaydays - 1, 0), int(remainingtime / 3600), int(remainingtime / 60) % 60)
else:
delaydays = 0
remainingtime = 0
uploader = achanges.get('changed-by')
uploader = re.sub(r'^\s*(\S.*)\s+<.*>', r'\1', uploader)
with open(changesfn, "rb") as f:
fingerprint = SignedFile(f.read(), keyrings=get_active_keyring_paths(), require_signature=False).fingerprint
if "Show-Deferred::LinkPath" in Cnf:
isnew = False
suites = get_suites_source_in(achanges['source'])
if 'unstable' not in suites and 'experimental' not in suites:
isnew = True
if not isnew:
# we don't link .changes because we don't want other people to
# upload it with the existing signature.
for afn in [x['name'] for x in achanges['files']]:
lfn = os.path.join(Cnf["Show-Deferred::LinkPath"], afn)
qfn = os.path.join(os.path.dirname(changesfn), afn)
if os.path.islink(lfn):
os.unlink(lfn)
if os.path.exists(qfn):
os.symlink(qfn, lfn)
os.chmod(qfn, 0o644)
return (max(delaydays - 1, 0) * 24 * 60 * 60 + remainingtime, changesname, delay, uploader, achanges.get('closes', '').split(), fingerprint, achanges, delaydays)
[docs]def list_uploads(filelist, rrd_dir):
uploads = sorted(get_upload_data(x) for x in filelist)
# print the summary page
print(header())
if uploads:
print(table_header())
print(''.join(table_row(*x[1:6]) for x in uploads))
print(table_footer())
else:
print('<h1>Currently no deferred uploads to Debian</h1>')
print(footer())
# machine readable summary
if "Show-Deferred::LinkPath" in Cnf:
fn = os.path.join(Cnf["Show-Deferred::LinkPath"], '.status.tmp')
f = open(fn, "w")
try:
counts = [0] * 16
for u in uploads:
counts[u[7]] += 1
print("Changes-file: %s" % u[1], file=f)
fields = """Location: DEFERRED
Delayed-Until: %s
Delay-Remaining: %s
Fingerprint: %s""" % (time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time() + u[0])), u[2], u[5])
print(fields, file=f)
encoded = u[6].dump()
print(encoded.rstrip(), file=f)
open(os.path.join(Cnf["Show-Deferred::LinkPath"], u[1]), "w").write(encoded + fields + '\n')
print(file=f)
f.close()
os.rename(os.path.join(Cnf["Show-Deferred::LinkPath"], '.status.tmp'),
os.path.join(Cnf["Show-Deferred::LinkPath"], 'status'))
update_graph_database(rrd_dir, *counts)
except:
os.unlink(fn)
raise
[docs]def usage(exit_code=0):
if exit_code:
f = sys.stderr
else:
f = sys.stdout
print("""Usage: dak show-deferred
-h, --help show this help and exit.
-p, --link-path [path] override output directory.
-d, --deferred-queue [path] path to the deferred queue
-r, --rrd=key Directory where rrd files to be updated are stored
""", file=f)
sys.exit(exit_code)
[docs]def init():
global Cnf, Options
Cnf = utils.get_conf()
Arguments = [('h', "help", "Show-Deferred::Options::Help"),
("p", "link-path", "Show-Deferred::LinkPath", "HasArg"),
("d", "deferred-queue", "Show-Deferred::DeferredQueue", "HasArg"),
('r', "rrd", "Show-Deferred::Options::Rrd", "HasArg")]
args = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
for i in ["help"]:
key = "Show-Deferred::Options::%s" % i
if key not in Cnf:
Cnf[key] = ""
for i, j in [("DeferredQueue", "--deferred-queue")]:
key = "Show-Deferred::%s" % i
if key not in Cnf:
print("""%s is mandatory.
set via config file or command-line option %s""" % (key, j), file=sys.stderr)
Options = Cnf.subtree("Show-Deferred::Options")
if Options["help"]:
usage()
# Initialise database connection
DBConn()
return args
[docs]def main():
args = init()
if len(args) != 0:
usage(1)
if "Show-Deferred::Options::Rrd" in Cnf:
rrd_dir = Cnf["Show-Deferred::Options::Rrd"]
elif "Dir::Rrd" in Cnf:
rrd_dir = Cnf["Dir::Rrd"]
else:
rrd_dir = None
filelist = []
for r, d, f in os.walk(Cnf["Show-Deferred::DeferredQueue"]):
filelist.extend(os.path.join(r, x) for x in f if x.endswith('.changes'))
list_uploads(filelist, rrd_dir)
available_changes = set(os.path.basename(x) for x in filelist)
if "Show-Deferred::LinkPath" in Cnf:
# remove dead links
for r, d, f in os.walk(Cnf["Show-Deferred::LinkPath"]):
for af in f:
afp = os.path.join(r, af)
if (not os.path.exists(afp)
or (af.endswith('.changes') and af not in available_changes)):
os.unlink(afp)
if __name__ == '__main__':
main()