Source code for dak.show_deferred

#! /usr/bin/env python3

"""Overview of the DEFERRED queue, based on queue-report"""
#    Copyright (C) 2001, 2002, 2003, 2005, 2006  James Troup <james@nocrew.org>
# Copyright (C) 2008 Thomas Viehmann <tv@beamnet.de>

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

################################################################################

import html
import os
import re
import sys
import time

import apt_pkg
import rrdtool

from daklib import utils
from daklib.dbconn import DBConn, get_active_keyring_paths, get_suites_source_in
from daklib.gpg import SignedFile
from debian import deb822

################################################################################








[docs]def table_header(): return """<h1>Deferred uploads</h1> <center><table border="0"> <tr> <th align="center">Change</th> <th align="center">Time remaining</th> <th align="center">Uploader</th> <th align="center">Closes</th> </tr> """
[docs]def table_row(changesname, delay, changed_by, closes, fingerprint): res = "<tr>" res += (2 * '<td valign="top">%s</td>') % tuple( html.escape(x, quote=False) for x in (changesname, delay) ) res += ( '<td valign="top">%s<br><span class="deferredfp">Fingerprint: %s</span></td>' % (html.escape(changed_by, quote=False), fingerprint) ) res += '<td valign="top">%s</td>' % "".join( [ '<a href="https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s">#%s</a><br>' % (close, close) for close in closes ] ) res += "</tr>\n" return res
[docs]def update_graph_database(rrd_dir, *counts): if not rrd_dir: return rrd_file = os.path.join(rrd_dir, "deferred.rrd") counts = [str(count) for count in counts] update = [rrd_file, "N:" + ":".join(counts)] try: rrdtool.update(*update) except rrdtool.error: create = ( [rrd_file] + """ --step 300 --start 0 DS:day0:GAUGE:7200:0:1000 DS:day1:GAUGE:7200:0:1000 DS:day2:GAUGE:7200:0:1000 DS:day3:GAUGE:7200:0:1000 DS:day4:GAUGE:7200:0:1000 DS:day5:GAUGE:7200:0:1000 DS:day6:GAUGE:7200:0:1000 DS:day7:GAUGE:7200:0:1000 DS:day8:GAUGE:7200:0:1000 DS:day9:GAUGE:7200:0:1000 DS:day10:GAUGE:7200:0:1000 DS:day11:GAUGE:7200:0:1000 DS:day12:GAUGE:7200:0:1000 DS:day13:GAUGE:7200:0:1000 DS:day14:GAUGE:7200:0:1000 DS:day15:GAUGE:7200:0:1000 RRA:AVERAGE:0.5:1:599 RRA:AVERAGE:0.5:6:700 RRA:AVERAGE:0.5:24:775 RRA:AVERAGE:0.5:288:795 RRA:MIN:0.5:1:600 RRA:MIN:0.5:6:700 RRA:MIN:0.5:24:775 RRA:MIN:0.5:288:795 RRA:MAX:0.5:1:600 RRA:MAX:0.5:6:700 RRA:MAX:0.5:24:775 RRA:MAX:0.5:288:795 """.strip().split( "\n" ) ) try: rrdtool.create(*create) rrdtool.update(*update) except rrdtool.error as e: print( ( "warning: queue_report: rrdtool error, skipping %s.rrd: %s" % (type, e) ) ) except NameError: pass
[docs]def get_upload_data(changesfn): with open(changesfn) as fd: achanges = deb822.Changes(fd) changesname = os.path.basename(changesfn) delay = os.path.basename(os.path.dirname(changesfn)) m = re.match(r"([0-9]+)-day", delay) if m: delaydays = int(m.group(1)) remainingtime = (delaydays > 0) * max( 0, 24 * 60 * 60 + os.stat(changesfn).st_mtime - time.time() ) delay = "%d days %02d:%02d" % ( max(delaydays - 1, 0), int(remainingtime / 3600), int(remainingtime / 60) % 60, ) else: delaydays = 0 remainingtime = 0 uploader = achanges.get("changed-by") uploader = re.sub(r"^\s*(\S.*)\s+<.*>", r"\1", uploader) with open(changesfn, "rb") as f: fingerprint = SignedFile( f.read(), keyrings=get_active_keyring_paths(), require_signature=False ).fingerprint if "Show-Deferred::LinkPath" in Cnf: isnew = False suites = get_suites_source_in(achanges["source"]) if "unstable" not in suites and "experimental" not in suites: isnew = True if not isnew: # we don't link .changes because we don't want other people to # upload it with the existing signature. for afn in [x["name"] for x in achanges["files"]]: lfn = os.path.join(Cnf["Show-Deferred::LinkPath"], afn) qfn = os.path.join(os.path.dirname(changesfn), afn) if os.path.islink(lfn): os.unlink(lfn) if os.path.exists(qfn): os.symlink(qfn, lfn) os.chmod(qfn, 0o644) return ( max(delaydays - 1, 0) * 24 * 60 * 60 + remainingtime, changesname, delay, uploader, achanges.get("closes", "").split(), fingerprint, achanges, delaydays, )
[docs]def list_uploads(filelist, rrd_dir): uploads = sorted(get_upload_data(x) for x in filelist) # print the summary page print(header()) if uploads: print(table_header()) print("".join(table_row(*x[1:6]) for x in uploads)) print(table_footer()) else: print("<h1>Currently no deferred uploads to Debian</h1>") print(footer()) # machine readable summary if "Show-Deferred::LinkPath" in Cnf: fn = os.path.join(Cnf["Show-Deferred::LinkPath"], ".status.tmp") f = open(fn, "w") try: counts = [0] * 16 for u in uploads: counts[u[7]] += 1 print("Changes-file: %s" % u[1], file=f) fields = """Location: DEFERRED Delayed-Until: %s Delay-Remaining: %s Fingerprint: %s""" % ( time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time() + u[0])), u[2], u[5], ) print(fields, file=f) encoded = u[6].dump() print(encoded.rstrip(), file=f) open(os.path.join(Cnf["Show-Deferred::LinkPath"], u[1]), "w").write( encoded + fields + "\n" ) print(file=f) f.close() os.rename( os.path.join(Cnf["Show-Deferred::LinkPath"], ".status.tmp"), os.path.join(Cnf["Show-Deferred::LinkPath"], "status"), ) update_graph_database(rrd_dir, *counts) except: os.unlink(fn) raise
[docs]def usage(exit_code=0): if exit_code: f = sys.stderr else: f = sys.stdout print( """Usage: dak show-deferred -h, --help show this help and exit. -p, --link-path [path] override output directory. -d, --deferred-queue [path] path to the deferred queue -r, --rrd=key Directory where rrd files to be updated are stored """, file=f, ) sys.exit(exit_code)
[docs]def init(): global Cnf, Options Cnf = utils.get_conf() Arguments = [ ("h", "help", "Show-Deferred::Options::Help"), ("p", "link-path", "Show-Deferred::LinkPath", "HasArg"), ("d", "deferred-queue", "Show-Deferred::DeferredQueue", "HasArg"), ("r", "rrd", "Show-Deferred::Options::Rrd", "HasArg"), ] args = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) for i in ["help"]: key = "Show-Deferred::Options::%s" % i if key not in Cnf: Cnf[key] = "" for i, j in [("DeferredQueue", "--deferred-queue")]: key = "Show-Deferred::%s" % i if key not in Cnf: print( """%s is mandatory. set via config file or command-line option %s""" % (key, j), file=sys.stderr, ) Options = Cnf.subtree("Show-Deferred::Options") if Options["help"]: usage() # Initialise database connection DBConn() return args
[docs]def main(): args = init() if len(args) != 0: usage(1) if "Show-Deferred::Options::Rrd" in Cnf: rrd_dir = Cnf["Show-Deferred::Options::Rrd"] elif "Dir::Rrd" in Cnf: rrd_dir = Cnf["Dir::Rrd"] else: rrd_dir = None filelist = [] for r, d, f in os.walk(Cnf["Show-Deferred::DeferredQueue"]): filelist.extend(os.path.join(r, x) for x in f if x.endswith(".changes")) list_uploads(filelist, rrd_dir) available_changes = set(os.path.basename(x) for x in filelist) if "Show-Deferred::LinkPath" in Cnf: # remove dead links for r, d, f in os.walk(Cnf["Show-Deferred::LinkPath"]): for af in f: afp = os.path.join(r, af) if not os.path.exists(afp) or ( af.endswith(".changes") and af not in available_changes ): os.unlink(afp)
if __name__ == "__main__": main()