#! /usr/bin/env python3
""" Produces a report on NEW and BYHAND packages """
# Copyright (C) 2001, 2002, 2003, 2005, 2006 James Troup <james@nocrew.org>
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
################################################################################
# <o-o> XP runs GCC, XFREE86, SSH etc etc,.,, I feel almost like linux....
# <o-o> I am very confident that I can replicate any Linux application on XP
# <willy> o-o: *boggle*
# <o-o> building from source.
# <o-o> Viiru: I already run GIMP under XP
# <willy> o-o: why do you capitalise the names of all pieces of software?
# <o-o> willy: because I want the EMPHASIZE them....
# <o-o> grr s/the/to/
# <willy> o-o: it makes you look like ZIPPY the PINHEAD
# <o-o> willy: no idea what you are talking about.
# <willy> o-o: do some research
# <o-o> willy: for what reason?
################################################################################
import html
import os
import sys
import time
import apt_pkg
import datetime
import functools
from daklib import utils
from daklib.utils import get_logins_from_ldap
from daklib.dbconn import DBConn, has_new_comment, PolicyQueue, get_uid_from_fingerprint
from daklib.policy import PolicyQueueUploadHandler
from daklib.textutils import fix_maintainer
from daklib.dak_exceptions import ParseMaintError
Cnf = None
direction = []
################################################################################
[docs]def usage(exit_code=0):
print("""Usage: dak queue-report
Prints a report of packages in queues (usually new and byhand).
-h, --help show this help and exit.
-8, --822 writes 822 formated output to the location set in dak.conf
-n, --new produce html-output
-s, --sort=key sort output according to key, see below.
-a, --age=key if using sort by age, how should time be treated?
If not given a default of hours will be used.
-r, --rrd=key Directory where rrd files to be updated are stored
-d, --directories=key A comma separated list of queues to be scanned
Sorting Keys: ao=age, oldest first. an=age, newest first.
na=name, ascending nd=name, descending
nf=notes, first nl=notes, last
Age Keys: m=minutes, h=hours, d=days, w=weeks, o=months, y=years
""")
sys.exit(exit_code)
################################################################################
[docs]def plural(x):
if x > 1:
return "s"
else:
return ""
################################################################################
[docs]def time_pp(x):
if x < 60:
unit = "second"
elif x < 3600:
x /= 60
unit = "minute"
elif x < 86400:
x /= 3600
unit = "hour"
elif x < 604800:
x /= 86400
unit = "day"
elif x < 2419200:
x /= 604800
unit = "week"
elif x < 29030400:
x /= 2419200
unit = "month"
else:
x /= 29030400
unit = "year"
x = int(x)
return "%s %s%s" % (x, unit, plural(x))
################################################################################
[docs]def sg_compare(a, b):
a = a[1]
b = b[1]
# Sort by have pending action, have note, time of oldest upload.
# Sort by have pending action
a_note_state = a["processed"]
b_note_state = b["processed"]
if a_note_state < b_note_state:
return -1
elif a_note_state > b_note_state:
return 1
# Sort by have note
a_note_state = a["note_state"]
b_note_state = b["note_state"]
if a_note_state < b_note_state:
return -1
elif a_note_state > b_note_state:
return 1
# Sort by time of oldest upload
return a["oldest"] - b["oldest"]
############################################################
[docs]def sortfunc(a, b):
for sorting in direction:
(sortkey, way, time) = sorting
ret = 0
if time == "m":
x = int(a[sortkey] / 60)
y = int(b[sortkey] / 60)
elif time == "h":
x = int(a[sortkey] / 3600)
y = int(b[sortkey] / 3600)
elif time == "d":
x = int(a[sortkey] / 86400)
y = int(b[sortkey] / 86400)
elif time == "w":
x = int(a[sortkey] / 604800)
y = int(b[sortkey] / 604800)
elif time == "o":
x = int(a[sortkey] / 2419200)
y = int(b[sortkey] / 2419200)
elif time == "y":
x = int(a[sortkey] / 29030400)
y = int(b[sortkey] / 29030400)
else:
x = a[sortkey]
y = b[sortkey]
if x < y:
ret = -1
elif x > y:
ret = 1
if ret != 0:
if way < 0:
ret = ret * -1
return ret
return 0
############################################################
[docs]def table_row(source, version, arch, last_mod, maint, distribution, closes, fingerprint, sponsor, changedby):
trclass = "sid"
session = DBConn().session()
for dist in distribution:
if dist == "experimental":
trclass = "exp"
query = '''SELECT source
FROM source_suite
WHERE source = :source
AND suite_name IN ('unstable', 'experimental')'''
if not session.execute(query, {'source': source}).rowcount:
trclass += " sourceNEW"
session.commit()
print("<tr class=\"%s\">" % (trclass))
if "sourceNEW" in trclass:
print("<td class=\"package\">%s</td>" % (source))
else:
print("<td class=\"package\"><a href=\"https://tracker.debian.org/pkg/%(source)s\">%(source)s</a></td>" % {'source': source})
print("<td class=\"version\">")
for vers in version.split():
print("<a href=\"new/%s_%s.html\">%s</a><br>" % (source, html.escape(vers), html.escape(vers, quote=False)))
print("</td>")
print("<td class=\"arch\">%s</td>" % (arch))
print("<td class=\"distribution\">")
for dist in distribution:
print("%s<br>" % (dist))
print("</td>")
print("<td class=\"age\"><abbr title=\"%s\">%s</abbr></td>" % (
datetime.datetime.utcfromtimestamp(int(time.time()) - last_mod).strftime('%a, %d %b %Y %T UTC'),
time_pp(last_mod),
))
(name, mail) = maint.split(":", 1)
print("<td class=\"upload-data\">")
print("<span class=\"maintainer\">Maintainer: <a href=\"https://qa.debian.org/developer.php?login=%s\">%s</a></span><br>" % (html.escape(mail), html.escape(name, quote=False)))
(name, mail) = changedby.split(":", 1)
print("<span class=\"changed-by\">Changed-By: <a href=\"https://qa.debian.org/developer.php?login=%s\">%s</a></span><br>" % (html.escape(mail), html.escape(name, quote=False)))
if sponsor:
print("<span class=\"sponsor\">Sponsor: <a href=\"https://qa.debian.org/developer.php?login=%s\">%s</a>@debian.org</span><br>" % (html.escape(sponsor), html.escape(sponsor, quote=False)))
print("<span class=\"signature\">Fingerprint: %s</span>" % (fingerprint))
print("</td>")
print("<td class=\"closes\">")
for close in closes:
print("<a href=\"https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s\">#%s</a><br>" % (html.escape(close), html.escape(close, quote=False)))
print("</td></tr>")
############################################################
[docs]def update_graph_database(rrd_dir, type, n_source, n_binary):
if not rrd_dir:
return
import rrdtool
rrd_file = os.path.join(rrd_dir, type.lower() + '.rrd')
update = [rrd_file, "N:%s:%s" % (n_source, n_binary)]
try:
rrdtool.update(*update)
except rrdtool.error:
create = [rrd_file] + """
--step
300
--start
0
DS:ds0:GAUGE:7200:0:1000
DS:ds1:GAUGE:7200:0:1000
RRA:AVERAGE:0.5:1:599
RRA:AVERAGE:0.5:6:700
RRA:AVERAGE:0.5:24:775
RRA:AVERAGE:0.5:288:795
RRA:MAX:0.5:1:600
RRA:MAX:0.5:6:700
RRA:MAX:0.5:24:775
RRA:MAX:0.5:288:795
""".strip().split("\n")
try:
rrdtool.create(*create)
rrdtool.update(*update)
except rrdtool.error as e:
print(('warning: queue_report: rrdtool error, skipping %s.rrd: %s' % (type, e)))
except NameError:
pass
############################################################
[docs]def process_queue(queue, log, rrd_dir):
msg = ""
type = queue.queue_name
session = DBConn().session()
# Divide the .changes into per-source groups
per_source = {}
total_pending = 0
for upload in queue.uploads:
source = upload.changes.source
if source not in per_source:
per_source[source] = {}
per_source[source]["list"] = []
per_source[source]["processed"] = ""
handler = PolicyQueueUploadHandler(upload, session)
if handler.get_action():
per_source[source]["processed"] = "PENDING %s" % handler.get_action()
total_pending += 1
per_source[source]["list"].append(upload)
per_source[source]["list"].sort(key=lambda x: x.changes.created, reverse=True)
# Determine oldest time and have note status for each source group
for source in list(per_source.keys()):
source_list = per_source[source]["list"]
first = source_list[0]
oldest = time.mktime(first.changes.created.timetuple())
have_note = 0
for d in per_source[source]["list"]:
mtime = time.mktime(d.changes.created.timetuple())
if "Queue-Report::Options::New" in Cnf:
if mtime > oldest:
oldest = mtime
else:
if mtime < oldest:
oldest = mtime
have_note += has_new_comment(d.policy_queue, d.changes.source, d.changes.version)
per_source[source]["oldest"] = oldest
if not have_note:
per_source[source]["note_state"] = 0 # none
elif have_note < len(source_list):
per_source[source]["note_state"] = 1 # some
else:
per_source[source]["note_state"] = 2 # all
per_source_items = list(per_source.items())
per_source_items.sort(key=functools.cmp_to_key(sg_compare))
update_graph_database(rrd_dir, type, len(per_source_items), len(queue.uploads))
entries = []
max_source_len = 0
max_version_len = 0
max_arch_len = 0
try:
logins = get_logins_from_ldap()
except:
logins = dict()
for i in per_source_items:
maintainer = {}
maint = ""
distribution = ""
closes = ""
fingerprint = ""
changeby = {}
changedby = ""
sponsor = ""
filename = i[1]["list"][0].changes.changesname
last_modified = time.time() - i[1]["oldest"]
source = i[1]["list"][0].changes.source
if len(source) > max_source_len:
max_source_len = len(source)
binary_list = i[1]["list"][0].binaries
binary = ', '.join([b.package for b in binary_list])
arches = set()
versions = set()
for j in i[1]["list"]:
dbc = j.changes
changesbase = dbc.changesname
if "Queue-Report::Options::New" in Cnf or "Queue-Report::Options::822" in Cnf:
try:
(maintainer["maintainer822"], maintainer["maintainer2047"],
maintainer["maintainername"], maintainer["maintaineremail"]) = \
fix_maintainer(dbc.maintainer)
except ParseMaintError as msg:
print("Problems while parsing maintainer address\n")
maintainer["maintainername"] = "Unknown"
maintainer["maintaineremail"] = "Unknown"
maint = "%s:%s" % (maintainer["maintainername"], maintainer["maintaineremail"])
# ...likewise for the Changed-By: field if it exists.
try:
(changeby["changedby822"], changeby["changedby2047"],
changeby["changedbyname"], changeby["changedbyemail"]) = \
fix_maintainer(dbc.changedby)
except ParseMaintError as msg:
(changeby["changedby822"], changeby["changedby2047"],
changeby["changedbyname"], changeby["changedbyemail"]) = \
("", "", "", "")
changedby = "%s:%s" % (changeby["changedbyname"], changeby["changedbyemail"])
distribution = dbc.distribution.split()
closes = dbc.closes
fingerprint = dbc.fingerprint
sponsor_uid = get_uid_from_fingerprint(fingerprint, session)
sponsor_name = sponsor_uid.name
sponsor_login = sponsor_uid.uid
if '@' in sponsor_login:
if fingerprint in logins:
sponsor_login = logins[fingerprint]
if (sponsor_name != maintainer["maintainername"]
and sponsor_name != changeby["changedbyname"]
and sponsor_login + '@debian.org' != maintainer["maintaineremail"]
and sponsor_name != changeby["changedbyemail"]):
sponsor = sponsor_login
for arch in dbc.architecture.split():
arches.add(arch)
versions.add(dbc.version)
arches_list = sorted(arches, key=utils.ArchKey)
arch_list = " ".join(arches_list)
version_list = " ".join(sorted(versions, reverse=True))
if len(version_list) > max_version_len:
max_version_len = len(version_list)
if len(arch_list) > max_arch_len:
max_arch_len = len(arch_list)
if i[1]["note_state"]:
note = " | [N]"
else:
note = ""
entries.append([source, binary, version_list, arch_list, per_source[source]["processed"], note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, filename])
# direction entry consists of "Which field, which direction, time-consider" where
# time-consider says how we should treat last_modified. Thats all.
# Look for the options for sort and then do the sort.
age = "h"
if "Queue-Report::Options::Age" in Cnf:
age = Cnf["Queue-Report::Options::Age"]
if "Queue-Report::Options::New" in Cnf:
# If we produce html we always have oldest first.
direction.append([6, -1, "ao"])
else:
if "Queue-Report::Options::Sort" in Cnf:
for i in Cnf["Queue-Report::Options::Sort"].split(","):
if i == "ao":
# Age, oldest first.
direction.append([6, -1, age])
elif i == "an":
# Age, newest first.
direction.append([6, 1, age])
elif i == "na":
# Name, Ascending.
direction.append([0, 1, 0])
elif i == "nd":
# Name, Descending.
direction.append([0, -1, 0])
elif i == "nl":
# Notes last.
direction.append([5, 1, 0])
elif i == "nf":
# Notes first.
direction.append([5, -1, 0])
entries.sort(key=functools.cmp_to_key(sortfunc))
# Yes, in theory you can add several sort options at the commandline with. But my mind is to small
# at the moment to come up with a real good sorting function that considers all the sidesteps you
# have with it. (If you combine options it will simply take the last one at the moment).
# Will be enhanced in the future.
if "Queue-Report::Options::822" in Cnf:
# print stuff out in 822 format
for entry in entries:
(source, binary, version_list, arch_list, processed, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, changes_file) = entry
# We'll always have Source, Version, Arch, Mantainer, and Dist
# For the rest, check to see if we have them, then print them out
log.write("Source: " + source + "\n")
log.write("Binary: " + binary + "\n")
log.write("Version: " + version_list + "\n")
log.write("Architectures: ")
log.write((", ".join(arch_list.split(" "))) + "\n")
log.write("Age: " + time_pp(last_modified) + "\n")
log.write("Last-Modified: " + str(int(time.time()) - int(last_modified)) + "\n")
log.write("Queue: " + type + "\n")
(name, mail) = maint.split(":", 1)
log.write("Maintainer: " + name + " <" + mail + ">" + "\n")
if changedby:
(name, mail) = changedby.split(":", 1)
log.write("Changed-By: " + name + " <" + mail + ">" + "\n")
if sponsor:
log.write("Sponsored-By: %s@debian.org\n" % sponsor)
log.write("Distribution:")
for dist in distribution:
log.write(" " + dist)
log.write("\n")
log.write("Fingerprint: " + fingerprint + "\n")
if closes:
bug_string = ""
for bugs in closes:
bug_string += "#" + bugs + ", "
log.write("Closes: " + bug_string[:-2] + "\n")
log.write("Changes-File: " + os.path.basename(changes_file) + "\n")
log.write("\n")
total_count = len(queue.uploads)
source_count = len(per_source_items)
if "Queue-Report::Options::New" in Cnf:
direction.append([6, 1, "ao"])
entries.sort(key=functools.cmp_to_key(sortfunc))
# Output for a html file. First table header. then table_footer.
# Any line between them is then a <tr> printed from subroutine table_row.
if len(entries) > 0:
table_header(type.upper(), source_count, total_count)
for entry in entries:
(source, binary, version_list, arch_list, processed, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, _) = entry
table_row(source, version_list, arch_list, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby)
table_footer(type.upper())
elif "Queue-Report::Options::822" not in Cnf:
# The "normal" output without any formatting.
msg = ""
for entry in entries:
(source, binary, version_list, arch_list, processed, note, last_modified, _, _, _, _, _, _, _) = entry
if processed:
format = "%%-%ds | %%-%ds | %%-%ds | %%s\n" % (max_source_len, max_version_len, max_arch_len)
msg += format % (source, version_list, arch_list, processed)
else:
format = "%%-%ds | %%-%ds | %%-%ds%%s | %%s old\n" % (max_source_len, max_version_len, max_arch_len)
msg += format % (source, version_list, arch_list, note, time_pp(last_modified))
if msg:
print(type.upper())
print("-" * len(type))
print()
print(msg)
print(("%s %s source package%s / %s %s package%s in total / %s %s package%s to be processed." %
(source_count, type, plural(source_count),
total_count, type, plural(total_count),
total_pending, type, plural(total_pending))))
print()
################################################################################
[docs]def main():
global Cnf
Cnf = utils.get_conf()
Arguments = [('h', "help", "Queue-Report::Options::Help"),
('n', "new", "Queue-Report::Options::New"),
('8', '822', "Queue-Report::Options::822"),
('s', "sort", "Queue-Report::Options::Sort", "HasArg"),
('a', "age", "Queue-Report::Options::Age", "HasArg"),
('r', "rrd", "Queue-Report::Options::Rrd", "HasArg"),
('d', "directories", "Queue-Report::Options::Directories", "HasArg")]
for i in ["help"]:
key = "Queue-Report::Options::%s" % i
if key not in Cnf:
Cnf[key] = ""
apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
Options = Cnf.subtree("Queue-Report::Options")
if Options["Help"]:
usage()
if "Queue-Report::Options::New" in Cnf:
header()
queue_names = []
if "Queue-Report::Options::Directories" in Cnf:
for i in Cnf["Queue-Report::Options::Directories"].split(","):
queue_names.append(i)
elif "Queue-Report::Directories" in Cnf:
queue_names = Cnf.value_list("Queue-Report::Directories")
else:
queue_names = ["byhand", "new"]
if "Queue-Report::Options::Rrd" in Cnf:
rrd_dir = Cnf["Queue-Report::Options::Rrd"]
elif "Dir::Rrd" in Cnf:
rrd_dir = Cnf["Dir::Rrd"]
else:
rrd_dir = None
f = None
if "Queue-Report::Options::822" in Cnf:
# Open the report file
f = sys.stdout
filename822 = Cnf.get("Queue-Report::ReportLocations::822Location")
if filename822:
f = open(filename822, "w")
session = DBConn().session()
for queue_name in queue_names:
queue = session.query(PolicyQueue).filter_by(queue_name=queue_name).first()
if queue is not None:
process_queue(queue, f, rrd_dir)
else:
utils.warn("Cannot find queue %s" % queue_name)
if "Queue-Report::Options::822" in Cnf:
f.close()
if "Queue-Report::Options::New" in Cnf:
footer()
################################################################################
if __name__ == '__main__':
main()