1#! /usr/bin/env python3
3""" Overview of the DEFERRED queue, based on queue-report """
4# Copyright (C) 2001, 2002, 2003, 2005, 2006 James Troup <james@nocrew.org>
5# Copyright (C) 2008 Thomas Viehmann <tv@beamnet.de>
7# This program is free software; you can redistribute it and/or modify
8# it under the terms of the GNU General Public License as published by
9# the Free Software Foundation; either version 2 of the License, or
10# (at your option) any later version.
12# This program is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU General Public License for more details.
17# You should have received a copy of the GNU General Public License
18# along with this program; if not, write to the Free Software
19# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21################################################################################
23import html
24import sys
25import os
26import re
27import time
28import apt_pkg
29import rrdtool
31from debian import deb822
33from daklib.dbconn import *
34from daklib.gpg import SignedFile
35from daklib import utils
37################################################################################
40def header():
41 return """<!DOCTYPE html>
42 <html lang="en"><head><meta charset="utf-8">
43 <title>Deferred uploads to Debian</title>
44 <link rel="stylesheet" href="style.css">
45 <link rel="shortcut icon" href="https://www.debian.org/favicon.ico">
46 </head>
47 <body>
48 <div align="center">
49 <a href="https://www.debian.org/">
50 <img src="https://www.debian.org/logos/openlogo-nd-50.png" border="0" hspace="0" vspace="0" alt=""></a>
51 <a href="https://www.debian.org/">
52 <img src="https://www.debian.org/Pics/debian.png" border="0" hspace="0" vspace="0" alt="Debian Project"></a>
53 </div>
54 <br>
55 <table class="reddy" width="100%">
56 <tr>
57 <td class="reddy">
58 <img src="https://www.debian.org/Pics/red-upperleft.png" align="left" border="0" hspace="0" vspace="0"
59 alt="" width="15" height="16"></td>
60 <td rowspan="2" class="reddy">Deferred uploads to Debian</td>
61 <td class="reddy">
62 <img src="https://www.debian.org/Pics/red-upperright.png" align="right" border="0" hspace="0" vspace="0"
63 alt="" width="16" height="16"></td>
64 </tr>
65 <tr>
66 <td class="reddy">
67 <img src="https://www.debian.org/Pics/red-lowerleft.png" align="left" border="0" hspace="0" vspace="0"
68 alt="" width="16" height="16"></td>
69 <td class="reddy">
70 <img src="https://www.debian.org/Pics/red-lowerright.png" align="right" border="0" hspace="0" vspace="0"
71 alt="" width="15" height="16"></td>
72 </tr>
73 </table>
74 """
77def footer():
78 res = "<p class=\"validate\">Timestamp: %s (UTC)</p>" % (time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime()))
79 res += "<p class=\"timestamp\">There are <a href=\"/stat.html\">graphs about the queues</a> available.</p>"
80 res += "</body></html>"
81 return res
84def table_header():
85 return """<h1>Deferred uploads</h1>
86 <center><table border="0">
87 <tr>
88 <th align="center">Change</th>
89 <th align="center">Time remaining</th>
90 <th align="center">Uploader</th>
91 <th align="center">Closes</th>
92 </tr>
93 """
96def table_footer():
97 return '</table><br><p>non-NEW uploads are <a href="/deferred/">available</a> (<a href="/deferred/status">machine readable version</a>), see the <a href="ftp://ftp.upload.debian.org/pub/UploadQueue/README">UploadQueue-README</a> and <a href="https://www.debian.org/doc/manuals/developers-reference/ch05.en.html#delayed-incoming">Developer\'s reference</a> for more information on the DELAYED queue.</p></center><br>\n'
100def table_row(changesname, delay, changed_by, closes, fingerprint):
101 res = '<tr>'
102 res += (2 * '<td valign="top">%s</td>') % tuple(html.escape(x, quote=False) for x in (changesname, delay))
103 res += '<td valign="top">%s<br><span class=\"deferredfp\">Fingerprint: %s</span></td>' % (html.escape(changed_by, quote=False), fingerprint)
104 res += ('<td valign="top">%s</td>' %
105 ''.join(['<a href="https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s">#%s</a><br>' % (close, close) for close in closes]))
106 res += '</tr>\n'
107 return res
110def update_graph_database(rrd_dir, *counts):
111 if not rrd_dir:
112 return
114 rrd_file = os.path.join(rrd_dir, 'deferred.rrd')
115 counts = [str(count) for count in counts]
116 update = [rrd_file, "N:" + ":".join(counts)]
118 try:
119 rrdtool.update(*update)
120 except rrdtool.error:
121 create = [rrd_file] + """
122--step
123300
124--start
1250
126DS:day0:GAUGE:7200:0:1000
127DS:day1:GAUGE:7200:0:1000
128DS:day2:GAUGE:7200:0:1000
129DS:day3:GAUGE:7200:0:1000
130DS:day4:GAUGE:7200:0:1000
131DS:day5:GAUGE:7200:0:1000
132DS:day6:GAUGE:7200:0:1000
133DS:day7:GAUGE:7200:0:1000
134DS:day8:GAUGE:7200:0:1000
135DS:day9:GAUGE:7200:0:1000
136DS:day10:GAUGE:7200:0:1000
137DS:day11:GAUGE:7200:0:1000
138DS:day12:GAUGE:7200:0:1000
139DS:day13:GAUGE:7200:0:1000
140DS:day14:GAUGE:7200:0:1000
141DS:day15:GAUGE:7200:0:1000
142RRA:AVERAGE:0.5:1:599
143RRA:AVERAGE:0.5:6:700
144RRA:AVERAGE:0.5:24:775
145RRA:AVERAGE:0.5:288:795
146RRA:MIN:0.5:1:600
147RRA:MIN:0.5:6:700
148RRA:MIN:0.5:24:775
149RRA:MIN:0.5:288:795
150RRA:MAX:0.5:1:600
151RRA:MAX:0.5:6:700
152RRA:MAX:0.5:24:775
153RRA:MAX:0.5:288:795
154""".strip().split("\n")
155 try:
156 rc = rrdtool.create(*create)
157 ru = rrdtool.update(*update)
158 except rrdtool.error as e:
159 print(('warning: queue_report: rrdtool error, skipping %s.rrd: %s' % (type, e)))
160 except NameError:
161 pass
164def get_upload_data(changesfn):
165 with open(changesfn) as fd:
166 achanges = deb822.Changes(fd)
167 changesname = os.path.basename(changesfn)
168 delay = os.path.basename(os.path.dirname(changesfn))
169 m = re.match(r'([0-9]+)-day', delay)
170 if m:
171 delaydays = int(m.group(1))
172 remainingtime = (delaydays > 0) * max(0, 24 * 60 * 60 + os.stat(changesfn).st_mtime - time.time())
173 delay = "%d days %02d:%02d" % (max(delaydays - 1, 0), int(remainingtime / 3600), int(remainingtime / 60) % 60)
174 else:
175 delaydays = 0
176 remainingtime = 0
178 uploader = achanges.get('changed-by')
179 uploader = re.sub(r'^\s*(\S.*)\s+<.*>', r'\1', uploader)
180 with open(changesfn, "rb") as f:
181 fingerprint = SignedFile(f.read(), keyrings=get_active_keyring_paths(), require_signature=False).fingerprint
182 if "Show-Deferred::LinkPath" in Cnf:
183 isnew = False
184 suites = get_suites_source_in(achanges['source'])
185 if 'unstable' not in suites and 'experimental' not in suites:
186 isnew = True
188 if not isnew:
189 # we don't link .changes because we don't want other people to
190 # upload it with the existing signature.
191 for afn in [x['name'] for x in achanges['files']]:
192 lfn = os.path.join(Cnf["Show-Deferred::LinkPath"], afn)
193 qfn = os.path.join(os.path.dirname(changesfn), afn)
194 if os.path.islink(lfn):
195 os.unlink(lfn)
196 if os.path.exists(qfn):
197 os.symlink(qfn, lfn)
198 os.chmod(qfn, 0o644)
199 return (max(delaydays - 1, 0) * 24 * 60 * 60 + remainingtime, changesname, delay, uploader, achanges.get('closes', '').split(), fingerprint, achanges, delaydays)
202def list_uploads(filelist, rrd_dir):
203 uploads = sorted(get_upload_data(x) for x in filelist)
204 # print the summary page
205 print(header())
206 if uploads:
207 print(table_header())
208 print(''.join(table_row(*x[1:6]) for x in uploads))
209 print(table_footer())
210 else:
211 print('<h1>Currently no deferred uploads to Debian</h1>')
212 print(footer())
213 # machine readable summary
214 if "Show-Deferred::LinkPath" in Cnf:
215 fn = os.path.join(Cnf["Show-Deferred::LinkPath"], '.status.tmp')
216 f = open(fn, "w")
217 try:
218 counts = [0] * 16
219 for u in uploads:
220 counts[u[7]] += 1
221 print("Changes-file: %s" % u[1], file=f)
222 fields = """Location: DEFERRED
223Delayed-Until: %s
224Delay-Remaining: %s
225Fingerprint: %s""" % (time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time() + u[0])), u[2], u[5])
226 print(fields, file=f)
227 encoded = u[6].dump()
228 print(encoded.rstrip(), file=f)
229 open(os.path.join(Cnf["Show-Deferred::LinkPath"], u[1]), "w").write(encoded + fields + '\n')
230 print(file=f)
231 f.close()
232 os.rename(os.path.join(Cnf["Show-Deferred::LinkPath"], '.status.tmp'),
233 os.path.join(Cnf["Show-Deferred::LinkPath"], 'status'))
234 update_graph_database(rrd_dir, *counts)
235 except:
236 os.unlink(fn)
237 raise
240def usage(exit_code=0):
241 if exit_code: 241 ↛ 242line 241 didn't jump to line 242, because the condition on line 241 was never true
242 f = sys.stderr
243 else:
244 f = sys.stdout
245 print("""Usage: dak show-deferred
246 -h, --help show this help and exit.
247 -p, --link-path [path] override output directory.
248 -d, --deferred-queue [path] path to the deferred queue
249 -r, --rrd=key Directory where rrd files to be updated are stored
250 """, file=f)
251 sys.exit(exit_code)
254def init():
255 global Cnf, Options
256 Cnf = utils.get_conf()
257 Arguments = [('h', "help", "Show-Deferred::Options::Help"),
258 ("p", "link-path", "Show-Deferred::LinkPath", "HasArg"),
259 ("d", "deferred-queue", "Show-Deferred::DeferredQueue", "HasArg"),
260 ('r', "rrd", "Show-Deferred::Options::Rrd", "HasArg")]
261 args = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
262 for i in ["help"]:
263 key = "Show-Deferred::Options::%s" % i
264 if key not in Cnf: 264 ↛ 265line 264 didn't jump to line 265, because the condition on line 264 was never true
265 Cnf[key] = ""
266 for i, j in [("DeferredQueue", "--deferred-queue")]:
267 key = "Show-Deferred::%s" % i
268 if key not in Cnf: 268 ↛ 266line 268 didn't jump to line 266, because the condition on line 268 was never false
269 print("""%s is mandatory.
270 set via config file or command-line option %s""" % (key, j), file=sys.stderr)
272 Options = Cnf.subtree("Show-Deferred::Options")
273 if Options["help"]: 273 ↛ 277line 273 didn't jump to line 277, because the condition on line 273 was never false
274 usage()
276 # Initialise database connection
277 DBConn()
279 return args
282def main():
283 args = init()
284 if len(args) != 0:
285 usage(1)
287 if "Show-Deferred::Options::Rrd" in Cnf:
288 rrd_dir = Cnf["Show-Deferred::Options::Rrd"]
289 elif "Dir::Rrd" in Cnf:
290 rrd_dir = Cnf["Dir::Rrd"]
291 else:
292 rrd_dir = None
294 filelist = []
295 for r, d, f in os.walk(Cnf["Show-Deferred::DeferredQueue"]):
296 filelist.extend(os.path.join(r, x) for x in f if x.endswith('.changes'))
297 list_uploads(filelist, rrd_dir)
299 available_changes = set(os.path.basename(x) for x in filelist)
300 if "Show-Deferred::LinkPath" in Cnf:
301 # remove dead links
302 for r, d, f in os.walk(Cnf["Show-Deferred::LinkPath"]):
303 for af in f:
304 afp = os.path.join(r, af)
305 if (not os.path.exists(afp)
306 or (af.endswith('.changes') and af not in available_changes)):
307 os.unlink(afp)
310if __name__ == '__main__':
311 main()