Coverage for dak/show_deferred.py: 23%
151 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-01-04 16:18 +0000
1#! /usr/bin/env python3
3"""Overview of the DEFERRED queue, based on queue-report"""
4# Copyright (C) 2001, 2002, 2003, 2005, 2006 James Troup <james@nocrew.org>
5# Copyright (C) 2008 Thomas Viehmann <tv@beamnet.de>
7# This program is free software; you can redistribute it and/or modify
8# it under the terms of the GNU General Public License as published by
9# the Free Software Foundation; either version 2 of the License, or
10# (at your option) any later version.
12# This program is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU General Public License for more details.
17# You should have received a copy of the GNU General Public License
18# along with this program; if not, write to the Free Software
19# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21################################################################################
23import html
24import os
25import re
26import sys
27import time
29import apt_pkg
30import rrdtool
32from daklib import utils
33from daklib.dbconn import DBConn, get_active_keyring_paths, get_suites_source_in
34from daklib.gpg import SignedFile
35from debian import deb822
37Cnf: apt_pkg.Configuration
38Options: apt_pkg.Configuration
40################################################################################
43def header():
44 return """<!DOCTYPE html>
45 <html lang="en"><head><meta charset="utf-8">
46 <title>Deferred uploads to Debian</title>
47 <link rel="stylesheet" href="style.css">
48 <link rel="shortcut icon" href="https://www.debian.org/favicon.ico">
49 </head>
50 <body>
51 <div align="center">
52 <a href="https://www.debian.org/">
53 <img src="https://www.debian.org/logos/openlogo-nd-50.png" border="0" hspace="0" vspace="0" alt=""></a>
54 <a href="https://www.debian.org/">
55 <img src="https://www.debian.org/Pics/debian.png" border="0" hspace="0" vspace="0" alt="Debian Project"></a>
56 </div>
57 <br>
58 <table class="reddy" width="100%">
59 <tr>
60 <td class="reddy">
61 <img src="https://www.debian.org/Pics/red-upperleft.png" align="left" border="0" hspace="0" vspace="0"
62 alt="" width="15" height="16"></td>
63 <td rowspan="2" class="reddy">Deferred uploads to Debian</td>
64 <td class="reddy">
65 <img src="https://www.debian.org/Pics/red-upperright.png" align="right" border="0" hspace="0" vspace="0"
66 alt="" width="16" height="16"></td>
67 </tr>
68 <tr>
69 <td class="reddy">
70 <img src="https://www.debian.org/Pics/red-lowerleft.png" align="left" border="0" hspace="0" vspace="0"
71 alt="" width="16" height="16"></td>
72 <td class="reddy">
73 <img src="https://www.debian.org/Pics/red-lowerright.png" align="right" border="0" hspace="0" vspace="0"
74 alt="" width="15" height="16"></td>
75 </tr>
76 </table>
77 """
80def footer():
81 res = '<p class="validate">Timestamp: %s (UTC)</p>' % (
82 time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime())
83 )
84 res += '<p class="timestamp">There are <a href="/stat.html">graphs about the queues</a> available.</p>'
85 res += "</body></html>"
86 return res
89def table_header():
90 return """<h1>Deferred uploads</h1>
91 <center><table border="0">
92 <tr>
93 <th align="center">Change</th>
94 <th align="center">Time remaining</th>
95 <th align="center">Uploader</th>
96 <th align="center">Closes</th>
97 </tr>
98 """
101def table_footer():
102 return '</table><br><p>non-NEW uploads are <a href="/deferred/">available</a> (<a href="/deferred/status">machine readable version</a>), see the <a href="https://www.debian.org/doc/manuals/developers-reference/ch05.en.html#delayed-incoming">Developer\'s reference</a> for more information on the DELAYED queue.</p></center><br>\n'
105def table_row(changesname, delay, changed_by, closes, fingerprint):
106 res = "<tr>"
107 res += (2 * '<td valign="top">%s</td>') % tuple(
108 html.escape(x, quote=False) for x in (changesname, delay)
109 )
110 res += (
111 '<td valign="top">%s<br><span class="deferredfp">Fingerprint: %s</span></td>'
112 % (html.escape(changed_by or "Unknown", quote=False), fingerprint)
113 )
114 res += '<td valign="top">%s</td>' % "".join(
115 [
116 '<a href="https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s">#%s</a><br>'
117 % (close, close)
118 for close in closes
119 ]
120 )
121 res += "</tr>\n"
122 return res
125def update_graph_database(rrd_dir, *counts):
126 if not rrd_dir:
127 return
129 rrd_file = os.path.join(rrd_dir, "deferred.rrd")
130 update = [rrd_file, "N:" + ":".join(str(count) for count in counts)]
132 try:
133 rrdtool.update(*update)
134 except rrdtool.error:
135 create = (
136 [rrd_file]
137 + """
138--step
139300
140--start
1410
142DS:day0:GAUGE:7200:0:1000
143DS:day1:GAUGE:7200:0:1000
144DS:day2:GAUGE:7200:0:1000
145DS:day3:GAUGE:7200:0:1000
146DS:day4:GAUGE:7200:0:1000
147DS:day5:GAUGE:7200:0:1000
148DS:day6:GAUGE:7200:0:1000
149DS:day7:GAUGE:7200:0:1000
150DS:day8:GAUGE:7200:0:1000
151DS:day9:GAUGE:7200:0:1000
152DS:day10:GAUGE:7200:0:1000
153DS:day11:GAUGE:7200:0:1000
154DS:day12:GAUGE:7200:0:1000
155DS:day13:GAUGE:7200:0:1000
156DS:day14:GAUGE:7200:0:1000
157DS:day15:GAUGE:7200:0:1000
158RRA:AVERAGE:0.5:1:599
159RRA:AVERAGE:0.5:6:700
160RRA:AVERAGE:0.5:24:775
161RRA:AVERAGE:0.5:288:795
162RRA:MIN:0.5:1:600
163RRA:MIN:0.5:6:700
164RRA:MIN:0.5:24:775
165RRA:MIN:0.5:288:795
166RRA:MAX:0.5:1:600
167RRA:MAX:0.5:6:700
168RRA:MAX:0.5:24:775
169RRA:MAX:0.5:288:795
170""".strip().split(
171 "\n"
172 )
173 )
174 try:
175 rrdtool.create(*create)
176 rrdtool.update(*update)
177 except rrdtool.error as e:
178 print(
179 (
180 "warning: queue_report: rrdtool error, skipping %s.rrd: %s"
181 % (type, e)
182 )
183 )
184 except NameError:
185 pass
188def get_upload_data(changesfn):
189 with open(changesfn) as fd:
190 achanges = deb822.Changes(fd)
191 changesname = os.path.basename(changesfn)
192 delay = os.path.basename(os.path.dirname(changesfn))
193 m = re.match(r"([0-9]+)-day", delay)
194 if m:
195 delaydays = int(m.group(1))
196 remainingtime = (delaydays > 0) * max(
197 0, 24 * 60 * 60 + os.stat(changesfn).st_mtime - time.time()
198 )
199 delay = "%d days %02d:%02d" % (
200 max(delaydays - 1, 0),
201 int(remainingtime / 3600),
202 int(remainingtime / 60) % 60,
203 )
204 else:
205 delaydays = 0
206 remainingtime = 0
208 uploader = achanges.get("changed-by")
209 if uploader is not None:
210 uploader = re.sub(r"^\s*(\S.*)\s+<.*>", r"\1", uploader)
212 with open(changesfn, "rb") as f:
213 fingerprint = SignedFile(
214 f.read(), keyrings=get_active_keyring_paths(), require_signature=False
215 ).fingerprint
216 if "Show-Deferred::LinkPath" in Cnf:
217 isnew = False
218 suites = get_suites_source_in(achanges["source"])
219 if "unstable" not in suites and "experimental" not in suites:
220 isnew = True
222 if not isnew:
223 # we don't link .changes because we don't want other people to
224 # upload it with the existing signature.
225 for afn in [x["name"] for x in achanges["files"]]:
226 lfn = os.path.join(Cnf["Show-Deferred::LinkPath"], afn)
227 qfn = os.path.join(os.path.dirname(changesfn), afn)
228 if os.path.islink(lfn):
229 os.unlink(lfn)
230 if os.path.exists(qfn):
231 os.symlink(qfn, lfn)
232 os.chmod(qfn, 0o644)
233 return (
234 max(delaydays - 1, 0) * 24 * 60 * 60 + remainingtime,
235 changesname,
236 delay,
237 uploader,
238 achanges.get("closes", "").split(),
239 fingerprint,
240 achanges,
241 delaydays,
242 )
245def list_uploads(filelist, rrd_dir):
246 uploads = sorted(get_upload_data(x) for x in filelist)
247 # print the summary page
248 print(header())
249 if uploads:
250 print(table_header())
251 print("".join(table_row(*x[1:6]) for x in uploads))
252 print(table_footer())
253 else:
254 print("<h1>Currently no deferred uploads to Debian</h1>")
255 print(footer())
256 # machine readable summary
257 if "Show-Deferred::LinkPath" in Cnf:
258 fn = os.path.join(Cnf["Show-Deferred::LinkPath"], ".status.tmp")
259 f = open(fn, "w")
260 try:
261 counts = [0] * 16
262 for u in uploads:
263 counts[u[7]] += 1
264 print("Changes-file: %s" % u[1], file=f)
265 fields = """Location: DEFERRED
266Delayed-Until: %s
267Delay-Remaining: %s
268Fingerprint: %s""" % (
269 time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time() + u[0])),
270 u[2],
271 u[5],
272 )
273 print(fields, file=f)
274 encoded = u[6].dump()
275 print(encoded.rstrip(), file=f)
276 open(os.path.join(Cnf["Show-Deferred::LinkPath"], u[1]), "w").write(
277 encoded + fields + "\n"
278 )
279 print(file=f)
280 f.close()
281 os.rename(
282 os.path.join(Cnf["Show-Deferred::LinkPath"], ".status.tmp"),
283 os.path.join(Cnf["Show-Deferred::LinkPath"], "status"),
284 )
285 update_graph_database(rrd_dir, *counts)
286 except:
287 os.unlink(fn)
288 raise
291def usage(exit_code=0):
292 if exit_code: 292 ↛ 293line 292 didn't jump to line 293 because the condition on line 292 was never true
293 f = sys.stderr
294 else:
295 f = sys.stdout
296 print(
297 """Usage: dak show-deferred
298 -h, --help show this help and exit.
299 -p, --link-path [path] override output directory.
300 -d, --deferred-queue [path] path to the deferred queue
301 -r, --rrd=key Directory where rrd files to be updated are stored
302 """,
303 file=f,
304 )
305 sys.exit(exit_code)
308def init():
309 global Cnf, Options
310 Cnf = utils.get_conf()
311 Arguments = [
312 ("h", "help", "Show-Deferred::Options::Help"),
313 ("p", "link-path", "Show-Deferred::LinkPath", "HasArg"),
314 ("d", "deferred-queue", "Show-Deferred::DeferredQueue", "HasArg"),
315 ("r", "rrd", "Show-Deferred::Options::Rrd", "HasArg"),
316 ]
317 args = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) # type: ignore[attr-defined]
318 for i in ["help"]:
319 key = "Show-Deferred::Options::%s" % i
320 if key not in Cnf: 320 ↛ 321line 320 didn't jump to line 321 because the condition on line 320 was never true
321 Cnf[key] = "" # type: ignore[index]
322 for i, j in [("DeferredQueue", "--deferred-queue")]:
323 key = "Show-Deferred::%s" % i
324 if key not in Cnf: 324 ↛ 322line 324 didn't jump to line 322 because the condition on line 324 was always true
325 print(
326 """%s is mandatory.
327 set via config file or command-line option %s"""
328 % (key, j),
329 file=sys.stderr,
330 )
332 Options = Cnf.subtree("Show-Deferred::Options") # type: ignore[attr-defined]
333 if Options["help"]: 333 ↛ 337line 333 didn't jump to line 337 because the condition on line 333 was always true
334 usage()
336 # Initialise database connection
337 DBConn()
339 return args
342def main():
343 args = init()
344 if len(args) != 0:
345 usage(1)
347 if "Show-Deferred::Options::Rrd" in Cnf:
348 rrd_dir = Cnf["Show-Deferred::Options::Rrd"]
349 elif "Dir::Rrd" in Cnf:
350 rrd_dir = Cnf["Dir::Rrd"]
351 else:
352 rrd_dir = None
354 filelist: list[str] = []
355 for r, d, f in os.walk(Cnf["Show-Deferred::DeferredQueue"]):
356 filelist.extend(os.path.join(r, x) for x in f if x.endswith(".changes"))
357 list_uploads(filelist, rrd_dir)
359 available_changes = set(os.path.basename(x) for x in filelist)
360 if "Show-Deferred::LinkPath" in Cnf:
361 # remove dead links
362 for r, d, f in os.walk(Cnf["Show-Deferred::LinkPath"]):
363 for af in f:
364 afp = os.path.join(r, af)
365 if not os.path.exists(afp) or (
366 af.endswith(".changes") and af not in available_changes
367 ):
368 os.unlink(afp)
371if __name__ == "__main__":
372 main()