Coverage for dak/show_deferred.py: 23%
151 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-08-03 16:46 +0000
1"""Overview of the DEFERRED queue, based on queue-report"""
3# Copyright (C) 2001, 2002, 2003, 2005, 2006 James Troup <james@nocrew.org>
4# Copyright (C) 2008 Thomas Viehmann <tv@beamnet.de>
6# This program is free software; you can redistribute it and/or modify
7# it under the terms of the GNU General Public License as published by
8# the Free Software Foundation; either version 2 of the License, or
9# (at your option) any later version.
11# This program is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14# GNU General Public License for more details.
16# You should have received a copy of the GNU General Public License
17# along with this program; if not, write to the Free Software
18# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20################################################################################
22import html
23import os
24import re
25import sys
26import time
28import apt_pkg
29import rrdtool
31from daklib import utils
32from daklib.dbconn import DBConn, get_active_keyring_paths, get_suites_source_in
33from daklib.gpg import SignedFile
34from debian import deb822
36Cnf: apt_pkg.Configuration
37Options: apt_pkg.Configuration
39################################################################################
42def header():
43 return """<!DOCTYPE html>
44 <html lang="en"><head><meta charset="utf-8">
45 <title>Deferred uploads to Debian</title>
46 <link rel="stylesheet" href="style.css">
47 <link rel="shortcut icon" href="https://www.debian.org/favicon.ico">
48 </head>
49 <body>
50 <div align="center">
51 <a href="https://www.debian.org/">
52 <img src="https://www.debian.org/logos/openlogo-nd-50.png" border="0" hspace="0" vspace="0" alt=""></a>
53 <a href="https://www.debian.org/">
54 <img src="https://www.debian.org/Pics/debian.png" border="0" hspace="0" vspace="0" alt="Debian Project"></a>
55 </div>
56 <br>
57 <table class="reddy" width="100%">
58 <tr>
59 <td class="reddy">
60 <img src="https://www.debian.org/Pics/red-upperleft.png" align="left" border="0" hspace="0" vspace="0"
61 alt="" width="15" height="16"></td>
62 <td rowspan="2" class="reddy">Deferred uploads to Debian</td>
63 <td class="reddy">
64 <img src="https://www.debian.org/Pics/red-upperright.png" align="right" border="0" hspace="0" vspace="0"
65 alt="" width="16" height="16"></td>
66 </tr>
67 <tr>
68 <td class="reddy">
69 <img src="https://www.debian.org/Pics/red-lowerleft.png" align="left" border="0" hspace="0" vspace="0"
70 alt="" width="16" height="16"></td>
71 <td class="reddy">
72 <img src="https://www.debian.org/Pics/red-lowerright.png" align="right" border="0" hspace="0" vspace="0"
73 alt="" width="15" height="16"></td>
74 </tr>
75 </table>
76 """
79def footer():
80 res = '<p class="validate">Timestamp: %s (UTC)</p>' % (
81 time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime())
82 )
83 res += '<p class="timestamp">There are <a href="/stat.html">graphs about the queues</a> available.</p>'
84 res += "</body></html>"
85 return res
88def table_header():
89 return """<h1>Deferred uploads</h1>
90 <center><table border="0">
91 <tr>
92 <th align="center">Change</th>
93 <th align="center">Time remaining</th>
94 <th align="center">Uploader</th>
95 <th align="center">Closes</th>
96 </tr>
97 """
100def table_footer():
101 return '</table><br><p>non-NEW uploads are <a href="/deferred/">available</a> (<a href="/deferred/status">machine readable version</a>), see the <a href="https://www.debian.org/doc/manuals/developers-reference/ch05.en.html#delayed-incoming">Developer\'s reference</a> for more information on the DELAYED queue.</p></center><br>\n'
104def table_row(changesname, delay, changed_by, closes, fingerprint):
105 res = "<tr>"
106 res += (2 * '<td valign="top">%s</td>') % tuple(
107 html.escape(x, quote=False) for x in (changesname, delay)
108 )
109 res += (
110 '<td valign="top">%s<br><span class="deferredfp">Fingerprint: %s</span></td>'
111 % (html.escape(changed_by or "Unknown", quote=False), fingerprint)
112 )
113 res += '<td valign="top">%s</td>' % "".join(
114 [
115 '<a href="https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s">#%s</a><br>'
116 % (close, close)
117 for close in closes
118 ]
119 )
120 res += "</tr>\n"
121 return res
124def update_graph_database(rrd_dir, *counts):
125 if not rrd_dir:
126 return
128 rrd_file = os.path.join(rrd_dir, "deferred.rrd")
129 update = [rrd_file, "N:" + ":".join(str(count) for count in counts)]
131 try:
132 rrdtool.update(*update)
133 except rrdtool.error:
134 create = (
135 [rrd_file]
136 + """
137--step
138300
139--start
1400
141DS:day0:GAUGE:7200:0:1000
142DS:day1:GAUGE:7200:0:1000
143DS:day2:GAUGE:7200:0:1000
144DS:day3:GAUGE:7200:0:1000
145DS:day4:GAUGE:7200:0:1000
146DS:day5:GAUGE:7200:0:1000
147DS:day6:GAUGE:7200:0:1000
148DS:day7:GAUGE:7200:0:1000
149DS:day8:GAUGE:7200:0:1000
150DS:day9:GAUGE:7200:0:1000
151DS:day10:GAUGE:7200:0:1000
152DS:day11:GAUGE:7200:0:1000
153DS:day12:GAUGE:7200:0:1000
154DS:day13:GAUGE:7200:0:1000
155DS:day14:GAUGE:7200:0:1000
156DS:day15:GAUGE:7200:0:1000
157RRA:AVERAGE:0.5:1:599
158RRA:AVERAGE:0.5:6:700
159RRA:AVERAGE:0.5:24:775
160RRA:AVERAGE:0.5:288:795
161RRA:MIN:0.5:1:600
162RRA:MIN:0.5:6:700
163RRA:MIN:0.5:24:775
164RRA:MIN:0.5:288:795
165RRA:MAX:0.5:1:600
166RRA:MAX:0.5:6:700
167RRA:MAX:0.5:24:775
168RRA:MAX:0.5:288:795
169""".strip().split(
170 "\n"
171 )
172 )
173 try:
174 rrdtool.create(*create)
175 rrdtool.update(*update)
176 except rrdtool.error as e:
177 print(
178 "warning: queue_report: rrdtool error, skipping %s.rrd: %s" % (type, e)
179 )
180 except NameError:
181 pass
184def get_upload_data(changesfn):
185 with open(changesfn) as fd:
186 achanges = deb822.Changes(fd)
187 changesname = os.path.basename(changesfn)
188 delay = os.path.basename(os.path.dirname(changesfn))
189 m = re.match(r"([0-9]+)-day", delay)
190 if m:
191 delaydays = int(m.group(1))
192 remainingtime = (delaydays > 0) * max(
193 0, 24 * 60 * 60 + os.stat(changesfn).st_mtime - time.time()
194 )
195 delay = "%d days %02d:%02d" % (
196 max(delaydays - 1, 0),
197 int(remainingtime / 3600),
198 int(remainingtime / 60) % 60,
199 )
200 else:
201 delaydays = 0
202 remainingtime = 0
204 uploader = achanges.get("changed-by")
205 if uploader is not None:
206 uploader = re.sub(r"^\s*(\S.*)\s+<.*>", r"\1", uploader)
208 with open(changesfn, "rb") as f:
209 fingerprint = SignedFile(
210 f.read(), keyrings=get_active_keyring_paths(), require_signature=False
211 ).fingerprint
212 if "Show-Deferred::LinkPath" in Cnf:
213 isnew = False
214 suites = get_suites_source_in(achanges["source"])
215 if "unstable" not in suites and "experimental" not in suites:
216 isnew = True
218 if not isnew:
219 # we don't link .changes because we don't want other people to
220 # upload it with the existing signature.
221 for afn in [x["name"] for x in achanges["files"]]:
222 lfn = os.path.join(Cnf["Show-Deferred::LinkPath"], afn)
223 qfn = os.path.join(os.path.dirname(changesfn), afn)
224 if os.path.islink(lfn):
225 os.unlink(lfn)
226 if os.path.exists(qfn):
227 os.symlink(qfn, lfn)
228 os.chmod(qfn, 0o644)
229 return (
230 max(delaydays - 1, 0) * 24 * 60 * 60 + remainingtime,
231 changesname,
232 delay,
233 uploader,
234 achanges.get("closes", "").split(),
235 fingerprint,
236 achanges,
237 delaydays,
238 )
241def list_uploads(filelist, rrd_dir):
242 uploads = sorted(get_upload_data(x) for x in filelist)
243 # print the summary page
244 print(header())
245 if uploads:
246 print(table_header())
247 print("".join(table_row(*x[1:6]) for x in uploads))
248 print(table_footer())
249 else:
250 print("<h1>Currently no deferred uploads to Debian</h1>")
251 print(footer())
252 # machine readable summary
253 if "Show-Deferred::LinkPath" in Cnf:
254 fn = os.path.join(Cnf["Show-Deferred::LinkPath"], ".status.tmp")
255 f = open(fn, "w")
256 try:
257 counts = [0] * 16
258 for u in uploads:
259 counts[u[7]] += 1
260 print("Changes-file: %s" % u[1], file=f)
261 fields = """Location: DEFERRED
262Delayed-Until: %s
263Delay-Remaining: %s
264Fingerprint: %s""" % (
265 time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time() + u[0])),
266 u[2],
267 u[5],
268 )
269 print(fields, file=f)
270 encoded = u[6].dump()
271 print(encoded.rstrip(), file=f)
272 open(os.path.join(Cnf["Show-Deferred::LinkPath"], u[1]), "w").write(
273 encoded + fields + "\n"
274 )
275 print(file=f)
276 f.close()
277 os.rename(
278 os.path.join(Cnf["Show-Deferred::LinkPath"], ".status.tmp"),
279 os.path.join(Cnf["Show-Deferred::LinkPath"], "status"),
280 )
281 update_graph_database(rrd_dir, *counts)
282 except:
283 os.unlink(fn)
284 raise
287def usage(exit_code=0):
288 if exit_code: 288 ↛ 289line 288 didn't jump to line 289 because the condition on line 288 was never true
289 f = sys.stderr
290 else:
291 f = sys.stdout
292 print(
293 """Usage: dak show-deferred
294 -h, --help show this help and exit.
295 -p, --link-path [path] override output directory.
296 -d, --deferred-queue [path] path to the deferred queue
297 -r, --rrd=key Directory where rrd files to be updated are stored
298 """,
299 file=f,
300 )
301 sys.exit(exit_code)
304def init():
305 global Cnf, Options
306 Cnf = utils.get_conf()
307 Arguments = [
308 ("h", "help", "Show-Deferred::Options::Help"),
309 ("p", "link-path", "Show-Deferred::LinkPath", "HasArg"),
310 ("d", "deferred-queue", "Show-Deferred::DeferredQueue", "HasArg"),
311 ("r", "rrd", "Show-Deferred::Options::Rrd", "HasArg"),
312 ]
313 args = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) # type: ignore[attr-defined]
314 for i in ["help"]:
315 key = "Show-Deferred::Options::%s" % i
316 if key not in Cnf: 316 ↛ 317line 316 didn't jump to line 317 because the condition on line 316 was never true
317 Cnf[key] = "" # type: ignore[index]
318 for i, j in [("DeferredQueue", "--deferred-queue")]:
319 key = "Show-Deferred::%s" % i
320 if key not in Cnf: 320 ↛ 318line 320 didn't jump to line 318 because the condition on line 320 was always true
321 print(
322 """%s is mandatory.
323 set via config file or command-line option %s"""
324 % (key, j),
325 file=sys.stderr,
326 )
328 Options = Cnf.subtree("Show-Deferred::Options") # type: ignore[attr-defined]
329 if Options["help"]: 329 ↛ 333line 329 didn't jump to line 333 because the condition on line 329 was always true
330 usage()
332 # Initialise database connection
333 DBConn()
335 return args
338def main():
339 args = init()
340 if len(args) != 0:
341 usage(1)
343 if "Show-Deferred::Options::Rrd" in Cnf:
344 rrd_dir = Cnf["Show-Deferred::Options::Rrd"]
345 elif "Dir::Rrd" in Cnf:
346 rrd_dir = Cnf["Dir::Rrd"]
347 else:
348 rrd_dir = None
350 filelist: list[str] = []
351 for r, d, f in os.walk(Cnf["Show-Deferred::DeferredQueue"]):
352 filelist.extend(os.path.join(r, x) for x in f if x.endswith(".changes"))
353 list_uploads(filelist, rrd_dir)
355 available_changes = {os.path.basename(x) for x in filelist}
356 if "Show-Deferred::LinkPath" in Cnf:
357 # remove dead links
358 for r, d, f in os.walk(Cnf["Show-Deferred::LinkPath"]):
359 for af in f:
360 afp = os.path.join(r, af)
361 if not os.path.exists(afp) or (
362 af.endswith(".changes") and af not in available_changes
363 ):
364 os.unlink(afp)