dak.show_deferred

1 #! /usr/bin/env python3 2 3 """Overview of the DEFERRED queue, based on queue-report""" 4 # Copyright (C) 2001, 2002, 2003, 2005, 2006 James Troup <james@nocrew.org> 5 # Copyright (C) 2008 Thomas Viehmann <tv@beamnet.de> 6 7 # This program is free software; you can redistribute it and/or modify 8 # it under the terms of the GNU General Public License as published by 9 # the Free Software Foundation; either version 2 of the License, or 10 # (at your option) any later version. 11 12 # This program is distributed in the hope that it will be useful, 13 # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 # GNU General Public License for more details. 16 17 # You should have received a copy of the GNU General Public License 18 # along with this program; if not, write to the Free Software 19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 20 21 ################################################################################ 22 23 import html 24 import os 25 import re 26 import sys 27 import time 28 29 import apt_pkg 30 import rrdtool 31 32 from daklib import utils 33 from daklib.dbconn import DBConn, get_active_keyring_paths, get_suites_source_in 34 from daklib.gpg import SignedFile 35 from debian import deb822 36 37 ################################################################################ 38 39

40 -def header():

41 return """<!DOCTYPE html> 42 <html lang="en"><head><meta charset="utf-8"> 43 <title>Deferred uploads to Debian</title> 44 <link rel="stylesheet" href="style.css"> 45 <link rel="shortcut icon" href="https://www.debian.org/favicon.ico"> 46 </head> 47 <body> 48 <div align="center"> 49 <a href="https://www.debian.org/"> 50 <img src="https://www.debian.org/logos/openlogo-nd-50.png" border="0" hspace="0" vspace="0" alt=""></a> 51 <a href="https://www.debian.org/"> 52 <img src="https://www.debian.org/Pics/debian.png" border="0" hspace="0" vspace="0" alt="Debian Project"></a> 53 </div> 54 <br> 55 <table class="reddy" width="100%"> 56 <tr> 57 <td class="reddy"> 58 <img src="https://www.debian.org/Pics/red-upperleft.png" align="left" border="0" hspace="0" vspace="0" 59 alt="" width="15" height="16"></td> 60 <td rowspan="2" class="reddy">Deferred uploads to Debian</td> 61 <td class="reddy"> 62 <img src="https://www.debian.org/Pics/red-upperright.png" align="right" border="0" hspace="0" vspace="0" 63 alt="" width="16" height="16"></td> 64 </tr> 65 <tr> 66 <td class="reddy"> 67 <img src="https://www.debian.org/Pics/red-lowerleft.png" align="left" border="0" hspace="0" vspace="0" 68 alt="" width="16" height="16"></td> 69 <td class="reddy"> 70 <img src="https://www.debian.org/Pics/red-lowerright.png" align="right" border="0" hspace="0" vspace="0" 71 alt="" width="15" height="16"></td> 72 </tr> 73 </table> 74 """

75 76 84 85

86 -def table_header():

87 return """<h1>Deferred uploads</h1> 88 <center><table border="0"> 89 <tr> 90 <th align="center">Change</th> 91 <th align="center">Time remaining</th> 92 <th align="center">Uploader</th> 93 <th align="center">Closes</th> 94 </tr> 95 """

96 97

98 -def table_footer():

99 return '</table><br><p>non-NEW uploads are <a href="/deferred/">available</a> (<a href="/deferred/status">machine readable version</a>), see the <a href="https://www.debian.org/doc/manuals/developers-reference/ch05.en.html#delayed-incoming">Developer\'s reference</a> for more information on the DELAYED queue.</p></center><br>\n'

100 101

102 -def table_row(changesname, delay, changed_by, closes, fingerprint):

103 res = "<tr>" 104 res += (2 * '<td valign="top">%s</td>') % tuple( 105 html.escape(x, quote=False) for x in (changesname, delay) 106 ) 107 res += ( 108 '<td valign="top">%s<br><span class="deferredfp">Fingerprint: %s</span></td>' 109 % (html.escape(changed_by, quote=False), fingerprint) 110 ) 111 res += '<td valign="top">%s</td>' % "".join( 112 [ 113 '<a href="https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s">#%s</a><br>' 114 % (close, close) 115 for close in closes 116 ] 117 ) 118 res += "</tr>\n" 119 return res

120 121

122 -def update_graph_database(rrd_dir, *counts):

123 if not rrd_dir: 124 return 125 126 rrd_file = os.path.join(rrd_dir, "deferred.rrd") 127 counts = [str(count) for count in counts] 128 update = [rrd_file, "N:" + ":".join(counts)] 129 130 try: 131 rrdtool.update(*update) 132 except rrdtool.error: 133 create = ( 134 [rrd_file] 135 + """ 136 --step 137 300 138 --start 139 0 140 DS:day0:GAUGE:7200:0:1000 141 DS:day1:GAUGE:7200:0:1000 142 DS:day2:GAUGE:7200:0:1000 143 DS:day3:GAUGE:7200:0:1000 144 DS:day4:GAUGE:7200:0:1000 145 DS:day5:GAUGE:7200:0:1000 146 DS:day6:GAUGE:7200:0:1000 147 DS:day7:GAUGE:7200:0:1000 148 DS:day8:GAUGE:7200:0:1000 149 DS:day9:GAUGE:7200:0:1000 150 DS:day10:GAUGE:7200:0:1000 151 DS:day11:GAUGE:7200:0:1000 152 DS:day12:GAUGE:7200:0:1000 153 DS:day13:GAUGE:7200:0:1000 154 DS:day14:GAUGE:7200:0:1000 155 DS:day15:GAUGE:7200:0:1000 156 RRA:AVERAGE:0.5:1:599 157 RRA:AVERAGE:0.5:6:700 158 RRA:AVERAGE:0.5:24:775 159 RRA:AVERAGE:0.5:288:795 160 RRA:MIN:0.5:1:600 161 RRA:MIN:0.5:6:700 162 RRA:MIN:0.5:24:775 163 RRA:MIN:0.5:288:795 164 RRA:MAX:0.5:1:600 165 RRA:MAX:0.5:6:700 166 RRA:MAX:0.5:24:775 167 RRA:MAX:0.5:288:795 168 """.strip().split( 169 "\n" 170 ) 171 ) 172 try: 173 rrdtool.create(*create) 174 rrdtool.update(*update) 175 except rrdtool.error as e: 176 print( 177 ( 178 "warning: queue_report: rrdtool error, skipping %s.rrd: %s" 179 % (type, e) 180 ) 181 ) 182 except NameError: 183 pass

184 185

186 -def get_upload_data(changesfn):

187 with open(changesfn) as fd: 188 achanges = deb822.Changes(fd) 189 changesname = os.path.basename(changesfn) 190 delay = os.path.basename(os.path.dirname(changesfn)) 191 m = re.match(r"([0-9]+)-day", delay) 192 if m: 193 delaydays = int(m.group(1)) 194 remainingtime = (delaydays > 0) * max( 195 0, 24 * 60 * 60 + os.stat(changesfn).st_mtime - time.time() 196 ) 197 delay = "%d days %02d:%02d" % ( 198 max(delaydays - 1, 0), 199 int(remainingtime / 3600), 200 int(remainingtime / 60) % 60, 201 ) 202 else: 203 delaydays = 0 204 remainingtime = 0 205 206 uploader = achanges.get("changed-by") 207 uploader = re.sub(r"^\s*(\S.*)\s+<.*>", r"\1", uploader) 208 with open(changesfn, "rb") as f: 209 fingerprint = SignedFile( 210 f.read(), keyrings=get_active_keyring_paths(), require_signature=False 211 ).fingerprint 212 if "Show-Deferred::LinkPath" in Cnf: 213 isnew = False 214 suites = get_suites_source_in(achanges["source"]) 215 if "unstable" not in suites and "experimental" not in suites: 216 isnew = True 217 218 if not isnew: 219 # we don't link .changes because we don't want other people to 220 # upload it with the existing signature. 221 for afn in [x["name"] for x in achanges["files"]]: 222 lfn = os.path.join(Cnf["Show-Deferred::LinkPath"], afn) 223 qfn = os.path.join(os.path.dirname(changesfn), afn) 224 if os.path.islink(lfn): 225 os.unlink(lfn) 226 if os.path.exists(qfn): 227 os.symlink(qfn, lfn) 228 os.chmod(qfn, 0o644) 229 return ( 230 max(delaydays - 1, 0) * 24 * 60 * 60 + remainingtime, 231 changesname, 232 delay, 233 uploader, 234 achanges.get("closes", "").split(), 235 fingerprint, 236 achanges, 237 delaydays, 238 )

239 240

241 -def list_uploads(filelist, rrd_dir):

242 uploads = sorted(get_upload_data(x) for x in filelist) 243 # print the summary page 244 print(header()) 245 if uploads: 246 print(table_header()) 247 print("".join(table_row(*x[1:6]) for x in uploads)) 248 print(table_footer()) 249 else: 250 print("<h1>Currently no deferred uploads to Debian</h1>") 251 print(footer()) 252 # machine readable summary 253 if "Show-Deferred::LinkPath" in Cnf: 254 fn = os.path.join(Cnf["Show-Deferred::LinkPath"], ".status.tmp") 255 f = open(fn, "w") 256 try: 257 counts = [0] * 16 258 for u in uploads: 259 counts[u[7]] += 1 260 print("Changes-file: %s" % u[1], file=f) 261 fields = """Location: DEFERRED 262 Delayed-Until: %s 263 Delay-Remaining: %s 264 Fingerprint: %s""" % ( 265 time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time() + u[0])), 266 u[2], 267 u[5], 268 ) 269 print(fields, file=f) 270 encoded = u[6].dump() 271 print(encoded.rstrip(), file=f) 272 open(os.path.join(Cnf["Show-Deferred::LinkPath"], u[1]), "w").write( 273 encoded + fields + "\n" 274 ) 275 print(file=f) 276 f.close() 277 os.rename( 278 os.path.join(Cnf["Show-Deferred::LinkPath"], ".status.tmp"), 279 os.path.join(Cnf["Show-Deferred::LinkPath"], "status"), 280 ) 281 update_graph_database(rrd_dir, *counts) 282 except: 283 os.unlink(fn) 284 raise

285 286

287 -def usage(exit_code=0):

288 if exit_code: 289 f = sys.stderr 290 else: 291 f = sys.stdout 292 print( 293 """Usage: dak show-deferred 294 -h, --help show this help and exit. 295 -p, --link-path [path] override output directory. 296 -d, --deferred-queue [path] path to the deferred queue 297 -r, --rrd=key Directory where rrd files to be updated are stored 298 """, 299 file=f, 300 ) 301 sys.exit(exit_code)

302 303

304 -def init():

305 global Cnf, Options 306 Cnf = utils.get_conf() 307 Arguments = [ 308 ("h", "help", "Show-Deferred::Options::Help"), 309 ("p", "link-path", "Show-Deferred::LinkPath", "HasArg"), 310 ("d", "deferred-queue", "Show-Deferred::DeferredQueue", "HasArg"), 311 ("r", "rrd", "Show-Deferred::Options::Rrd", "HasArg"), 312 ] 313 args = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) 314 for i in ["help"]: 315 key = "Show-Deferred::Options::%s" % i 316 if key not in Cnf: 317 Cnf[key] = "" 318 for i, j in [("DeferredQueue", "--deferred-queue")]: 319 key = "Show-Deferred::%s" % i 320 if key not in Cnf: 321 print( 322 """%s is mandatory. 323 set via config file or command-line option %s""" 324 % (key, j), 325 file=sys.stderr, 326 ) 327 328 Options = Cnf.subtree("Show-Deferred::Options") 329 if Options["help"]: 330 usage() 331 332 # Initialise database connection 333 DBConn() 334 335 return args

336 337

338 -def main():

339 args = init() 340 if len(args) != 0: 341 usage(1) 342 343 if "Show-Deferred::Options::Rrd" in Cnf: 344 rrd_dir = Cnf["Show-Deferred::Options::Rrd"] 345 elif "Dir::Rrd" in Cnf: 346 rrd_dir = Cnf["Dir::Rrd"] 347 else: 348 rrd_dir = None 349 350 filelist = [] 351 for r, d, f in os.walk(Cnf["Show-Deferred::DeferredQueue"]): 352 filelist.extend(os.path.join(r, x) for x in f if x.endswith(".changes")) 353 list_uploads(filelist, rrd_dir) 354 355 available_changes = set(os.path.basename(x) for x in filelist) 356 if "Show-Deferred::LinkPath" in Cnf: 357 # remove dead links 358 for r, d, f in os.walk(Cnf["Show-Deferred::LinkPath"]): 359 for af in f: 360 afp = os.path.join(r, af) 361 if not os.path.exists(afp) or ( 362 af.endswith(".changes") and af not in available_changes 363 ): 364 os.unlink(afp)

365 366 367 if __name__ == "__main__": 368 main() 369

Source Code for Module dak.show_deferred