dak.show_deferred

1 #! /usr/bin/env python3 2 3 """ Overview of the DEFERRED queue, based on queue-report """ 4 # Copyright (C) 2001, 2002, 2003, 2005, 2006 James Troup <james@nocrew.org> 5 # Copyright (C) 2008 Thomas Viehmann <tv@beamnet.de> 6 7 # This program is free software; you can redistribute it and/or modify 8 # it under the terms of the GNU General Public License as published by 9 # the Free Software Foundation; either version 2 of the License, or 10 # (at your option) any later version. 11 12 # This program is distributed in the hope that it will be useful, 13 # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 # GNU General Public License for more details. 16 17 # You should have received a copy of the GNU General Public License 18 # along with this program; if not, write to the Free Software 19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 20 21 ################################################################################ 22 23 import html 24 import sys 25 import os 26 import re 27 import time 28 import apt_pkg 29 import rrdtool 30 31 from debian import deb822 32 33 from daklib.dbconn import * 34 from daklib.gpg import SignedFile 35 from daklib import utils 36 37 ################################################################################ 38 39

40 -def header():

41 return """<!DOCTYPE html> 42 <html lang="en"><head><meta charset="utf-8"> 43 <title>Deferred uploads to Debian</title> 44 <link rel="stylesheet" href="style.css"> 45 <link rel="shortcut icon" href="https://www.debian.org/favicon.ico"> 46 </head> 47 <body> 48 <div align="center"> 49 <a href="https://www.debian.org/"> 50 <img src="https://www.debian.org/logos/openlogo-nd-50.png" border="0" hspace="0" vspace="0" alt=""></a> 51 <a href="https://www.debian.org/"> 52 <img src="https://www.debian.org/Pics/debian.png" border="0" hspace="0" vspace="0" alt="Debian Project"></a> 53 </div> 54 <br> 55 <table class="reddy" width="100%"> 56 <tr> 57 <td class="reddy"> 58 <img src="https://www.debian.org/Pics/red-upperleft.png" align="left" border="0" hspace="0" vspace="0" 59 alt="" width="15" height="16"></td> 60 <td rowspan="2" class="reddy">Deferred uploads to Debian</td> 61 <td class="reddy"> 62 <img src="https://www.debian.org/Pics/red-upperright.png" align="right" border="0" hspace="0" vspace="0" 63 alt="" width="16" height="16"></td> 64 </tr> 65 <tr> 66 <td class="reddy"> 67 <img src="https://www.debian.org/Pics/red-lowerleft.png" align="left" border="0" hspace="0" vspace="0" 68 alt="" width="16" height="16"></td> 69 <td class="reddy"> 70 <img src="https://www.debian.org/Pics/red-lowerright.png" align="right" border="0" hspace="0" vspace="0" 71 alt="" width="15" height="16"></td> 72 </tr> 73 </table> 74 """

75 76 82 83

84 -def table_header():

85 return """<h1>Deferred uploads</h1> 86 <center><table border="0"> 87 <tr> 88 <th align="center">Change</th> 89 <th align="center">Time remaining</th> 90 <th align="center">Uploader</th> 91 <th align="center">Closes</th> 92 </tr> 93 """

94 95

96 -def table_footer():

97 return '</table><br><p>non-NEW uploads are <a href="/deferred/">available</a> (<a href="/deferred/status">machine readable version</a>), see the <a href="ftp://ftp.upload.debian.org/pub/UploadQueue/README">UploadQueue-README</a> and <a href="https://www.debian.org/doc/manuals/developers-reference/ch05.en.html#delayed-incoming">Developer\'s reference</a> for more information on the DELAYED queue.</p></center><br>\n'

98 99

100 -def table_row(changesname, delay, changed_by, closes, fingerprint):

101 res = '<tr>' 102 res += (2 * '<td valign="top">%s</td>') % tuple(html.escape(x, quote=False) for x in (changesname, delay)) 103 res += '<td valign="top">%s<br><span class=\"deferredfp\">Fingerprint: %s</span></td>' % (html.escape(changed_by, quote=False), fingerprint) 104 res += ('<td valign="top">%s</td>' % 105 ''.join(['<a href="https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s">#%s</a><br>' % (close, close) for close in closes])) 106 res += '</tr>\n' 107 return res

108 109

110 -def update_graph_database(rrd_dir, *counts):

111 if not rrd_dir: 112 return 113 114 rrd_file = os.path.join(rrd_dir, 'deferred.rrd') 115 counts = [str(count) for count in counts] 116 update = [rrd_file, "N:" + ":".join(counts)] 117 118 try: 119 rrdtool.update(*update) 120 except rrdtool.error: 121 create = [rrd_file] + """ 122 --step 123 300 124 --start 125 0 126 DS:day0:GAUGE:7200:0:1000 127 DS:day1:GAUGE:7200:0:1000 128 DS:day2:GAUGE:7200:0:1000 129 DS:day3:GAUGE:7200:0:1000 130 DS:day4:GAUGE:7200:0:1000 131 DS:day5:GAUGE:7200:0:1000 132 DS:day6:GAUGE:7200:0:1000 133 DS:day7:GAUGE:7200:0:1000 134 DS:day8:GAUGE:7200:0:1000 135 DS:day9:GAUGE:7200:0:1000 136 DS:day10:GAUGE:7200:0:1000 137 DS:day11:GAUGE:7200:0:1000 138 DS:day12:GAUGE:7200:0:1000 139 DS:day13:GAUGE:7200:0:1000 140 DS:day14:GAUGE:7200:0:1000 141 DS:day15:GAUGE:7200:0:1000 142 RRA:AVERAGE:0.5:1:599 143 RRA:AVERAGE:0.5:6:700 144 RRA:AVERAGE:0.5:24:775 145 RRA:AVERAGE:0.5:288:795 146 RRA:MIN:0.5:1:600 147 RRA:MIN:0.5:6:700 148 RRA:MIN:0.5:24:775 149 RRA:MIN:0.5:288:795 150 RRA:MAX:0.5:1:600 151 RRA:MAX:0.5:6:700 152 RRA:MAX:0.5:24:775 153 RRA:MAX:0.5:288:795 154 """.strip().split("\n") 155 try: 156 rc = rrdtool.create(*create) 157 ru = rrdtool.update(*update) 158 except rrdtool.error as e: 159 print(('warning: queue_report: rrdtool error, skipping %s.rrd: %s' % (type, e))) 160 except NameError: 161 pass

162 163

164 -def get_upload_data(changesfn):

165 with open(changesfn) as fd: 166 achanges = deb822.Changes(fd) 167 changesname = os.path.basename(changesfn) 168 delay = os.path.basename(os.path.dirname(changesfn)) 169 m = re.match(r'([0-9]+)-day', delay) 170 if m: 171 delaydays = int(m.group(1)) 172 remainingtime = (delaydays > 0) * max(0, 24 * 60 * 60 + os.stat(changesfn).st_mtime - time.time()) 173 delay = "%d days %02d:%02d" % (max(delaydays - 1, 0), int(remainingtime / 3600), int(remainingtime / 60) % 60) 174 else: 175 delaydays = 0 176 remainingtime = 0 177 178 uploader = achanges.get('changed-by') 179 uploader = re.sub(r'^\s*(\S.*)\s+<.*>', r'\1', uploader) 180 with open(changesfn, "rb") as f: 181 fingerprint = SignedFile(f.read(), keyrings=get_active_keyring_paths(), require_signature=False).fingerprint 182 if "Show-Deferred::LinkPath" in Cnf: 183 isnew = False 184 suites = get_suites_source_in(achanges['source']) 185 if 'unstable' not in suites and 'experimental' not in suites: 186 isnew = True 187 188 if not isnew: 189 # we don't link .changes because we don't want other people to 190 # upload it with the existing signature. 191 for afn in [x['name'] for x in achanges['files']]: 192 lfn = os.path.join(Cnf["Show-Deferred::LinkPath"], afn) 193 qfn = os.path.join(os.path.dirname(changesfn), afn) 194 if os.path.islink(lfn): 195 os.unlink(lfn) 196 if os.path.exists(qfn): 197 os.symlink(qfn, lfn) 198 os.chmod(qfn, 0o644) 199 return (max(delaydays - 1, 0) * 24 * 60 * 60 + remainingtime, changesname, delay, uploader, achanges.get('closes', '').split(), fingerprint, achanges, delaydays)

200 201

202 -def list_uploads(filelist, rrd_dir):

203 uploads = sorted(get_upload_data(x) for x in filelist) 204 # print the summary page 205 print(header()) 206 if uploads: 207 print(table_header()) 208 print(''.join(table_row(*x[1:6]) for x in uploads)) 209 print(table_footer()) 210 else: 211 print('<h1>Currently no deferred uploads to Debian</h1>') 212 print(footer()) 213 # machine readable summary 214 if "Show-Deferred::LinkPath" in Cnf: 215 fn = os.path.join(Cnf["Show-Deferred::LinkPath"], '.status.tmp') 216 f = open(fn, "w") 217 try: 218 counts = [0] * 16 219 for u in uploads: 220 counts[u[7]] += 1 221 print("Changes-file: %s" % u[1], file=f) 222 fields = """Location: DEFERRED 223 Delayed-Until: %s 224 Delay-Remaining: %s 225 Fingerprint: %s""" % (time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time() + u[0])), u[2], u[5]) 226 print(fields, file=f) 227 encoded = u[6].dump() 228 print(encoded.rstrip(), file=f) 229 open(os.path.join(Cnf["Show-Deferred::LinkPath"], u[1]), "w").write(encoded + fields + '\n') 230 print(file=f) 231 f.close() 232 os.rename(os.path.join(Cnf["Show-Deferred::LinkPath"], '.status.tmp'), 233 os.path.join(Cnf["Show-Deferred::LinkPath"], 'status')) 234 update_graph_database(rrd_dir, *counts) 235 except: 236 os.unlink(fn) 237 raise

238 239

240 -def usage(exit_code=0):

241 if exit_code: 242 f = sys.stderr 243 else: 244 f = sys.stdout 245 print("""Usage: dak show-deferred 246 -h, --help show this help and exit. 247 -p, --link-path [path] override output directory. 248 -d, --deferred-queue [path] path to the deferred queue 249 -r, --rrd=key Directory where rrd files to be updated are stored 250 """, file=f) 251 sys.exit(exit_code)

252 253

254 -def init():

255 global Cnf, Options 256 Cnf = utils.get_conf() 257 Arguments = [('h', "help", "Show-Deferred::Options::Help"), 258 ("p", "link-path", "Show-Deferred::LinkPath", "HasArg"), 259 ("d", "deferred-queue", "Show-Deferred::DeferredQueue", "HasArg"), 260 ('r', "rrd", "Show-Deferred::Options::Rrd", "HasArg")] 261 args = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) 262 for i in ["help"]: 263 key = "Show-Deferred::Options::%s" % i 264 if key not in Cnf: 265 Cnf[key] = "" 266 for i, j in [("DeferredQueue", "--deferred-queue")]: 267 key = "Show-Deferred::%s" % i 268 if key not in Cnf: 269 print("""%s is mandatory. 270 set via config file or command-line option %s""" % (key, j), file=sys.stderr) 271 272 Options = Cnf.subtree("Show-Deferred::Options") 273 if Options["help"]: 274 usage() 275 276 # Initialise database connection 277 DBConn() 278 279 return args

280 281

282 -def main():

283 args = init() 284 if len(args) != 0: 285 usage(1) 286 287 if "Show-Deferred::Options::Rrd" in Cnf: 288 rrd_dir = Cnf["Show-Deferred::Options::Rrd"] 289 elif "Dir::Rrd" in Cnf: 290 rrd_dir = Cnf["Dir::Rrd"] 291 else: 292 rrd_dir = None 293 294 filelist = [] 295 for r, d, f in os.walk(Cnf["Show-Deferred::DeferredQueue"]): 296 filelist.extend(os.path.join(r, x) for x in f if x.endswith('.changes')) 297 list_uploads(filelist, rrd_dir) 298 299 available_changes = set(os.path.basename(x) for x in filelist) 300 if "Show-Deferred::LinkPath" in Cnf: 301 # remove dead links 302 for r, d, f in os.walk(Cnf["Show-Deferred::LinkPath"]): 303 for af in f: 304 afp = os.path.join(r, af) 305 if (not os.path.exists(afp) 306 or (af.endswith('.changes') and af not in available_changes)): 307 os.unlink(afp)

308 309 310 if __name__ == '__main__': 311 main() 312

Source Code for Module dak.show_deferred