1
2
3 """ Produces a report on NEW and BYHAND packages """
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37 import html
38 import os
39 import sys
40 import time
41 import apt_pkg
42 import datetime
43 import functools
44
45 from daklib import utils
46 from daklib.utils import get_logins_from_ldap
47 from daklib.dbconn import DBConn, has_new_comment, PolicyQueue, get_uid_from_fingerprint
48 from daklib.policy import PolicyQueueUploadHandler
49 from daklib.textutils import fix_maintainer
50 from daklib.dak_exceptions import ParseMaintError
51
52 Cnf = None
53 direction = []
54
55
56
57
59 print("""Usage: dak queue-report
60 Prints a report of packages in queues (usually new and byhand).
61
62 -h, --help show this help and exit.
63 -8, --822 writes 822 formated output to the location set in dak.conf
64 -n, --new produce html-output
65 -s, --sort=key sort output according to key, see below.
66 -a, --age=key if using sort by age, how should time be treated?
67 If not given a default of hours will be used.
68 -r, --rrd=key Directory where rrd files to be updated are stored
69 -d, --directories=key A comma separated list of queues to be scanned
70
71 Sorting Keys: ao=age, oldest first. an=age, newest first.
72 na=name, ascending nd=name, descending
73 nf=notes, first nl=notes, last
74
75 Age Keys: m=minutes, h=hours, d=days, w=weeks, o=months, y=years
76
77 """)
78 sys.exit(exit_code)
79
80
81
82
84 if x > 1:
85 return "s"
86 else:
87 return ""
88
89
90
91
93 if x < 60:
94 unit = "second"
95 elif x < 3600:
96 x /= 60
97 unit = "minute"
98 elif x < 86400:
99 x /= 3600
100 unit = "hour"
101 elif x < 604800:
102 x /= 86400
103 unit = "day"
104 elif x < 2419200:
105 x /= 604800
106 unit = "week"
107 elif x < 29030400:
108 x /= 2419200
109 unit = "month"
110 else:
111 x /= 29030400
112 unit = "year"
113 x = int(x)
114 return "%s %s%s" % (x, unit, plural(x))
115
116
117
118
120 a = a[1]
121 b = b[1]
122
123
124 a_note_state = a["processed"]
125 b_note_state = b["processed"]
126 if a_note_state < b_note_state:
127 return -1
128 elif a_note_state > b_note_state:
129 return 1
130
131
132 a_note_state = a["note_state"]
133 b_note_state = b["note_state"]
134 if a_note_state < b_note_state:
135 return -1
136 elif a_note_state > b_note_state:
137 return 1
138
139
140 return a["oldest"] - b["oldest"]
141
142
143
144
146 for sorting in direction:
147 (sortkey, way, time) = sorting
148 ret = 0
149 if time == "m":
150 x = int(a[sortkey] / 60)
151 y = int(b[sortkey] / 60)
152 elif time == "h":
153 x = int(a[sortkey] / 3600)
154 y = int(b[sortkey] / 3600)
155 elif time == "d":
156 x = int(a[sortkey] / 86400)
157 y = int(b[sortkey] / 86400)
158 elif time == "w":
159 x = int(a[sortkey] / 604800)
160 y = int(b[sortkey] / 604800)
161 elif time == "o":
162 x = int(a[sortkey] / 2419200)
163 y = int(b[sortkey] / 2419200)
164 elif time == "y":
165 x = int(a[sortkey] / 29030400)
166 y = int(b[sortkey] / 29030400)
167 else:
168 x = a[sortkey]
169 y = b[sortkey]
170 if x < y:
171 ret = -1
172 elif x > y:
173 ret = 1
174 if ret != 0:
175 if way < 0:
176 ret = ret * -1
177 return ret
178 return 0
179
180
181
182
184 print("""<!DOCTYPE html>
185 <html lang="en">
186 <head>
187 <meta charset="utf-8">
188 <link rel="stylesheet" href="style.css">
189 <link rel="shortcut icon" href="https://www.debian.org/favicon.ico">
190 <title>
191 Debian NEW and BYHAND Packages
192 </title>
193 <script>
194 function togglePkg() {
195 for (const el of document.getElementsByClassName('sourceNEW')) {
196 el.style.display = el.style.display == '' ? 'none' : '';
197 }
198 }
199 </script>
200 </head>
201 <body id="NEW">
202 <div id="logo">
203 <a href="https://www.debian.org/">
204 <img src="https://www.debian.org/logos/openlogo-nd-50.png"
205 alt=""></a>
206 <a href="https://www.debian.org/">
207 <img src="https://www.debian.org/Pics/debian.png"
208 alt="Debian Project"></a>
209 </div>
210 <div id="titleblock">
211
212 <img src="https://www.debian.org/Pics/red-upperleft.png"
213 id="red-upperleft" alt="">
214 <img src="https://www.debian.org/Pics/red-lowerleft.png"
215 id="red-lowerleft" alt="">
216 <img src="https://www.debian.org/Pics/red-upperright.png"
217 id="red-upperright" alt="">
218 <img src="https://www.debian.org/Pics/red-lowerright.png"
219 id="red-lowerright" alt="">
220 <span class="title">
221 Debian NEW and BYHAND Packages
222 </span>
223 </div>
224 """)
225
226
244
245
247 print("<h1 class='sourceNEW'>Summary for: %s</h1>" % (type))
248 print("<h1 class='sourceNEW' style='display: none'>Summary for: binary-%s only</h1>" % (type))
249 print("""
250 <p class="togglepkg" onclick="togglePkg()">Click to toggle all/binary-NEW packages</p>
251 <table class="NEW">
252 <caption class="sourceNEW">
253 """)
254 print("Package count in <strong>%s</strong>: <em>%s</em> | Total Package count: <em>%s</em>" % (type, source_count, total_count))
255 print("""
256 </caption>
257 <thead>
258 <tr>
259 <th>Package</th>
260 <th>Version</th>
261 <th>Arch</th>
262 <th>Distribution</th>
263 <th>Age</th>
264 <th>Upload info</th>
265 <th>Closes</th>
266 </tr>
267 </thead>
268 <tbody>
269 """)
270
271
274
275
276 -def table_row(source, version, arch, last_mod, maint, distribution, closes, fingerprint, sponsor, changedby):
277 trclass = "sid"
278 session = DBConn().session()
279 for dist in distribution:
280 if dist == "experimental":
281 trclass = "exp"
282
283 query = '''SELECT source
284 FROM source_suite
285 WHERE source = :source
286 AND suite_name IN ('unstable', 'experimental')'''
287 if not session.execute(query, {'source': source}).rowcount:
288 trclass += " sourceNEW"
289 session.commit()
290
291 print("<tr class=\"%s\">" % (trclass))
292
293 if "sourceNEW" in trclass:
294 print("<td class=\"package\">%s</td>" % (source))
295 else:
296 print("<td class=\"package\"><a href=\"https://tracker.debian.org/pkg/%(source)s\">%(source)s</a></td>" % {'source': source})
297 print("<td class=\"version\">")
298 for vers in version.split():
299 print("<a href=\"new/%s_%s.html\">%s</a><br>" % (source, html.escape(vers), html.escape(vers, quote=False)))
300 print("</td>")
301 print("<td class=\"arch\">%s</td>" % (arch))
302 print("<td class=\"distribution\">")
303 for dist in distribution:
304 print("%s<br>" % (dist))
305 print("</td>")
306 print("<td class=\"age\"><abbr title=\"%s\">%s</abbr></td>" % (
307 datetime.datetime.utcfromtimestamp(int(time.time()) - last_mod).strftime('%a, %d %b %Y %T UTC'),
308 time_pp(last_mod),
309 ))
310 (name, mail) = maint.split(":", 1)
311
312 print("<td class=\"upload-data\">")
313 print("<span class=\"maintainer\">Maintainer: <a href=\"https://qa.debian.org/developer.php?login=%s\">%s</a></span><br>" % (html.escape(mail), html.escape(name, quote=False)))
314 (name, mail) = changedby.split(":", 1)
315 print("<span class=\"changed-by\">Changed-By: <a href=\"https://qa.debian.org/developer.php?login=%s\">%s</a></span><br>" % (html.escape(mail), html.escape(name, quote=False)))
316
317 if sponsor:
318 print("<span class=\"sponsor\">Sponsor: <a href=\"https://qa.debian.org/developer.php?login=%s\">%s</a>@debian.org</span><br>" % (html.escape(sponsor), html.escape(sponsor, quote=False)))
319
320 print("<span class=\"signature\">Fingerprint: %s</span>" % (fingerprint))
321 print("</td>")
322
323 print("<td class=\"closes\">")
324 for close in closes:
325 print("<a href=\"https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s\">#%s</a><br>" % (html.escape(close), html.escape(close, quote=False)))
326 print("</td></tr>")
327
328
329
330
332 if not rrd_dir:
333 return
334
335 import rrdtool
336
337 rrd_file = os.path.join(rrd_dir, type.lower() + '.rrd')
338 update = [rrd_file, "N:%s:%s" % (n_source, n_binary)]
339
340 try:
341 rrdtool.update(*update)
342 except rrdtool.error:
343 create = [rrd_file] + """
344 --step
345 300
346 --start
347 0
348 DS:ds0:GAUGE:7200:0:1000
349 DS:ds1:GAUGE:7200:0:1000
350 RRA:AVERAGE:0.5:1:599
351 RRA:AVERAGE:0.5:6:700
352 RRA:AVERAGE:0.5:24:775
353 RRA:AVERAGE:0.5:288:795
354 RRA:MAX:0.5:1:600
355 RRA:MAX:0.5:6:700
356 RRA:MAX:0.5:24:775
357 RRA:MAX:0.5:288:795
358 """.strip().split("\n")
359 try:
360 rrdtool.create(*create)
361 rrdtool.update(*update)
362 except rrdtool.error as e:
363 print(('warning: queue_report: rrdtool error, skipping %s.rrd: %s' % (type, e)))
364 except NameError:
365 pass
366
367
368
369
371 msg = ""
372 type = queue.queue_name
373 session = DBConn().session()
374
375
376 per_source = {}
377 total_pending = 0
378 for upload in queue.uploads:
379 source = upload.changes.source
380 if source not in per_source:
381 per_source[source] = {}
382 per_source[source]["list"] = []
383 per_source[source]["processed"] = ""
384 handler = PolicyQueueUploadHandler(upload, session)
385 if handler.get_action():
386 per_source[source]["processed"] = "PENDING %s" % handler.get_action()
387 total_pending += 1
388 per_source[source]["list"].append(upload)
389 per_source[source]["list"].sort(key=lambda x: x.changes.created, reverse=True)
390
391 for source in list(per_source.keys()):
392 source_list = per_source[source]["list"]
393 first = source_list[0]
394 oldest = time.mktime(first.changes.created.timetuple())
395 have_note = 0
396 for d in per_source[source]["list"]:
397 mtime = time.mktime(d.changes.created.timetuple())
398 if "Queue-Report::Options::New" in Cnf:
399 if mtime > oldest:
400 oldest = mtime
401 else:
402 if mtime < oldest:
403 oldest = mtime
404 have_note += has_new_comment(d.policy_queue, d.changes.source, d.changes.version)
405 per_source[source]["oldest"] = oldest
406 if not have_note:
407 per_source[source]["note_state"] = 0
408 elif have_note < len(source_list):
409 per_source[source]["note_state"] = 1
410 else:
411 per_source[source]["note_state"] = 2
412 per_source_items = list(per_source.items())
413 per_source_items.sort(key=functools.cmp_to_key(sg_compare))
414
415 update_graph_database(rrd_dir, type, len(per_source_items), len(queue.uploads))
416
417 entries = []
418 max_source_len = 0
419 max_version_len = 0
420 max_arch_len = 0
421 try:
422 logins = get_logins_from_ldap()
423 except:
424 logins = dict()
425 for i in per_source_items:
426 maintainer = {}
427 maint = ""
428 distribution = ""
429 closes = ""
430 fingerprint = ""
431 changeby = {}
432 changedby = ""
433 sponsor = ""
434 filename = i[1]["list"][0].changes.changesname
435 last_modified = time.time() - i[1]["oldest"]
436 source = i[1]["list"][0].changes.source
437 if len(source) > max_source_len:
438 max_source_len = len(source)
439 binary_list = i[1]["list"][0].binaries
440 binary = ', '.join([b.package for b in binary_list])
441 arches = set()
442 versions = set()
443 for j in i[1]["list"]:
444 dbc = j.changes
445 changesbase = dbc.changesname
446
447 if "Queue-Report::Options::New" in Cnf or "Queue-Report::Options::822" in Cnf:
448 try:
449 (maintainer["maintainer822"], maintainer["maintainer2047"],
450 maintainer["maintainername"], maintainer["maintaineremail"]) = \
451 fix_maintainer(dbc.maintainer)
452 except ParseMaintError as msg:
453 print("Problems while parsing maintainer address\n")
454 maintainer["maintainername"] = "Unknown"
455 maintainer["maintaineremail"] = "Unknown"
456 maint = "%s:%s" % (maintainer["maintainername"], maintainer["maintaineremail"])
457
458 try:
459 (changeby["changedby822"], changeby["changedby2047"],
460 changeby["changedbyname"], changeby["changedbyemail"]) = \
461 fix_maintainer(dbc.changedby)
462 except ParseMaintError as msg:
463 (changeby["changedby822"], changeby["changedby2047"],
464 changeby["changedbyname"], changeby["changedbyemail"]) = \
465 ("", "", "", "")
466 changedby = "%s:%s" % (changeby["changedbyname"], changeby["changedbyemail"])
467
468 distribution = dbc.distribution.split()
469 closes = dbc.closes
470
471 fingerprint = dbc.fingerprint
472 sponsor_uid = get_uid_from_fingerprint(fingerprint, session)
473 sponsor_name = sponsor_uid.name
474 sponsor_login = sponsor_uid.uid
475 if '@' in sponsor_login:
476 if fingerprint in logins:
477 sponsor_login = logins[fingerprint]
478 if (sponsor_name != maintainer["maintainername"]
479 and sponsor_name != changeby["changedbyname"]
480 and sponsor_login + '@debian.org' != maintainer["maintaineremail"]
481 and sponsor_name != changeby["changedbyemail"]):
482 sponsor = sponsor_login
483
484 for arch in dbc.architecture.split():
485 arches.add(arch)
486 versions.add(dbc.version)
487 arches_list = sorted(arches, key=utils.ArchKey)
488 arch_list = " ".join(arches_list)
489 version_list = " ".join(sorted(versions, reverse=True))
490 if len(version_list) > max_version_len:
491 max_version_len = len(version_list)
492 if len(arch_list) > max_arch_len:
493 max_arch_len = len(arch_list)
494 if i[1]["note_state"]:
495 note = " | [N]"
496 else:
497 note = ""
498 entries.append([source, binary, version_list, arch_list, per_source[source]["processed"], note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, filename])
499
500
501
502
503
504 age = "h"
505 if "Queue-Report::Options::Age" in Cnf:
506 age = Cnf["Queue-Report::Options::Age"]
507 if "Queue-Report::Options::New" in Cnf:
508
509 direction.append([6, -1, "ao"])
510 else:
511 if "Queue-Report::Options::Sort" in Cnf:
512 for i in Cnf["Queue-Report::Options::Sort"].split(","):
513 if i == "ao":
514
515 direction.append([6, -1, age])
516 elif i == "an":
517
518 direction.append([6, 1, age])
519 elif i == "na":
520
521 direction.append([0, 1, 0])
522 elif i == "nd":
523
524 direction.append([0, -1, 0])
525 elif i == "nl":
526
527 direction.append([5, 1, 0])
528 elif i == "nf":
529
530 direction.append([5, -1, 0])
531 entries.sort(key=functools.cmp_to_key(sortfunc))
532
533
534
535
536
537 if "Queue-Report::Options::822" in Cnf:
538
539 for entry in entries:
540 (source, binary, version_list, arch_list, processed, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, changes_file) = entry
541
542
543
544 log.write("Source: " + source + "\n")
545 log.write("Binary: " + binary + "\n")
546 log.write("Version: " + version_list + "\n")
547 log.write("Architectures: ")
548 log.write((", ".join(arch_list.split(" "))) + "\n")
549 log.write("Age: " + time_pp(last_modified) + "\n")
550 log.write("Last-Modified: " + str(int(time.time()) - int(last_modified)) + "\n")
551 log.write("Queue: " + type + "\n")
552
553 (name, mail) = maint.split(":", 1)
554 log.write("Maintainer: " + name + " <" + mail + ">" + "\n")
555 if changedby:
556 (name, mail) = changedby.split(":", 1)
557 log.write("Changed-By: " + name + " <" + mail + ">" + "\n")
558 if sponsor:
559 log.write("Sponsored-By: %s@debian.org\n" % sponsor)
560 log.write("Distribution:")
561 for dist in distribution:
562 log.write(" " + dist)
563 log.write("\n")
564 log.write("Fingerprint: " + fingerprint + "\n")
565 if closes:
566 bug_string = ""
567 for bugs in closes:
568 bug_string += "#" + bugs + ", "
569 log.write("Closes: " + bug_string[:-2] + "\n")
570 log.write("Changes-File: " + os.path.basename(changes_file) + "\n")
571 log.write("\n")
572
573 total_count = len(queue.uploads)
574 source_count = len(per_source_items)
575
576 if "Queue-Report::Options::New" in Cnf:
577 direction.append([6, 1, "ao"])
578 entries.sort(key=functools.cmp_to_key(sortfunc))
579
580
581 if len(entries) > 0:
582 table_header(type.upper(), source_count, total_count)
583 for entry in entries:
584 (source, binary, version_list, arch_list, processed, note, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby, _) = entry
585 table_row(source, version_list, arch_list, last_modified, maint, distribution, closes, fingerprint, sponsor, changedby)
586 table_footer(type.upper())
587 elif "Queue-Report::Options::822" not in Cnf:
588
589 msg = ""
590 for entry in entries:
591 (source, binary, version_list, arch_list, processed, note, last_modified, _, _, _, _, _, _, _) = entry
592 if processed:
593 format = "%%-%ds | %%-%ds | %%-%ds | %%s\n" % (max_source_len, max_version_len, max_arch_len)
594 msg += format % (source, version_list, arch_list, processed)
595 else:
596 format = "%%-%ds | %%-%ds | %%-%ds%%s | %%s old\n" % (max_source_len, max_version_len, max_arch_len)
597 msg += format % (source, version_list, arch_list, note, time_pp(last_modified))
598
599 if msg:
600 print(type.upper())
601 print("-" * len(type))
602 print()
603 print(msg)
604 print(("%s %s source package%s / %s %s package%s in total / %s %s package%s to be processed." %
605 (source_count, type, plural(source_count),
606 total_count, type, plural(total_count),
607 total_pending, type, plural(total_pending))))
608 print()
609
610
611
612
614 global Cnf
615
616 Cnf = utils.get_conf()
617 Arguments = [('h', "help", "Queue-Report::Options::Help"),
618 ('n', "new", "Queue-Report::Options::New"),
619 ('8', '822', "Queue-Report::Options::822"),
620 ('s', "sort", "Queue-Report::Options::Sort", "HasArg"),
621 ('a', "age", "Queue-Report::Options::Age", "HasArg"),
622 ('r', "rrd", "Queue-Report::Options::Rrd", "HasArg"),
623 ('d', "directories", "Queue-Report::Options::Directories", "HasArg")]
624 for i in ["help"]:
625 key = "Queue-Report::Options::%s" % i
626 if key not in Cnf:
627 Cnf[key] = ""
628
629 apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
630
631 Options = Cnf.subtree("Queue-Report::Options")
632 if Options["Help"]:
633 usage()
634
635 if "Queue-Report::Options::New" in Cnf:
636 header()
637
638 queue_names = []
639
640 if "Queue-Report::Options::Directories" in Cnf:
641 for i in Cnf["Queue-Report::Options::Directories"].split(","):
642 queue_names.append(i)
643 elif "Queue-Report::Directories" in Cnf:
644 queue_names = Cnf.value_list("Queue-Report::Directories")
645 else:
646 queue_names = ["byhand", "new"]
647
648 if "Queue-Report::Options::Rrd" in Cnf:
649 rrd_dir = Cnf["Queue-Report::Options::Rrd"]
650 elif "Dir::Rrd" in Cnf:
651 rrd_dir = Cnf["Dir::Rrd"]
652 else:
653 rrd_dir = None
654
655 f = None
656 if "Queue-Report::Options::822" in Cnf:
657
658 f = sys.stdout
659 filename822 = Cnf.get("Queue-Report::ReportLocations::822Location")
660 if filename822:
661 f = open(filename822, "w")
662
663 session = DBConn().session()
664
665 for queue_name in queue_names:
666 queue = session.query(PolicyQueue).filter_by(queue_name=queue_name).first()
667 if queue is not None:
668 process_queue(queue, f, rrd_dir)
669 else:
670 utils.warn("Cannot find queue %s" % queue_name)
671
672 if "Queue-Report::Options::822" in Cnf:
673 f.close()
674
675 if "Queue-Report::Options::New" in Cnf:
676 footer()
677
678
679
680
681 if __name__ == '__main__':
682 main()
683