1
2
3 """Produces a report on NEW and BYHAND packages"""
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37 import datetime
38 import functools
39 import html
40 import os
41 import sys
42 import time
43
44 import apt_pkg
45
46 from daklib import utils
47 from daklib.dak_exceptions import ParseMaintError
48 from daklib.dbconn import DBConn, PolicyQueue, get_uid_from_fingerprint, has_new_comment
49 from daklib.policy import PolicyQueueUploadHandler
50 from daklib.textutils import fix_maintainer
51 from daklib.utils import get_logins_from_ldap
52
53 Cnf = None
54 direction = []
55
56
57
58
60 print(
61 """Usage: dak queue-report
62 Prints a report of packages in queues (usually new and byhand).
63
64 -h, --help show this help and exit.
65 -8, --822 writes 822 formated output to the location set in dak.conf
66 -n, --new produce html-output
67 -s, --sort=key sort output according to key, see below.
68 -a, --age=key if using sort by age, how should time be treated?
69 If not given a default of hours will be used.
70 -r, --rrd=key Directory where rrd files to be updated are stored
71 -d, --directories=key A comma separated list of queues to be scanned
72
73 Sorting Keys: ao=age, oldest first. an=age, newest first.
74 na=name, ascending nd=name, descending
75 nf=notes, first nl=notes, last
76
77 Age Keys: m=minutes, h=hours, d=days, w=weeks, o=months, y=years
78
79 """
80 )
81 sys.exit(exit_code)
82
83
84
85
86
88 if x > 1:
89 return "s"
90 else:
91 return ""
92
93
94
95
96
98 if x < 60:
99 unit = "second"
100 elif x < 3600:
101 x /= 60
102 unit = "minute"
103 elif x < 86400:
104 x /= 3600
105 unit = "hour"
106 elif x < 604800:
107 x /= 86400
108 unit = "day"
109 elif x < 2419200:
110 x /= 604800
111 unit = "week"
112 elif x < 29030400:
113 x /= 2419200
114 unit = "month"
115 else:
116 x /= 29030400
117 unit = "year"
118 x = int(x)
119 return "%s %s%s" % (x, unit, plural(x))
120
121
122
123
124
126 a = a[1]
127 b = b[1]
128
129
130 a_note_state = a["processed"]
131 b_note_state = b["processed"]
132 if a_note_state < b_note_state:
133 return -1
134 elif a_note_state > b_note_state:
135 return 1
136
137
138 a_note_state = a["note_state"]
139 b_note_state = b["note_state"]
140 if a_note_state < b_note_state:
141 return -1
142 elif a_note_state > b_note_state:
143 return 1
144
145
146 return a["oldest"] - b["oldest"]
147
148
149
150
151
153 for sorting in direction:
154 (sortkey, way, time) = sorting
155 ret = 0
156 if time == "m":
157 x = int(a[sortkey] / 60)
158 y = int(b[sortkey] / 60)
159 elif time == "h":
160 x = int(a[sortkey] / 3600)
161 y = int(b[sortkey] / 3600)
162 elif time == "d":
163 x = int(a[sortkey] / 86400)
164 y = int(b[sortkey] / 86400)
165 elif time == "w":
166 x = int(a[sortkey] / 604800)
167 y = int(b[sortkey] / 604800)
168 elif time == "o":
169 x = int(a[sortkey] / 2419200)
170 y = int(b[sortkey] / 2419200)
171 elif time == "y":
172 x = int(a[sortkey] / 29030400)
173 y = int(b[sortkey] / 29030400)
174 else:
175 x = a[sortkey]
176 y = b[sortkey]
177 if x < y:
178 ret = -1
179 elif x > y:
180 ret = 1
181 if ret != 0:
182 if way < 0:
183 ret = ret * -1
184 return ret
185 return 0
186
187
188
189
190
192 print(
193 """<!DOCTYPE html>
194 <html lang="en">
195 <head>
196 <meta charset="utf-8">
197 <link rel="stylesheet" href="style.css">
198 <link rel="shortcut icon" href="https://www.debian.org/favicon.ico">
199 <title>
200 Debian NEW and BYHAND Packages
201 </title>
202 <script>
203 function togglePkg() {
204 for (const el of document.getElementsByClassName('sourceNEW')) {
205 el.style.display = el.style.display == '' ? 'none' : '';
206 }
207 }
208 </script>
209 </head>
210 <body id="NEW">
211 <div id="logo">
212 <a href="https://www.debian.org/">
213 <img src="https://www.debian.org/logos/openlogo-nd-50.png"
214 alt=""></a>
215 <a href="https://www.debian.org/">
216 <img src="https://www.debian.org/Pics/debian.png"
217 alt="Debian Project"></a>
218 </div>
219 <div id="titleblock">
220
221 <img src="https://www.debian.org/Pics/red-upperleft.png"
222 id="red-upperleft" alt="">
223 <img src="https://www.debian.org/Pics/red-lowerleft.png"
224 id="red-lowerleft" alt="">
225 <img src="https://www.debian.org/Pics/red-upperright.png"
226 id="red-upperright" alt="">
227 <img src="https://www.debian.org/Pics/red-lowerright.png"
228 id="red-lowerright" alt="">
229 <span class="title">
230 Debian NEW and BYHAND Packages
231 </span>
232 </div>
233 """
234 )
235
236
261
262
264 print("<h1 class='sourceNEW'>Summary for: %s</h1>" % (type))
265 print(
266 "<h1 class='sourceNEW' style='display: none'>Summary for: binary-%s only</h1>"
267 % (type)
268 )
269 print(
270 """
271 <p class="togglepkg" onclick="togglePkg()">Click to toggle all/binary-NEW packages</p>
272 <table class="NEW">
273 <caption class="sourceNEW">
274 """
275 )
276 print(
277 "Package count in <strong>%s</strong>: <em>%s</em> | Total Package count: <em>%s</em>"
278 % (type, source_count, total_count)
279 )
280 print(
281 """
282 </caption>
283 <thead>
284 <tr>
285 <th>Package</th>
286 <th>Version</th>
287 <th>Arch</th>
288 <th>Distribution</th>
289 <th>Age</th>
290 <th>Upload info</th>
291 <th>Closes</th>
292 </tr>
293 </thead>
294 <tbody>
295 """
296 )
297
298
301
302
303 -def table_row(
304 source,
305 version,
306 arch,
307 last_mod,
308 maint,
309 distribution,
310 closes,
311 fingerprint,
312 sponsor,
313 changedby,
314 ):
315 trclass = "sid"
316 session = DBConn().session()
317 for dist in distribution:
318 if dist == "experimental":
319 trclass = "exp"
320
321 query = """SELECT source
322 FROM source_suite
323 WHERE source = :source
324 AND suite_name IN ('unstable', 'experimental')"""
325 if not session.execute(query, {"source": source}).rowcount:
326 trclass += " sourceNEW"
327 session.commit()
328
329 print('<tr class="%s">' % (trclass))
330
331 if "sourceNEW" in trclass:
332 print('<td class="package">%s</td>' % (source))
333 else:
334 print(
335 '<td class="package"><a href="https://tracker.debian.org/pkg/%(source)s">%(source)s</a></td>'
336 % {"source": source}
337 )
338 print('<td class="version">')
339 for vers in version.split():
340 print(
341 '<a href="new/%s_%s.html">%s</a><br>'
342 % (source, html.escape(vers), html.escape(vers, quote=False))
343 )
344 print("</td>")
345 print('<td class="arch">%s</td>' % (arch))
346 print('<td class="distribution">')
347 for dist in distribution:
348 print("%s<br>" % (dist))
349 print("</td>")
350 print(
351 '<td class="age"><abbr title="%s">%s</abbr></td>'
352 % (
353 datetime.datetime.utcfromtimestamp(int(time.time()) - last_mod).strftime(
354 "%a, %d %b %Y %T UTC"
355 ),
356 time_pp(last_mod),
357 )
358 )
359 (name, mail) = maint.split(":", 1)
360
361 print('<td class="upload-data">')
362 print(
363 '<span class="maintainer">Maintainer: <a href="https://qa.debian.org/developer.php?login=%s">%s</a></span><br>'
364 % (html.escape(mail), html.escape(name, quote=False))
365 )
366 (name, mail) = changedby.split(":", 1)
367 print(
368 '<span class="changed-by">Changed-By: <a href="https://qa.debian.org/developer.php?login=%s">%s</a></span><br>'
369 % (html.escape(mail), html.escape(name, quote=False))
370 )
371
372 if sponsor:
373 print(
374 '<span class="sponsor">Sponsor: <a href="https://qa.debian.org/developer.php?login=%s">%s</a>@debian.org</span><br>'
375 % (html.escape(sponsor), html.escape(sponsor, quote=False))
376 )
377
378 print('<span class="signature">Fingerprint: %s</span>' % (fingerprint))
379 print("</td>")
380
381 print('<td class="closes">')
382 for close in closes:
383 print(
384 '<a href="https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s">#%s</a><br>'
385 % (html.escape(close), html.escape(close, quote=False))
386 )
387 print("</td></tr>")
388
389
390
391
392
394 if not rrd_dir:
395 return
396
397 import rrdtool
398
399 rrd_file = os.path.join(rrd_dir, type.lower() + ".rrd")
400 update = [rrd_file, "N:%s:%s" % (n_source, n_binary)]
401
402 try:
403 rrdtool.update(*update)
404 except rrdtool.error:
405 create = (
406 [rrd_file]
407 + """
408 --step
409 300
410 --start
411 0
412 DS:ds0:GAUGE:7200:0:1000
413 DS:ds1:GAUGE:7200:0:1000
414 RRA:AVERAGE:0.5:1:599
415 RRA:AVERAGE:0.5:6:700
416 RRA:AVERAGE:0.5:24:775
417 RRA:AVERAGE:0.5:288:795
418 RRA:MAX:0.5:1:600
419 RRA:MAX:0.5:6:700
420 RRA:MAX:0.5:24:775
421 RRA:MAX:0.5:288:795
422 """.strip().split(
423 "\n"
424 )
425 )
426 try:
427 rrdtool.create(*create)
428 rrdtool.update(*update)
429 except rrdtool.error as e:
430 print(
431 (
432 "warning: queue_report: rrdtool error, skipping %s.rrd: %s"
433 % (type, e)
434 )
435 )
436 except NameError:
437 pass
438
439
440
441
442
444 msg = ""
445 type = queue.queue_name
446 session = DBConn().session()
447
448
449 per_source = {}
450 total_pending = 0
451 for upload in queue.uploads:
452 source = upload.changes.source
453 if source not in per_source:
454 per_source[source] = {}
455 per_source[source]["list"] = []
456 per_source[source]["processed"] = ""
457 handler = PolicyQueueUploadHandler(upload, session)
458 if handler.get_action():
459 per_source[source]["processed"] = "PENDING %s" % handler.get_action()
460 total_pending += 1
461 per_source[source]["list"].append(upload)
462 per_source[source]["list"].sort(key=lambda x: x.changes.created, reverse=True)
463
464 for source in list(per_source.keys()):
465 source_list = per_source[source]["list"]
466 first = source_list[0]
467 oldest = time.mktime(first.changes.created.timetuple())
468 have_note = 0
469 for d in per_source[source]["list"]:
470 mtime = time.mktime(d.changes.created.timetuple())
471 if "Queue-Report::Options::New" in Cnf:
472 if mtime > oldest:
473 oldest = mtime
474 else:
475 if mtime < oldest:
476 oldest = mtime
477 have_note += has_new_comment(
478 d.policy_queue, d.changes.source, d.changes.version
479 )
480 per_source[source]["oldest"] = oldest
481 if not have_note:
482 per_source[source]["note_state"] = 0
483 elif have_note < len(source_list):
484 per_source[source]["note_state"] = 1
485 else:
486 per_source[source]["note_state"] = 2
487 per_source_items = list(per_source.items())
488 per_source_items.sort(key=functools.cmp_to_key(sg_compare))
489
490 update_graph_database(rrd_dir, type, len(per_source_items), len(queue.uploads))
491
492 entries = []
493 max_source_len = 0
494 max_version_len = 0
495 max_arch_len = 0
496 try:
497 logins = get_logins_from_ldap()
498 except:
499 logins = dict()
500 for i in per_source_items:
501 maintainer = {}
502 maint = ""
503 distribution = ""
504 closes = ""
505 fingerprint = ""
506 changeby = {}
507 changedby = ""
508 sponsor = ""
509 filename = i[1]["list"][0].changes.changesname
510 last_modified = time.time() - i[1]["oldest"]
511 source = i[1]["list"][0].changes.source
512 if len(source) > max_source_len:
513 max_source_len = len(source)
514 binary_list = i[1]["list"][0].binaries
515 binary = ", ".join([b.package for b in binary_list])
516 arches = set()
517 versions = set()
518 for j in i[1]["list"]:
519 dbc = j.changes
520
521 if (
522 "Queue-Report::Options::New" in Cnf
523 or "Queue-Report::Options::822" in Cnf
524 ):
525 try:
526 (
527 maintainer["maintainer822"],
528 maintainer["maintainer2047"],
529 maintainer["maintainername"],
530 maintainer["maintaineremail"],
531 ) = fix_maintainer(dbc.maintainer)
532 except ParseMaintError:
533 print("Problems while parsing maintainer address\n")
534 maintainer["maintainername"] = "Unknown"
535 maintainer["maintaineremail"] = "Unknown"
536 maint = "%s:%s" % (
537 maintainer["maintainername"],
538 maintainer["maintaineremail"],
539 )
540
541 try:
542 (
543 changeby["changedby822"],
544 changeby["changedby2047"],
545 changeby["changedbyname"],
546 changeby["changedbyemail"],
547 ) = fix_maintainer(dbc.changedby)
548 except ParseMaintError:
549 (
550 changeby["changedby822"],
551 changeby["changedby2047"],
552 changeby["changedbyname"],
553 changeby["changedbyemail"],
554 ) = ("", "", "", "")
555 changedby = "%s:%s" % (
556 changeby["changedbyname"],
557 changeby["changedbyemail"],
558 )
559
560 distribution = dbc.distribution.split()
561 closes = dbc.closes
562
563 fingerprint = dbc.fingerprint
564 sponsor_uid = get_uid_from_fingerprint(fingerprint, session)
565 sponsor_name = sponsor_uid.name
566 sponsor_login = sponsor_uid.uid
567 if "@" in sponsor_login:
568 if fingerprint in logins:
569 sponsor_login = logins[fingerprint]
570 if (
571 sponsor_name != maintainer["maintainername"]
572 and sponsor_name != changeby["changedbyname"]
573 and sponsor_login + "@debian.org" != maintainer["maintaineremail"]
574 and sponsor_name != changeby["changedbyemail"]
575 ):
576 sponsor = sponsor_login
577
578 for arch in dbc.architecture.split():
579 arches.add(arch)
580 versions.add(dbc.version)
581 arches_list = sorted(arches, key=utils.ArchKey)
582 arch_list = " ".join(arches_list)
583 version_list = " ".join(sorted(versions, reverse=True))
584 if len(version_list) > max_version_len:
585 max_version_len = len(version_list)
586 if len(arch_list) > max_arch_len:
587 max_arch_len = len(arch_list)
588 if i[1]["note_state"]:
589 note = " | [N]"
590 else:
591 note = ""
592 entries.append(
593 [
594 source,
595 binary,
596 version_list,
597 arch_list,
598 per_source[source]["processed"],
599 note,
600 last_modified,
601 maint,
602 distribution,
603 closes,
604 fingerprint,
605 sponsor,
606 changedby,
607 filename,
608 ]
609 )
610
611
612
613
614
615 age = "h"
616 if "Queue-Report::Options::Age" in Cnf:
617 age = Cnf["Queue-Report::Options::Age"]
618 if "Queue-Report::Options::New" in Cnf:
619
620 direction.append([6, -1, "ao"])
621 else:
622 if "Queue-Report::Options::Sort" in Cnf:
623 for i in Cnf["Queue-Report::Options::Sort"].split(","):
624 if i == "ao":
625
626 direction.append([6, -1, age])
627 elif i == "an":
628
629 direction.append([6, 1, age])
630 elif i == "na":
631
632 direction.append([0, 1, 0])
633 elif i == "nd":
634
635 direction.append([0, -1, 0])
636 elif i == "nl":
637
638 direction.append([5, 1, 0])
639 elif i == "nf":
640
641 direction.append([5, -1, 0])
642 entries.sort(key=functools.cmp_to_key(sortfunc))
643
644
645
646
647
648 if "Queue-Report::Options::822" in Cnf:
649
650 for entry in entries:
651 (
652 source,
653 binary,
654 version_list,
655 arch_list,
656 processed,
657 note,
658 last_modified,
659 maint,
660 distribution,
661 closes,
662 fingerprint,
663 sponsor,
664 changedby,
665 changes_file,
666 ) = entry
667
668
669
670 log.write("Source: " + source + "\n")
671 log.write("Binary: " + binary + "\n")
672 log.write("Version: " + version_list + "\n")
673 log.write("Architectures: ")
674 log.write((", ".join(arch_list.split(" "))) + "\n")
675 log.write("Age: " + time_pp(last_modified) + "\n")
676 log.write(
677 "Last-Modified: " + str(int(time.time()) - int(last_modified)) + "\n"
678 )
679 log.write("Queue: " + type + "\n")
680
681 (name, mail) = maint.split(":", 1)
682 log.write("Maintainer: " + name + " <" + mail + ">" + "\n")
683 if changedby:
684 (name, mail) = changedby.split(":", 1)
685 log.write("Changed-By: " + name + " <" + mail + ">" + "\n")
686 if sponsor:
687 log.write("Sponsored-By: %s@debian.org\n" % sponsor)
688 log.write("Distribution:")
689 for dist in distribution:
690 log.write(" " + dist)
691 log.write("\n")
692 log.write("Fingerprint: " + fingerprint + "\n")
693 if closes:
694 bug_string = ""
695 for bugs in closes:
696 bug_string += "#" + bugs + ", "
697 log.write("Closes: " + bug_string[:-2] + "\n")
698 log.write("Changes-File: " + os.path.basename(changes_file) + "\n")
699 log.write("\n")
700
701 total_count = len(queue.uploads)
702 source_count = len(per_source_items)
703
704 if "Queue-Report::Options::New" in Cnf:
705 direction.append([6, 1, "ao"])
706 entries.sort(key=functools.cmp_to_key(sortfunc))
707
708
709 if len(entries) > 0:
710 table_header(type.upper(), source_count, total_count)
711 for entry in entries:
712 (
713 source,
714 binary,
715 version_list,
716 arch_list,
717 processed,
718 note,
719 last_modified,
720 maint,
721 distribution,
722 closes,
723 fingerprint,
724 sponsor,
725 changedby,
726 _,
727 ) = entry
728 table_row(
729 source,
730 version_list,
731 arch_list,
732 last_modified,
733 maint,
734 distribution,
735 closes,
736 fingerprint,
737 sponsor,
738 changedby,
739 )
740 table_footer(type.upper())
741 elif "Queue-Report::Options::822" not in Cnf:
742
743 msg = ""
744 for entry in entries:
745 (
746 source,
747 binary,
748 version_list,
749 arch_list,
750 processed,
751 note,
752 last_modified,
753 _,
754 _,
755 _,
756 _,
757 _,
758 _,
759 _,
760 ) = entry
761 if processed:
762 format = "%%-%ds | %%-%ds | %%-%ds | %%s\n" % (
763 max_source_len,
764 max_version_len,
765 max_arch_len,
766 )
767 msg += format % (source, version_list, arch_list, processed)
768 else:
769 format = "%%-%ds | %%-%ds | %%-%ds%%s | %%s old\n" % (
770 max_source_len,
771 max_version_len,
772 max_arch_len,
773 )
774 msg += format % (
775 source,
776 version_list,
777 arch_list,
778 note,
779 time_pp(last_modified),
780 )
781
782 if msg:
783 print(type.upper())
784 print("-" * len(type))
785 print()
786 print(msg)
787 print(
788 (
789 "%s %s source package%s / %s %s package%s in total / %s %s package%s to be processed."
790 % (
791 source_count,
792 type,
793 plural(source_count),
794 total_count,
795 type,
796 plural(total_count),
797 total_pending,
798 type,
799 plural(total_pending),
800 )
801 )
802 )
803 print()
804
805
806
807
808
810 global Cnf
811
812 Cnf = utils.get_conf()
813 Arguments = [
814 ("h", "help", "Queue-Report::Options::Help"),
815 ("n", "new", "Queue-Report::Options::New"),
816 ("8", "822", "Queue-Report::Options::822"),
817 ("s", "sort", "Queue-Report::Options::Sort", "HasArg"),
818 ("a", "age", "Queue-Report::Options::Age", "HasArg"),
819 ("r", "rrd", "Queue-Report::Options::Rrd", "HasArg"),
820 ("d", "directories", "Queue-Report::Options::Directories", "HasArg"),
821 ]
822 for i in ["help"]:
823 key = "Queue-Report::Options::%s" % i
824 if key not in Cnf:
825 Cnf[key] = ""
826
827 apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
828
829 Options = Cnf.subtree("Queue-Report::Options")
830 if Options["Help"]:
831 usage()
832
833 if "Queue-Report::Options::New" in Cnf:
834 header()
835
836 queue_names = []
837
838 if "Queue-Report::Options::Directories" in Cnf:
839 for i in Cnf["Queue-Report::Options::Directories"].split(","):
840 queue_names.append(i)
841 elif "Queue-Report::Directories" in Cnf:
842 queue_names = Cnf.value_list("Queue-Report::Directories")
843 else:
844 queue_names = ["byhand", "new"]
845
846 if "Queue-Report::Options::Rrd" in Cnf:
847 rrd_dir = Cnf["Queue-Report::Options::Rrd"]
848 elif "Dir::Rrd" in Cnf:
849 rrd_dir = Cnf["Dir::Rrd"]
850 else:
851 rrd_dir = None
852
853 f = None
854 if "Queue-Report::Options::822" in Cnf:
855
856 f = sys.stdout
857 filename822 = Cnf.get("Queue-Report::ReportLocations::822Location")
858 if filename822:
859 f = open(filename822, "w")
860
861 session = DBConn().session()
862
863 for queue_name in queue_names:
864 queue = session.query(PolicyQueue).filter_by(queue_name=queue_name).first()
865 if queue is not None:
866 process_queue(queue, f, rrd_dir)
867 else:
868 utils.warn("Cannot find queue %s" % queue_name)
869
870 if "Queue-Report::Options::822" in Cnf:
871 f.close()
872
873 if "Queue-Report::Options::New" in Cnf:
874 footer()
875
876
877
878
879
880 if __name__ == "__main__":
881 main()
882