1
2
3 """
4 Script to automate some parts of checking NEW packages
5
6 Most functions are written in a functional programming style. They
7 return a string avoiding the side effect of directly printing the string
8 to stdout. Those functions can be used in multithreaded parts of dak.
9
10 @contact: Debian FTP Master <ftpmaster@debian.org>
11 @copyright: 2000, 2001, 2002, 2003, 2006 James Troup <james@nocrew.org>
12 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
13 @license: GNU General Public License version 2 or later
14 """
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45 import errno
46 import hashlib
47 import html
48 import os
49 import re
50 import subprocess
51 import sys
52 import tarfile
53 import tempfile
54 import threading
55
56 import apt_pkg
57
58 from daklib import utils
59 from daklib.config import Config
60 from daklib.dbconn import DBConn, get_component_by_package_suite
61 from daklib.gpg import SignedFile
62 from daklib.regexes import (
63 re_contrib,
64 re_file_binary,
65 re_localhost,
66 re_newlinespace,
67 re_nonfree,
68 re_spacestrip,
69 re_version,
70 )
71
72
73
74 Cnf = None
75 Cnf = utils.get_conf()
76
77 printed = threading.local()
78 printed.copyrights = {}
79 package_relations = {}
80
81
82 use_html = False
83
84
85
86
88 print(
89 """Usage: dak examine-package [PACKAGE]...
90 Check NEW package(s).
91
92 -h, --help show this help and exit
93 -H, --html-output output html page with inspection result
94 -f, --file-name filename for the html page
95
96 PACKAGE can be a .changes, .dsc, .deb or .udeb filename."""
97 )
98
99 sys.exit(exit_code)
100
101
102
103
104
105
107 if use_html:
108 return html.escape(s)
109 else:
110 return s
111
112
113 -def headline(s, level=2, bodyelement=None):
114 if use_html:
115 if bodyelement:
116 return """<thead>
117 <tr><th colspan="2" class="title" onclick="toggle('%(bodyelement)s', 'table-row-group', 'table-row-group')">%(title)s <span class="toggle-msg">(click to toggle)</span></th></tr>
118 </thead>\n""" % {
119 "bodyelement": bodyelement,
120 "title": html.escape(os.path.basename(s), quote=False),
121 }
122 else:
123 return "<h%d>%s</h%d>\n" % (level, html.escape(s, quote=False), level)
124 else:
125 return "---- %s ----\n" % (s)
126
127
128
129
130 ansi_colours = {
131 "main": "\033[36m",
132 "contrib": "\033[33m",
133 "nonfree": "\033[31m",
134 "provides": "\033[35m",
135 "arch": "\033[32m",
136 "end": "\033[0m",
137 "bold": "\033[1m",
138 "maintainer": "\033[32m",
139 "distro": "\033[1m\033[41m",
140 "error": "\033[1m\033[41m",
141 }
142
143 html_colours = {
144 "main": ('<span style="color: green">', "</span>"),
145 "contrib": ('<span style="color: orange">', "</span>"),
146 "nonfree": ('<span style="color: red">', "</span>"),
147 "provides": ('<span style="color: magenta">', "</span>"),
148 "arch": ('<span style="color: green">', "</span>"),
149 "bold": ('<span style="font-weight: bold">', "</span>"),
150 "maintainer": ('<span style="color: green">', "</span>"),
151 "distro": ('<span style="font-weight: bold; background-color: red">', "</span>"),
152 "error": ('<span style="font-weight: bold; background-color: red">', "</span>"),
153 }
154
155
165
166
167 -def escaped_text(s, strip=False):
168 if use_html:
169 if strip:
170 s = s.strip()
171 return "<pre>%s</pre>" % (s)
172 else:
173 return s
174
175
177 if use_html:
178 if strip:
179 s = s.strip()
180 return "<pre>%s</pre>" % (html.escape(s, quote=False))
181 else:
182 return s
183
184
186 if use_html:
187 return """<tr><td>""" + s + """</td></tr>"""
188 else:
189 return s
190
191
197
198
200 d = {"elementnameprefix": elementnameprefix}
201 result = ""
202 if use_html:
203 result += (
204 """<div id="%(elementnameprefix)s-wrap"><a name="%(elementnameprefix)s"></a>
205 <table class="infobox rfc822">\n"""
206 % d
207 )
208 result += headline(title, bodyelement="%(elementnameprefix)s-body" % d)
209 if use_html:
210 result += (
211 """ <tbody id="%(elementnameprefix)s-body" class="infobody">\n""" % d
212 )
213 if norow:
214 result += content + "\n"
215 else:
216 result += output_row(content) + "\n"
217 if use_html:
218 result += """</tbody></table></div>"""
219 return result
220
221
222
223
224
226 v_match = re_version.match(depend)
227 if v_match:
228 d_parts = {"name": v_match.group(1), "version": v_match.group(2)}
229 else:
230 d_parts = {"name": depend, "version": ""}
231 return d_parts
232
233
235 or_list = depend.split("|")
236 return or_list
237
238
240 dep_list = depend.split(",")
241 return dep_list
242
243
245
246
247 d_str = re_spacestrip.sub("", d_str)
248 depends_tree = []
249
250 dep_list = get_comma_list(d_str)
251 d = 0
252 while d < len(dep_list):
253
254 depends_tree.append([dep_list[d]])
255 d += 1
256 d = 0
257 while d < len(depends_tree):
258 k = 0
259
260 depends_tree[d] = get_or_list(depends_tree[d][0])
261 while k < len(depends_tree[d]):
262
263 depends_tree[d][k] = get_depends_parts(depends_tree[d][k])
264 k += 1
265 d += 1
266 return depends_tree
267
268
270 recommends = []
271 predepends = []
272 depends = []
273 section = ""
274 maintainer = ""
275 arch = ""
276
277 try:
278 extracts = utils.deb_extract_control(filename)
279 control = apt_pkg.TagSection(extracts)
280 except:
281 print(formatted_text("can't parse control info"))
282 raise
283
284 control_keys = list(control.keys())
285
286 if "Pre-Depends" in control:
287 predepends_str = control["Pre-Depends"]
288 predepends = split_depends(predepends_str)
289
290 if "Depends" in control:
291 depends_str = control["Depends"]
292
293 depends = split_depends(depends_str)
294
295 if "Recommends" in control:
296 recommends_str = control["Recommends"]
297 recommends = split_depends(recommends_str)
298
299 if "Section" in control:
300 section_str = control["Section"]
301
302 c_match = re_contrib.search(section_str)
303 nf_match = re_nonfree.search(section_str)
304 if c_match:
305
306 section = colour_output(section_str, "contrib")
307 elif nf_match:
308
309 section = colour_output(section_str, "nonfree")
310 else:
311
312 section = colour_output(section_str, "main")
313 if "Architecture" in control:
314 arch_str = control["Architecture"]
315 arch = colour_output(arch_str, "arch")
316
317 if "Maintainer" in control:
318 maintainer = control["Maintainer"]
319 localhost = re_localhost.search(maintainer)
320 if localhost:
321
322 maintainer = colour_output(maintainer, "maintainer")
323 else:
324 maintainer = escape_if_needed(maintainer)
325
326 return (
327 control,
328 control_keys,
329 section,
330 predepends,
331 depends,
332 recommends,
333 arch,
334 maintainer,
335 )
336
337
339 dsc = {}
340
341 try:
342 dsc = utils.parse_changes(filename, dsc_file=True)
343 except:
344 return formatted_text("can't parse .dsc control info")
345
346 filecontents = strip_pgp_signature(filename)
347 keysinorder = []
348 for line in filecontents.split("\n"):
349 m = re.match(r"([-a-zA-Z0-9]*):", line)
350 if m:
351 keysinorder.append(m.group(1))
352
353 for k in list(dsc.keys()):
354 if k in ("build-depends", "build-depends-indep"):
355 dsc[k] = create_depends_string(suite, split_depends(dsc[k]), session)
356 elif k == "architecture":
357 if dsc["architecture"] != "any":
358 dsc["architecture"] = colour_output(dsc["architecture"], "arch")
359 elif k == "distribution":
360 if dsc["distribution"] not in ("unstable", "experimental"):
361 dsc["distribution"] = colour_output(dsc["distribution"], "distro")
362 elif k in ("files", "changes", "description"):
363 if use_html:
364 dsc[k] = formatted_text(dsc[k], strip=True)
365 else:
366 dsc[k] = (
367 "\n" + "\n".join(" " + x for x in dsc[k].split("\n"))
368 ).rstrip()
369 else:
370 dsc[k] = escape_if_needed(dsc[k])
371
372 filecontents = (
373 "\n".join(
374 format_field(x, dsc[x.lower()])
375 for x in keysinorder
376 if not x.lower().startswith("checksums-")
377 )
378 + "\n"
379 )
380 return filecontents
381
382
384 provides = set()
385 session = DBConn().session()
386 query = """SELECT DISTINCT value
387 FROM binaries_metadata m
388 JOIN bin_associations b
389 ON b.bin = m.bin_id
390 WHERE key_id = (
391 SELECT key_id
392 FROM metadata_keys
393 WHERE key = 'Provides' )
394 AND b.suite = (
395 SELECT id
396 FROM suite
397 WHERE suite_name = :suite
398 OR codename = :suite)"""
399 for p in session.execute(query, {"suite": suite}):
400 for e in p:
401 for i in e.split(","):
402 provides.add(i.strip())
403 session.close()
404 return provides
405
406
408 result = ""
409 if suite == "experimental":
410 suite_list = ["experimental", "unstable"]
411 else:
412 suite_list = [suite]
413
414 provides = set()
415 comma_count = 1
416 for item in depends_tree:
417 if comma_count >= 2:
418 result += ", "
419 or_count = 1
420 for d in item:
421 if or_count >= 2:
422 result += " | "
423
424
425 component = get_component_by_package_suite(
426 d["name"], suite_list, session=session
427 )
428 if component is not None:
429 adepends = d["name"]
430 if d["version"] != "":
431 adepends += " (%s)" % (d["version"])
432
433 if component == "contrib":
434 result += colour_output(adepends, "contrib")
435 elif component in ("non-free-firmware", "non-free"):
436 result += colour_output(adepends, "nonfree")
437 else:
438 result += colour_output(adepends, "main")
439 else:
440 adepends = d["name"]
441 if d["version"] != "":
442 adepends += " (%s)" % (d["version"])
443 if not provides:
444 provides = get_provides(suite)
445 if d["name"] in provides:
446 result += colour_output(adepends, "provides")
447 else:
448 result += colour_output(adepends, "bold")
449 or_count += 1
450 comma_count += 1
451 return result
452
453
477
478
480 (
481 control,
482 control_keys,
483 section,
484 predepends,
485 depends,
486 recommends,
487 arch,
488 maintainer,
489 ) = read_control(filename)
490
491 if control == "":
492 return formatted_text("no control info")
493 to_print = ""
494 if packagename not in package_relations:
495 package_relations[packagename] = {}
496 for key in control_keys:
497 if key == "Source":
498 field_value = escape_if_needed(control.find(key))
499 if use_html:
500 field_value = '<a href="https://tracker.debian.org/pkg/{0}" rel="nofollow">{0}</a>'.format(
501 field_value
502 )
503 elif key == "Pre-Depends":
504 field_value = create_depends_string(suite, predepends, session)
505 package_relations[packagename][key] = field_value
506 elif key == "Depends":
507 field_value = create_depends_string(suite, depends, session)
508 package_relations[packagename][key] = field_value
509 elif key == "Recommends":
510 field_value = create_depends_string(suite, recommends, session)
511 package_relations[packagename][key] = field_value
512 elif key == "Section":
513 field_value = section
514 elif key == "Architecture":
515 field_value = arch
516 elif key == "Maintainer":
517 field_value = maintainer
518 elif key in ("Homepage", "Vcs-Browser"):
519 field_value = escape_if_needed(control.find(key))
520 if use_html:
521 field_value = '<a href="%s" rel="nofollow">%s</a>' % (
522 field_value,
523 field_value,
524 )
525 elif key == "Description":
526 if use_html:
527 field_value = formatted_text(control.find(key), strip=True)
528 else:
529 desc = control.find(key)
530 desc = re_newlinespace.sub("\n ", desc)
531 field_value = escape_if_needed(desc)
532 else:
533 field_value = escape_if_needed(control.find(key))
534 to_print += " " + format_field(key, field_value) + "\n"
535 return to_print
536
537
539 result = subprocess.run(command, stdout=subprocess.PIPE, text=True)
540 if escaped:
541 return escaped_text(result.stdout)
542 else:
543 return formatted_text(result.stdout)
544
545
547 cnf = Config()
548 cmd = []
549
550 user = cnf.get("Dinstall::UnprivUser") or None
551 if user is not None:
552 cmd.extend(["sudo", "-H", "-u", user])
553
554 color = "always"
555 if use_html:
556 color = "html"
557
558 cmd.extend(["lintian", "--show-overrides", "--color", color, "--", filename])
559
560 try:
561 return do_command(cmd, escaped=True)
562 except OSError as e:
563 return colour_output("Running lintian failed: %s" % (e), "error")
564
565
567 with tempfile.TemporaryFile() as tmpfh:
568 dpkg_cmd = ("dpkg-deb", "--fsys-tarfile", deb_filename)
569 subprocess.check_call(dpkg_cmd, stdout=tmpfh)
570
571 tmpfh.seek(0)
572 with tarfile.open(fileobj=tmpfh, mode="r") as tar:
573 matched_member = None
574 for member in tar:
575 if member.isfile() and match.match(member.name):
576 matched_member = member
577 break
578
579 if not matched_member:
580 return None, None
581
582 fh = tar.extractfile(matched_member)
583 matched_data = fh.read()
584 fh.close()
585
586 return matched_member.name, matched_data
587
588
590 global printed
591
592 re_copyright = re.compile(r"\./usr(/share)?/doc/(?P<package>[^/]+)/copyright")
593 cright_path, cright = extract_one_file_from_deb(deb_filename, re_copyright)
594
595 if not cright_path:
596 return formatted_text(
597 "WARNING: No copyright found, please check package manually."
598 )
599
600 package = re_file_binary.match(os.path.basename(deb_filename)).group("package")
601 doc_directory = re_copyright.match(cright_path).group("package")
602 if package != doc_directory:
603 return formatted_text(
604 "WARNING: wrong doc directory (expected %s, got %s)."
605 % (package, doc_directory)
606 )
607
608 copyrightmd5 = hashlib.md5(cright).hexdigest()
609
610 res = ""
611 if copyrightmd5 in printed.copyrights and printed.copyrights[
612 copyrightmd5
613 ] != "%s (%s)" % (package, os.path.basename(deb_filename)):
614 res += formatted_text(
615 "NOTE: Copyright is the same as %s.\n\n"
616 % (printed.copyrights[copyrightmd5])
617 )
618 else:
619 printed.copyrights[copyrightmd5] = "%s (%s)" % (
620 package,
621 os.path.basename(deb_filename),
622 )
623 return res + formatted_text(cright.decode())
624
625
627 with tempfile.TemporaryDirectory(prefix="dak-examine-package") as tempdir:
628 targetdir = os.path.join(tempdir, "source")
629
630 cmd = ("dpkg-source", "--no-check", "--no-copy", "-x", dsc_filename, targetdir)
631 try:
632 subprocess.check_output(cmd, stderr=subprocess.STDOUT)
633 except subprocess.CalledProcessError as e:
634 res = "How is education supposed to make me feel smarter? Besides, every time I learn something new, it pushes some\n old stuff out of my brain. Remember when I took that home winemaking course, and I forgot how to drive?\n"
635 res += "Error, couldn't extract source, WTF?\n"
636 res += "'dpkg-source -x' failed. return code: %s.\n\n" % (e.returncode)
637 res += e.output
638 return res
639
640 path = os.path.join(targetdir, "debian/README.source")
641 res = ""
642 if os.path.exists(path):
643 with open(path, "r") as fh:
644 res += formatted_text(fh.read())
645 else:
646 res += "No README.source in this package\n\n"
647
648 return res
649
650
651 -def check_dsc(suite, dsc_filename, session=None):
652 dsc = read_changes_or_dsc(suite, dsc_filename, session)
653 dsc_basename = os.path.basename(dsc_filename)
654 cdsc = (
655 foldable_output(dsc_filename, "dsc", dsc, norow=True)
656 + "\n"
657 + foldable_output(
658 "lintian {} check for {}".format(get_lintian_version(), dsc_basename),
659 "source-lintian",
660 do_lintian(dsc_filename),
661 )
662 + "\n"
663 + foldable_output(
664 "README.source for %s" % dsc_basename,
665 "source-readmesource",
666 get_readme_source(dsc_filename),
667 )
668 )
669 return cdsc
670
671
672 -def check_deb(suite, deb_filename, session=None):
673 filename = os.path.basename(deb_filename)
674 packagename = filename.split("_")[0]
675
676 if filename.endswith(".udeb"):
677 is_a_udeb = 1
678 else:
679 is_a_udeb = 0
680
681 result = (
682 foldable_output(
683 "control file for %s" % (filename),
684 "binary-%s-control" % packagename,
685 output_deb_info(suite, deb_filename, packagename, session),
686 norow=True,
687 )
688 + "\n"
689 )
690
691 if is_a_udeb:
692 result += (
693 foldable_output(
694 "skipping lintian check for udeb", "binary-%s-lintian" % packagename, ""
695 )
696 + "\n"
697 )
698 else:
699 result += (
700 foldable_output(
701 "lintian {} check for {}".format(get_lintian_version(), filename),
702 "binary-%s-lintian" % packagename,
703 do_lintian(deb_filename),
704 )
705 + "\n"
706 )
707
708 result += (
709 foldable_output(
710 "contents of %s" % (filename),
711 "binary-%s-contents" % packagename,
712 do_command(["dpkg", "-c", deb_filename]),
713 )
714 + "\n"
715 )
716
717 if is_a_udeb:
718 result += (
719 foldable_output(
720 "skipping copyright for udeb", "binary-%s-copyright" % packagename, ""
721 )
722 + "\n"
723 )
724 else:
725 result += (
726 foldable_output(
727 "copyright of %s" % (filename),
728 "binary-%s-copyright" % packagename,
729 get_copyright(deb_filename),
730 )
731 + "\n"
732 )
733
734 return result
735
736
737
738
739
740
742 with open(filename, "rb") as f:
743 data = f.read()
744 signedfile = SignedFile(data, keyrings=(), require_signature=False)
745 return signedfile.contents.decode()
746
747
753
754
756 try:
757 changes = utils.parse_changes(changes_filename)
758 except UnicodeDecodeError:
759 utils.warn("Encoding problem with changes file %s" % (changes_filename))
760 output = display_changes(changes["distribution"], changes_filename)
761
762 files = utils.build_file_list(changes)
763 for f in files.keys():
764 if f.endswith(".deb") or f.endswith(".udeb"):
765 output += check_deb(changes["distribution"], f)
766 if f.endswith(".dsc"):
767 output += check_dsc(changes["distribution"], f)
768
769 return output
770
771
773 global Cnf, db_files, waste, excluded
774
775
776
777 Arguments = [
778 ("h", "help", "Examine-Package::Options::Help"),
779 ("H", "html-output", "Examine-Package::Options::Html-Output"),
780 ]
781 for i in ["Help", "Html-Output", "partial-html"]:
782 key = "Examine-Package::Options::%s" % i
783 if key not in Cnf:
784 Cnf[key] = ""
785
786 args = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
787 Options = Cnf.subtree("Examine-Package::Options")
788
789 if Options["Help"]:
790 usage()
791
792 if Options["Html-Output"]:
793 global use_html
794 use_html = True
795
796 for f in args:
797 try:
798 if not Options["Html-Output"]:
799
800 less_cmd = ("less", "-r", "-")
801 less_process = subprocess.Popen(
802 less_cmd, stdin=subprocess.PIPE, bufsize=0, text=True
803 )
804 less_fd = less_process.stdin
805
806 my_fd = less_fd
807 else:
808 my_fd = sys.stdout
809
810 try:
811 if f.endswith(".changes"):
812 my_fd.write(check_changes(f))
813 elif f.endswith(".deb") or f.endswith(".udeb"):
814
815
816 my_fd.write(check_deb("unstable", f))
817 elif f.endswith(".dsc"):
818 my_fd.write(check_dsc("unstable", f))
819 else:
820 utils.fubar("Unrecognised file type: '%s'." % (f))
821 finally:
822 my_fd.write(output_package_relations())
823 if not Options["Html-Output"]:
824
825 less_fd.close()
826 less_process.wait()
827 except OSError as e:
828 if e.errno == errno.EPIPE:
829 utils.warn("[examine-package] Caught EPIPE; skipping.")
830 pass
831 else:
832 raise
833 except KeyboardInterrupt:
834 utils.warn("[examine-package] Caught C-c; skipping.")
835 pass
836
837
845
846
847
848
849 if __name__ == "__main__":
850 main()
851