1
2
3 """
4 Script to automate some parts of checking NEW packages
5
6 Most functions are written in a functional programming style. They
7 return a string avoiding the side effect of directly printing the string
8 to stdout. Those functions can be used in multithreaded parts of dak.
9
10 @contact: Debian FTP Master <ftpmaster@debian.org>
11 @copyright: 2000, 2001, 2002, 2003, 2006 James Troup <james@nocrew.org>
12 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
13 @license: GNU General Public License version 2 or later
14 """
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45 import errno
46 import hashlib
47 import html
48 import os
49 import re
50 import sys
51 import apt_pkg
52 import shutil
53 import subprocess
54 import tarfile
55 import tempfile
56 import threading
57
58 from daklib import utils
59 from daklib.config import Config
60 from daklib.dbconn import DBConn, get_component_by_package_suite
61 from daklib.gpg import SignedFile
62 from daklib.regexes import re_version, re_spacestrip, \
63 re_contrib, re_nonfree, re_localhost, re_newlinespace, \
64 re_package, re_doc_directory, re_file_binary
65
66
67
68 Cnf = None
69 Cnf = utils.get_conf()
70
71 printed = threading.local()
72 printed.copyrights = {}
73 package_relations = {}
74
75
76 use_html = False
77
78
79
80
82 print("""Usage: dak examine-package [PACKAGE]...
83 Check NEW package(s).
84
85 -h, --help show this help and exit
86 -H, --html-output output html page with inspection result
87 -f, --file-name filename for the html page
88
89 PACKAGE can be a .changes, .dsc, .deb or .udeb filename.""")
90
91 sys.exit(exit_code)
92
93
94
95
96
98 if use_html:
99 return html.escape(s)
100 else:
101 return s
102
103
104 -def headline(s, level=2, bodyelement=None):
105 if use_html:
106 if bodyelement:
107 return """<thead>
108 <tr><th colspan="2" class="title" onclick="toggle('%(bodyelement)s', 'table-row-group', 'table-row-group')">%(title)s <span class="toggle-msg">(click to toggle)</span></th></tr>
109 </thead>\n""" % {"bodyelement": bodyelement, "title": html.escape(os.path.basename(s), quote=False)}
110 else:
111 return "<h%d>%s</h%d>\n" % (level, html.escape(s, quote=False), level)
112 else:
113 return "---- %s ----\n" % (s)
114
115
116
117
118 ansi_colours = {
119 'main': "\033[36m",
120 'contrib': "\033[33m",
121 'nonfree': "\033[31m",
122 'provides': "\033[35m",
123 'arch': "\033[32m",
124 'end': "\033[0m",
125 'bold': "\033[1m",
126 'maintainer': "\033[32m",
127 'distro': "\033[1m\033[41m",
128 'error': "\033[1m\033[41m",
129 }
130
131 html_colours = {
132 'main': ('<span style="color: green">', "</span>"),
133 'contrib': ('<span style="color: orange">', "</span>"),
134 'nonfree': ('<span style="color: red">', "</span>"),
135 'provides': ('<span style="color: magenta">', "</span>"),
136 'arch': ('<span style="color: green">', "</span>"),
137 'bold': ('<span style="font-weight: bold">', "</span>"),
138 'maintainer': ('<span style="color: green">', "</span>"),
139 'distro': ('<span style="font-weight: bold; background-color: red">', "</span>"),
140 'error': ('<span style="font-weight: bold; background-color: red">', "</span>"),
141 }
142
143
149
150
151 -def escaped_text(s, strip=False):
152 if use_html:
153 if strip:
154 s = s.strip()
155 return "<pre>%s</pre>" % (s)
156 else:
157 return s
158
159
161 if use_html:
162 if strip:
163 s = s.strip()
164 return "<pre>%s</pre>" % (html.escape(s, quote=False))
165 else:
166 return s
167
168
170 if use_html:
171 return """<tr><td>""" + s + """</td></tr>"""
172 else:
173 return s
174
175
181
182
184 d = {'elementnameprefix': elementnameprefix}
185 result = ''
186 if use_html:
187 result += """<div id="%(elementnameprefix)s-wrap"><a name="%(elementnameprefix)s"></a>
188 <table class="infobox rfc822">\n""" % d
189 result += headline(title, bodyelement="%(elementnameprefix)s-body" % d)
190 if use_html:
191 result += """ <tbody id="%(elementnameprefix)s-body" class="infobody">\n""" % d
192 if norow:
193 result += content + "\n"
194 else:
195 result += output_row(content) + "\n"
196 if use_html:
197 result += """</tbody></table></div>"""
198 return result
199
200
201
202
204 v_match = re_version.match(depend)
205 if v_match:
206 d_parts = {'name': v_match.group(1), 'version': v_match.group(2)}
207 else:
208 d_parts = {'name': depend, 'version': ''}
209 return d_parts
210
211
213 or_list = depend.split("|")
214 return or_list
215
216
218 dep_list = depend.split(",")
219 return dep_list
220
221
223
224
225 d_str = re_spacestrip.sub('', d_str)
226 depends_tree = []
227
228 dep_list = get_comma_list(d_str)
229 d = 0
230 while d < len(dep_list):
231
232 depends_tree.append([dep_list[d]])
233 d += 1
234 d = 0
235 while d < len(depends_tree):
236 k = 0
237
238 depends_tree[d] = get_or_list(depends_tree[d][0])
239 while k < len(depends_tree[d]):
240
241 depends_tree[d][k] = get_depends_parts(depends_tree[d][k])
242 k += 1
243 d += 1
244 return depends_tree
245
246
248 recommends = []
249 predepends = []
250 depends = []
251 section = ''
252 maintainer = ''
253 arch = ''
254
255 try:
256 extracts = utils.deb_extract_control(filename)
257 control = apt_pkg.TagSection(extracts)
258 except:
259 print(formatted_text("can't parse control info"))
260 raise
261
262 control_keys = list(control.keys())
263
264 if "Pre-Depends" in control:
265 predepends_str = control["Pre-Depends"]
266 predepends = split_depends(predepends_str)
267
268 if "Depends" in control:
269 depends_str = control["Depends"]
270
271 depends = split_depends(depends_str)
272
273 if "Recommends" in control:
274 recommends_str = control["Recommends"]
275 recommends = split_depends(recommends_str)
276
277 if "Section" in control:
278 section_str = control["Section"]
279
280 c_match = re_contrib.search(section_str)
281 nf_match = re_nonfree.search(section_str)
282 if c_match:
283
284 section = colour_output(section_str, 'contrib')
285 elif nf_match:
286
287 section = colour_output(section_str, 'nonfree')
288 else:
289
290 section = colour_output(section_str, 'main')
291 if "Architecture" in control:
292 arch_str = control["Architecture"]
293 arch = colour_output(arch_str, 'arch')
294
295 if "Maintainer" in control:
296 maintainer = control["Maintainer"]
297 localhost = re_localhost.search(maintainer)
298 if localhost:
299
300 maintainer = colour_output(maintainer, 'maintainer')
301 else:
302 maintainer = escape_if_needed(maintainer)
303
304 return (control, control_keys, section, predepends, depends, recommends, arch, maintainer)
305
306
308 dsc = {}
309
310 with open(filename) as dsc_file:
311 try:
312 dsc = utils.parse_changes(filename, dsc_file=True)
313 except:
314 return formatted_text("can't parse .dsc control info")
315
316 filecontents = strip_pgp_signature(filename)
317 keysinorder = []
318 for l in filecontents.split('\n'):
319 m = re.match(r'([-a-zA-Z0-9]*):', l)
320 if m:
321 keysinorder.append(m.group(1))
322
323 for k in list(dsc.keys()):
324 if k in ("build-depends", "build-depends-indep"):
325 dsc[k] = create_depends_string(suite, split_depends(dsc[k]), session)
326 elif k == "architecture":
327 if (dsc["architecture"] != "any"):
328 dsc['architecture'] = colour_output(dsc["architecture"], 'arch')
329 elif k == "distribution":
330 if dsc["distribution"] not in ('unstable', 'experimental'):
331 dsc['distribution'] = colour_output(dsc["distribution"], 'distro')
332 elif k in ("files", "changes", "description"):
333 if use_html:
334 dsc[k] = formatted_text(dsc[k], strip=True)
335 else:
336 dsc[k] = ('\n' + '\n'.join(' ' + x for x in dsc[k].split('\n'))).rstrip()
337 else:
338 dsc[k] = escape_if_needed(dsc[k])
339
340 filecontents = '\n'.join(format_field(x, dsc[x.lower()])
341 for x in keysinorder if not x.lower().startswith('checksums-')
342 ) + '\n'
343 return filecontents
344
345
347 provides = set()
348 session = DBConn().session()
349 query = '''SELECT DISTINCT value
350 FROM binaries_metadata m
351 JOIN bin_associations b
352 ON b.bin = m.bin_id
353 WHERE key_id = (
354 SELECT key_id
355 FROM metadata_keys
356 WHERE key = 'Provides' )
357 AND b.suite = (
358 SELECT id
359 FROM suite
360 WHERE suite_name = :suite
361 OR codename = :suite)'''
362 for p in session.execute(query, {'suite': suite}):
363 for e in p:
364 for i in e.split(','):
365 provides.add(i.strip())
366 session.close()
367 return provides
368
369
371 result = ""
372 if suite == 'experimental':
373 suite_list = ['experimental', 'unstable']
374 else:
375 suite_list = [suite]
376
377 provides = set()
378 comma_count = 1
379 for l in depends_tree:
380 if (comma_count >= 2):
381 result += ", "
382 or_count = 1
383 for d in l:
384 if (or_count >= 2):
385 result += " | "
386
387
388 component = get_component_by_package_suite(d['name'], suite_list,
389 session=session)
390 if component is not None:
391 adepends = d['name']
392 if d['version'] != '':
393 adepends += " (%s)" % (d['version'])
394
395 if component == "contrib":
396 result += colour_output(adepends, "contrib")
397 elif component in ("non-free-firmware", "non-free"):
398 result += colour_output(adepends, "nonfree")
399 else:
400 result += colour_output(adepends, "main")
401 else:
402 adepends = d['name']
403 if d['version'] != '':
404 adepends += " (%s)" % (d['version'])
405 if not provides:
406 provides = get_provides(suite)
407 if d['name'] in provides:
408 result += colour_output(adepends, "provides")
409 else:
410 result += colour_output(adepends, "bold")
411 or_count += 1
412 comma_count += 1
413 return result
414
415
435
436
438 (control, control_keys, section, predepends, depends, recommends, arch, maintainer) = read_control(filename)
439
440 if control == '':
441 return formatted_text("no control info")
442 to_print = ""
443 if packagename not in package_relations:
444 package_relations[packagename] = {}
445 for key in control_keys:
446 if key == 'Source':
447 field_value = escape_if_needed(control.find(key))
448 if use_html:
449 field_value = '<a href="https://tracker.debian.org/pkg/{0}" rel="nofollow">{0}</a>'.format(
450 field_value)
451 elif key == 'Pre-Depends':
452 field_value = create_depends_string(suite, predepends, session)
453 package_relations[packagename][key] = field_value
454 elif key == 'Depends':
455 field_value = create_depends_string(suite, depends, session)
456 package_relations[packagename][key] = field_value
457 elif key == 'Recommends':
458 field_value = create_depends_string(suite, recommends, session)
459 package_relations[packagename][key] = field_value
460 elif key == 'Section':
461 field_value = section
462 elif key == 'Architecture':
463 field_value = arch
464 elif key == 'Maintainer':
465 field_value = maintainer
466 elif key in ('Homepage', 'Vcs-Browser'):
467 field_value = escape_if_needed(control.find(key))
468 if use_html:
469 field_value = '<a href="%s" rel="nofollow">%s</a>' % \
470 (field_value, field_value)
471 elif key == 'Description':
472 if use_html:
473 field_value = formatted_text(control.find(key), strip=True)
474 else:
475 desc = control.find(key)
476 desc = re_newlinespace.sub('\n ', desc)
477 field_value = escape_if_needed(desc)
478 else:
479 field_value = escape_if_needed(control.find(key))
480 to_print += " " + format_field(key, field_value) + '\n'
481 return to_print
482
483
485 result = subprocess.run(command, stdout=subprocess.PIPE, text=True)
486 if escaped:
487 return escaped_text(result.stdout)
488 else:
489 return formatted_text(result.stdout)
490
491
493 cnf = Config()
494 cmd = []
495
496 user = cnf.get('Dinstall::UnprivUser') or None
497 if user is not None:
498 cmd.extend(['sudo', '-H', '-u', user])
499
500 color = 'always'
501 if use_html:
502 color = 'html'
503
504 cmd.extend(['lintian', '--show-overrides', '--color', color, "--", filename])
505
506 try:
507 return do_command(cmd, escaped=True)
508 except OSError as e:
509 return (colour_output("Running lintian failed: %s" % (e), "error"))
510
511
513 with tempfile.TemporaryFile() as tmpfh:
514 dpkg_cmd = ('dpkg-deb', '--fsys-tarfile', deb_filename)
515 subprocess.check_call(dpkg_cmd, stdout=tmpfh)
516
517 tmpfh.seek(0)
518 with tarfile.open(fileobj=tmpfh, mode="r") as tar:
519 matched_member = None
520 for member in tar:
521 if member.isfile() and match.match(member.name):
522 matched_member = member
523 break
524
525 if not matched_member:
526 return None, None
527
528 fh = tar.extractfile(matched_member)
529 matched_data = fh.read()
530 fh.close()
531
532 return matched_member.name, matched_data
533
534
536 global printed
537
538 re_copyright = re.compile(r"\./usr(/share)?/doc/(?P<package>[^/]+)/copyright")
539 cright_path, cright = extract_one_file_from_deb(deb_filename, re_copyright)
540
541 if not cright_path:
542 return formatted_text("WARNING: No copyright found, please check package manually.")
543
544 package = re_file_binary.match(os.path.basename(deb_filename)).group('package')
545 doc_directory = re_copyright.match(cright_path).group('package')
546 if package != doc_directory:
547 return formatted_text("WARNING: wrong doc directory (expected %s, got %s)." % (package, doc_directory))
548
549 copyrightmd5 = hashlib.md5(cright).hexdigest()
550
551 res = ""
552 if copyrightmd5 in printed.copyrights and printed.copyrights[copyrightmd5] != "%s (%s)" % (package, os.path.basename(deb_filename)):
553 res += formatted_text("NOTE: Copyright is the same as %s.\n\n" %
554 (printed.copyrights[copyrightmd5]))
555 else:
556 printed.copyrights[copyrightmd5] = "%s (%s)" % (package, os.path.basename(deb_filename))
557 return res + formatted_text(cright.decode())
558
559
561 with tempfile.TemporaryDirectory(prefix="dak-examine-package") as tempdir:
562 targetdir = os.path.join(tempdir, "source")
563
564 cmd = ('dpkg-source', '--no-check', '--no-copy', '-x', dsc_filename, targetdir)
565 try:
566 subprocess.check_output(cmd, stderr=subprocess.STDOUT)
567 except subprocess.CalledProcessError as e:
568 res = "How is education supposed to make me feel smarter? Besides, every time I learn something new, it pushes some\n old stuff out of my brain. Remember when I took that home winemaking course, and I forgot how to drive?\n"
569 res += "Error, couldn't extract source, WTF?\n"
570 res += "'dpkg-source -x' failed. return code: %s.\n\n" % (e.returncode)
571 res += e.output
572 return res
573
574 path = os.path.join(targetdir, 'debian/README.source')
575 res = ""
576 if os.path.exists(path):
577 with open(path, 'r') as fh:
578 res += formatted_text(fh.read())
579 else:
580 res += "No README.source in this package\n\n"
581
582 return res
583
584
585 -def check_dsc(suite, dsc_filename, session=None):
586 dsc = read_changes_or_dsc(suite, dsc_filename, session)
587 dsc_basename = os.path.basename(dsc_filename)
588 cdsc = foldable_output(dsc_filename, "dsc", dsc, norow=True) + \
589 "\n" + \
590 foldable_output("lintian {} check for {}".format(
591 get_lintian_version(), dsc_basename),
592 "source-lintian", do_lintian(dsc_filename)) + \
593 "\n" + \
594 foldable_output("README.source for %s" % dsc_basename,
595 "source-readmesource", get_readme_source(dsc_filename))
596 return cdsc
597
598
599 -def check_deb(suite, deb_filename, session=None):
600 filename = os.path.basename(deb_filename)
601 packagename = filename.split('_')[0]
602
603 if filename.endswith(".udeb"):
604 is_a_udeb = 1
605 else:
606 is_a_udeb = 0
607
608 result = foldable_output("control file for %s" % (filename), "binary-%s-control" % packagename,
609 output_deb_info(suite, deb_filename, packagename, session), norow=True) + "\n"
610
611 if is_a_udeb:
612 result += foldable_output("skipping lintian check for udeb",
613 "binary-%s-lintian" % packagename, "") + "\n"
614 else:
615 result += foldable_output("lintian {} check for {}".format(
616 get_lintian_version(), filename),
617 "binary-%s-lintian" % packagename, do_lintian(deb_filename)) + "\n"
618
619 result += foldable_output("contents of %s" % (filename), "binary-%s-contents" % packagename,
620 do_command(["dpkg", "-c", deb_filename])) + "\n"
621
622 if is_a_udeb:
623 result += foldable_output("skipping copyright for udeb",
624 "binary-%s-copyright" % packagename, "") + "\n"
625 else:
626 result += foldable_output("copyright of %s" % (filename),
627 "binary-%s-copyright" % packagename, get_copyright(deb_filename)) + "\n"
628
629 return result
630
631
632
633
634
636 with open(filename, 'rb') as f:
637 data = f.read()
638 signedfile = SignedFile(data, keyrings=(), require_signature=False)
639 return signedfile.contents.decode()
640
641
647
648
650 try:
651 changes = utils.parse_changes(changes_filename)
652 except UnicodeDecodeError:
653 utils.warn("Encoding problem with changes file %s" % (changes_filename))
654 output = display_changes(changes['distribution'], changes_filename)
655
656 files = utils.build_file_list(changes)
657 for f in files.keys():
658 if f.endswith(".deb") or f.endswith(".udeb"):
659 output += check_deb(changes['distribution'], f)
660 if f.endswith(".dsc"):
661 output += check_dsc(changes['distribution'], f)
662
663 return output
664
665
667 global Cnf, db_files, waste, excluded
668
669
670
671 Arguments = [('h', "help", "Examine-Package::Options::Help"),
672 ('H', "html-output", "Examine-Package::Options::Html-Output"),
673 ]
674 for i in ["Help", "Html-Output", "partial-html"]:
675 key = "Examine-Package::Options::%s" % i
676 if key not in Cnf:
677 Cnf[key] = ""
678
679 args = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
680 Options = Cnf.subtree("Examine-Package::Options")
681
682 if Options["Help"]:
683 usage()
684
685 if Options["Html-Output"]:
686 global use_html
687 use_html = True
688
689 for f in args:
690 try:
691 if not Options["Html-Output"]:
692
693 less_cmd = ("less", "-r", "-")
694 less_process = subprocess.Popen(less_cmd, stdin=subprocess.PIPE, bufsize=0, text=True)
695 less_fd = less_process.stdin
696
697 my_fd = less_fd
698 else:
699 my_fd = sys.stdout
700
701 try:
702 if f.endswith(".changes"):
703 my_fd.write(check_changes(f))
704 elif f.endswith(".deb") or f.endswith(".udeb"):
705
706
707 my_fd.write(check_deb('unstable', f))
708 elif f.endswith(".dsc"):
709 my_fd.write(check_dsc('unstable', f))
710 else:
711 utils.fubar("Unrecognised file type: '%s'." % (f))
712 finally:
713 my_fd.write(output_package_relations())
714 if not Options["Html-Output"]:
715
716 less_fd.close()
717 less_process.wait()
718 except OSError as e:
719 if e.errno == errno.EPIPE:
720 utils.warn("[examine-package] Caught EPIPE; skipping.")
721 pass
722 else:
723 raise
724 except KeyboardInterrupt:
725 utils.warn("[examine-package] Caught C-c; skipping.")
726 pass
727
728
736
737
738
739
740 if __name__ == '__main__':
741 main()
742