Package dak :: Module examine_package
[hide private]
[frames] | no frames]

Source Code for Module dak.examine_package

  1  #! /usr/bin/env python3 
  2   
  3  """ 
  4  Script to automate some parts of checking NEW packages 
  5   
  6  Most functions are written in a functional programming style. They 
  7  return a string avoiding the side effect of directly printing the string 
  8  to stdout. Those functions can be used in multithreaded parts of dak. 
  9   
 10  @contact: Debian FTP Master <ftpmaster@debian.org> 
 11  @copyright: 2000, 2001, 2002, 2003, 2006  James Troup <james@nocrew.org> 
 12  @copyright: 2009  Joerg Jaspert <joerg@debian.org> 
 13  @license: GNU General Public License version 2 or later 
 14  """ 
 15   
 16  # This program is free software; you can redistribute it and/or modify 
 17  # it under the terms of the GNU General Public License as published by 
 18  # the Free Software Foundation; either version 2 of the License, or 
 19  # (at your option) any later version. 
 20   
 21  # This program is distributed in the hope that it will be useful, 
 22  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 23  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 24  # GNU General Public License for more details. 
 25   
 26  # You should have received a copy of the GNU General Public License 
 27  # along with this program; if not, write to the Free Software 
 28  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
 29   
 30  ################################################################################ 
 31   
 32  # <Omnic> elmo wrote docs?!!?!?!?!?!?! 
 33  # <aj> as if he wasn't scary enough before!! 
 34  # * aj imagines a little red furry toy sitting hunched over a computer 
 35  #   tapping furiously and giggling to himself 
 36  # <aj> eventually he stops, and his heads slowly spins around and you 
 37  #      see this really evil grin and then he sees you, and picks up a 
 38  #      knife from beside the keyboard and throws it at you, and as you 
 39  #      breathe your last breath, he starts giggling again 
 40  # <aj> but i should be telling this to my psychiatrist, not you guys, 
 41  #      right? :) 
 42   
 43  ################################################################################ 
 44   
 45  import errno 
 46  import hashlib 
 47  import html 
 48  import os 
 49  import re 
 50  import sys 
 51  import apt_pkg 
 52  import shutil 
 53  import subprocess 
 54  import tarfile 
 55  import tempfile 
 56  import threading 
 57   
 58  from daklib import utils 
 59  from daklib.config import Config 
 60  from daklib.dbconn import DBConn, get_component_by_package_suite 
 61  from daklib.gpg import SignedFile 
 62  from daklib.regexes import re_version, re_spacestrip, \ 
 63                             re_contrib, re_nonfree, re_localhost, re_newlinespace, \ 
 64                             re_package, re_doc_directory, re_file_binary 
 65   
 66  ################################################################################ 
 67   
 68  Cnf = None 
 69  Cnf = utils.get_conf() 
 70   
 71  printed = threading.local() 
 72  printed.copyrights = {} 
 73  package_relations = {}           #: Store relations of packages for later output 
 74   
 75  # default is to not output html. 
 76  use_html = False 
 77   
 78  ################################################################################ 
 79   
 80   
81 -def usage(exit_code=0):
82 print("""Usage: dak examine-package [PACKAGE]... 83 Check NEW package(s). 84 85 -h, --help show this help and exit 86 -H, --html-output output html page with inspection result 87 -f, --file-name filename for the html page 88 89 PACKAGE can be a .changes, .dsc, .deb or .udeb filename.""") 90 91 sys.exit(exit_code)
92 93 ################################################################################ 94 # probably xml.sax.saxutils would work as well 95 96
97 -def escape_if_needed(s):
98 if use_html: 99 return html.escape(s) 100 else: 101 return s
102 103
104 -def headline(s, level=2, bodyelement=None):
105 if use_html: 106 if bodyelement: 107 return """<thead> 108 <tr><th colspan="2" class="title" onclick="toggle('%(bodyelement)s', 'table-row-group', 'table-row-group')">%(title)s <span class="toggle-msg">(click to toggle)</span></th></tr> 109 </thead>\n""" % {"bodyelement": bodyelement, "title": html.escape(os.path.basename(s), quote=False)} 110 else: 111 return "<h%d>%s</h%d>\n" % (level, html.escape(s, quote=False), level) 112 else: 113 return "---- %s ----\n" % (s)
114 115 116 # Colour definitions, 'end' isn't really for use 117 118 ansi_colours = { 119 'main': "\033[36m", 120 'contrib': "\033[33m", 121 'nonfree': "\033[31m", 122 'provides': "\033[35m", 123 'arch': "\033[32m", 124 'end': "\033[0m", 125 'bold': "\033[1m", 126 'maintainer': "\033[32m", 127 'distro': "\033[1m\033[41m", 128 'error': "\033[1m\033[41m", 129 } 130 131 html_colours = { 132 'main': ('<span style="color: green">', "</span>"), 133 'contrib': ('<span style="color: orange">', "</span>"), 134 'nonfree': ('<span style="color: red">', "</span>"), 135 'provides': ('<span style="color: magenta">', "</span>"), 136 'arch': ('<span style="color: green">', "</span>"), 137 'bold': ('<span style="font-weight: bold">', "</span>"), 138 'maintainer': ('<span style="color: green">', "</span>"), 139 'distro': ('<span style="font-weight: bold; background-color: red">', "</span>"), 140 'error': ('<span style="font-weight: bold; background-color: red">', "</span>"), 141 } 142 143
144 -def colour_output(s, colour):
145 if use_html: 146 return ("%s%s%s" % (html_colours[colour][0], html.escape(s, quote=False), html_colours[colour][1])) 147 else: 148 return ("%s%s%s" % (ansi_colours[colour], s, ansi_colours['end']))
149 150
151 -def escaped_text(s, strip=False):
152 if use_html: 153 if strip: 154 s = s.strip() 155 return "<pre>%s</pre>" % (s) 156 else: 157 return s
158 159
160 -def formatted_text(s, strip=False):
161 if use_html: 162 if strip: 163 s = s.strip() 164 return "<pre>%s</pre>" % (html.escape(s, quote=False)) 165 else: 166 return s
167 168
169 -def output_row(s):
170 if use_html: 171 return """<tr><td>""" + s + """</td></tr>""" 172 else: 173 return s
174 175
176 -def format_field(k, v):
177 if use_html: 178 return """<tr><td class="key">%s:</td><td class="val">%s</td></tr>""" % (k, v) 179 else: 180 return "%s: %s" % (k, v)
181 182
183 -def foldable_output(title, elementnameprefix, content, norow=False):
184 d = {'elementnameprefix': elementnameprefix} 185 result = '' 186 if use_html: 187 result += """<div id="%(elementnameprefix)s-wrap"><a name="%(elementnameprefix)s"></a> 188 <table class="infobox rfc822">\n""" % d 189 result += headline(title, bodyelement="%(elementnameprefix)s-body" % d) 190 if use_html: 191 result += """ <tbody id="%(elementnameprefix)s-body" class="infobody">\n""" % d 192 if norow: 193 result += content + "\n" 194 else: 195 result += output_row(content) + "\n" 196 if use_html: 197 result += """</tbody></table></div>""" 198 return result
199 200 ################################################################################ 201 202
203 -def get_depends_parts(depend):
204 v_match = re_version.match(depend) 205 if v_match: 206 d_parts = {'name': v_match.group(1), 'version': v_match.group(2)} 207 else: 208 d_parts = {'name': depend, 'version': ''} 209 return d_parts
210 211
212 -def get_or_list(depend):
213 or_list = depend.split("|") 214 return or_list
215 216
217 -def get_comma_list(depend):
218 dep_list = depend.split(",") 219 return dep_list
220 221
222 -def split_depends(d_str):
223 # creates a list of lists of dictionaries of depends (package,version relation) 224 225 d_str = re_spacestrip.sub('', d_str) 226 depends_tree = [] 227 # first split depends string up amongs comma delimiter 228 dep_list = get_comma_list(d_str) 229 d = 0 230 while d < len(dep_list): 231 # put depends into their own list 232 depends_tree.append([dep_list[d]]) 233 d += 1 234 d = 0 235 while d < len(depends_tree): 236 k = 0 237 # split up Or'd depends into a multi-item list 238 depends_tree[d] = get_or_list(depends_tree[d][0]) 239 while k < len(depends_tree[d]): 240 # split depends into {package, version relation} 241 depends_tree[d][k] = get_depends_parts(depends_tree[d][k]) 242 k += 1 243 d += 1 244 return depends_tree
245 246
247 -def read_control(filename):
248 recommends = [] 249 predepends = [] 250 depends = [] 251 section = '' 252 maintainer = '' 253 arch = '' 254 255 try: 256 extracts = utils.deb_extract_control(filename) 257 control = apt_pkg.TagSection(extracts) 258 except: 259 print(formatted_text("can't parse control info")) 260 raise 261 262 control_keys = list(control.keys()) 263 264 if "Pre-Depends" in control: 265 predepends_str = control["Pre-Depends"] 266 predepends = split_depends(predepends_str) 267 268 if "Depends" in control: 269 depends_str = control["Depends"] 270 # create list of dependancy lists 271 depends = split_depends(depends_str) 272 273 if "Recommends" in control: 274 recommends_str = control["Recommends"] 275 recommends = split_depends(recommends_str) 276 277 if "Section" in control: 278 section_str = control["Section"] 279 280 c_match = re_contrib.search(section_str) 281 nf_match = re_nonfree.search(section_str) 282 if c_match: 283 # contrib colour 284 section = colour_output(section_str, 'contrib') 285 elif nf_match: 286 # non-free colour 287 section = colour_output(section_str, 'nonfree') 288 else: 289 # main 290 section = colour_output(section_str, 'main') 291 if "Architecture" in control: 292 arch_str = control["Architecture"] 293 arch = colour_output(arch_str, 'arch') 294 295 if "Maintainer" in control: 296 maintainer = control["Maintainer"] 297 localhost = re_localhost.search(maintainer) 298 if localhost: 299 # highlight bad email 300 maintainer = colour_output(maintainer, 'maintainer') 301 else: 302 maintainer = escape_if_needed(maintainer) 303 304 return (control, control_keys, section, predepends, depends, recommends, arch, maintainer)
305 306
307 -def read_changes_or_dsc(suite, filename, session=None):
308 dsc = {} 309 310 with open(filename) as dsc_file: 311 try: 312 dsc = utils.parse_changes(filename, dsc_file=1) 313 except: 314 return formatted_text("can't parse .dsc control info") 315 316 filecontents = strip_pgp_signature(filename) 317 keysinorder = [] 318 for l in filecontents.split('\n'): 319 m = re.match(r'([-a-zA-Z0-9]*):', l) 320 if m: 321 keysinorder.append(m.group(1)) 322 323 for k in list(dsc.keys()): 324 if k in ("build-depends", "build-depends-indep"): 325 dsc[k] = create_depends_string(suite, split_depends(dsc[k]), session) 326 elif k == "architecture": 327 if (dsc["architecture"] != "any"): 328 dsc['architecture'] = colour_output(dsc["architecture"], 'arch') 329 elif k == "distribution": 330 if dsc["distribution"] not in ('unstable', 'experimental'): 331 dsc['distribution'] = colour_output(dsc["distribution"], 'distro') 332 elif k in ("files", "changes", "description"): 333 if use_html: 334 dsc[k] = formatted_text(dsc[k], strip=True) 335 else: 336 dsc[k] = ('\n' + '\n'.join(' ' + x for x in dsc[k].split('\n'))).rstrip() 337 else: 338 dsc[k] = escape_if_needed(dsc[k]) 339 340 filecontents = '\n'.join(format_field(x, dsc[x.lower()]) 341 for x in keysinorder if not x.lower().startswith('checksums-') 342 ) + '\n' 343 return filecontents
344 345
346 -def get_provides(suite):
347 provides = set() 348 session = DBConn().session() 349 query = '''SELECT DISTINCT value 350 FROM binaries_metadata m 351 JOIN bin_associations b 352 ON b.bin = m.bin_id 353 WHERE key_id = ( 354 SELECT key_id 355 FROM metadata_keys 356 WHERE key = 'Provides' ) 357 AND b.suite = ( 358 SELECT id 359 FROM suite 360 WHERE suite_name = :suite 361 OR codename = :suite)''' 362 for p in session.execute(query, {'suite': suite}): 363 for e in p: 364 for i in e.split(','): 365 provides.add(i.strip()) 366 session.close() 367 return provides
368 369
370 -def create_depends_string(suite, depends_tree, session=None):
371 result = "" 372 if suite == 'experimental': 373 suite_list = ['experimental', 'unstable'] 374 else: 375 suite_list = [suite] 376 377 provides = set() 378 comma_count = 1 379 for l in depends_tree: 380 if (comma_count >= 2): 381 result += ", " 382 or_count = 1 383 for d in l: 384 if (or_count >= 2): 385 result += " | " 386 # doesn't do version lookup yet. 387 388 component = get_component_by_package_suite(d['name'], suite_list, 389 session=session) 390 if component is not None: 391 adepends = d['name'] 392 if d['version'] != '': 393 adepends += " (%s)" % (d['version']) 394 395 if component == "contrib": 396 result += colour_output(adepends, "contrib") 397 elif component in ("non-free-firmware", "non-free"): 398 result += colour_output(adepends, "nonfree") 399 else: 400 result += colour_output(adepends, "main") 401 else: 402 adepends = d['name'] 403 if d['version'] != '': 404 adepends += " (%s)" % (d['version']) 405 if not provides: 406 provides = get_provides(suite) 407 if d['name'] in provides: 408 result += colour_output(adepends, "provides") 409 else: 410 result += colour_output(adepends, "bold") 411 or_count += 1 412 comma_count += 1 413 return result
414 415
416 -def output_package_relations():
417 """ 418 Output the package relations, if there is more than one package checked in this run. 419 """ 420 421 if len(package_relations) < 2: 422 # Only list something if we have more than one binary to compare 423 package_relations.clear() 424 result = "" 425 else: 426 to_print = "" 427 for package in package_relations: 428 for relation in package_relations[package]: 429 to_print += "%-15s: (%s) %s\n" % (package, relation, package_relations[package][relation]) 430 431 package_relations.clear() 432 result = foldable_output("Package relations", "relations", to_print) 433 package_relations.clear() 434 return result
435 436
437 -def output_deb_info(suite, filename, packagename, session=None):
438 (control, control_keys, section, predepends, depends, recommends, arch, maintainer) = read_control(filename) 439 440 if control == '': 441 return formatted_text("no control info") 442 to_print = "" 443 if packagename not in package_relations: 444 package_relations[packagename] = {} 445 for key in control_keys: 446 if key == 'Source': 447 field_value = escape_if_needed(control.find(key)) 448 if use_html: 449 field_value = '<a href="https://tracker.debian.org/pkg/{0}" rel="nofollow">{0}</a>'.format( 450 field_value) 451 elif key == 'Pre-Depends': 452 field_value = create_depends_string(suite, predepends, session) 453 package_relations[packagename][key] = field_value 454 elif key == 'Depends': 455 field_value = create_depends_string(suite, depends, session) 456 package_relations[packagename][key] = field_value 457 elif key == 'Recommends': 458 field_value = create_depends_string(suite, recommends, session) 459 package_relations[packagename][key] = field_value 460 elif key == 'Section': 461 field_value = section 462 elif key == 'Architecture': 463 field_value = arch 464 elif key == 'Maintainer': 465 field_value = maintainer 466 elif key in ('Homepage', 'Vcs-Browser'): 467 field_value = escape_if_needed(control.find(key)) 468 if use_html: 469 field_value = '<a href="%s" rel="nofollow">%s</a>' % \ 470 (field_value, field_value) 471 elif key == 'Description': 472 if use_html: 473 field_value = formatted_text(control.find(key), strip=True) 474 else: 475 desc = control.find(key) 476 desc = re_newlinespace.sub('\n ', desc) 477 field_value = escape_if_needed(desc) 478 else: 479 field_value = escape_if_needed(control.find(key)) 480 to_print += " " + format_field(key, field_value) + '\n' 481 return to_print
482 483
484 -def do_command(command, escaped=False):
485 result = subprocess.run(command, stdout=subprocess.PIPE, text=True) 486 if escaped: 487 return escaped_text(result.stdout) 488 else: 489 return formatted_text(result.stdout)
490 491
492 -def do_lintian(filename):
493 cnf = Config() 494 cmd = [] 495 496 user = cnf.get('Dinstall::UnprivUser') or None 497 if user is not None: 498 cmd.extend(['sudo', '-H', '-u', user]) 499 500 color = 'always' 501 if use_html: 502 color = 'html' 503 504 cmd.extend(['lintian', '--show-overrides', '--color', color, "--", filename]) 505 506 try: 507 return do_command(cmd, escaped=True) 508 except OSError as e: 509 return (colour_output("Running lintian failed: %s" % (e), "error"))
510 511
512 -def extract_one_file_from_deb(deb_filename, match):
513 with tempfile.TemporaryFile() as tmpfh: 514 dpkg_cmd = ('dpkg-deb', '--fsys-tarfile', deb_filename) 515 subprocess.check_call(dpkg_cmd, stdout=tmpfh) 516 517 tmpfh.seek(0) 518 with tarfile.open(fileobj=tmpfh, mode="r") as tar: 519 matched_member = None 520 for member in tar: 521 if member.isfile() and match.match(member.name): 522 matched_member = member 523 break 524 525 if not matched_member: 526 return None, None 527 528 fh = tar.extractfile(matched_member) 529 matched_data = fh.read() 530 fh.close() 531 532 return matched_member.name, matched_data
533 534 558 559
560 -def get_readme_source(dsc_filename):
561 with tempfile.TemporaryDirectory(prefix="dak-examine-package") as tempdir: 562 targetdir = os.path.join(tempdir, "source") 563 564 cmd = ('dpkg-source', '--no-check', '--no-copy', '-x', dsc_filename, targetdir) 565 try: 566 subprocess.check_output(cmd, stderr=subprocess.STDOUT) 567 except subprocess.CalledProcessError as e: 568 res = "How is education supposed to make me feel smarter? Besides, every time I learn something new, it pushes some\n old stuff out of my brain. Remember when I took that home winemaking course, and I forgot how to drive?\n" 569 res += "Error, couldn't extract source, WTF?\n" 570 res += "'dpkg-source -x' failed. return code: %s.\n\n" % (e.returncode) 571 res += e.output 572 return res 573 574 path = os.path.join(targetdir, 'debian/README.source') 575 res = "" 576 if os.path.exists(path): 577 with open(path, 'r') as fh: 578 res += formatted_text(fh.read()) 579 else: 580 res += "No README.source in this package\n\n" 581 582 return res
583 584
585 -def check_dsc(suite, dsc_filename, session=None):
586 dsc = read_changes_or_dsc(suite, dsc_filename, session) 587 dsc_basename = os.path.basename(dsc_filename) 588 cdsc = foldable_output(dsc_filename, "dsc", dsc, norow=True) + \ 589 "\n" + \ 590 foldable_output("lintian {} check for {}".format( 591 get_lintian_version(), dsc_basename), 592 "source-lintian", do_lintian(dsc_filename)) + \ 593 "\n" + \ 594 foldable_output("README.source for %s" % dsc_basename, 595 "source-readmesource", get_readme_source(dsc_filename)) 596 return cdsc
597 598
599 -def check_deb(suite, deb_filename, session=None):
600 filename = os.path.basename(deb_filename) 601 packagename = filename.split('_')[0] 602 603 if filename.endswith(".udeb"): 604 is_a_udeb = 1 605 else: 606 is_a_udeb = 0 607 608 result = foldable_output("control file for %s" % (filename), "binary-%s-control" % packagename, 609 output_deb_info(suite, deb_filename, packagename, session), norow=True) + "\n" 610 611 if is_a_udeb: 612 result += foldable_output("skipping lintian check for udeb", 613 "binary-%s-lintian" % packagename, "") + "\n" 614 else: 615 result += foldable_output("lintian {} check for {}".format( 616 get_lintian_version(), filename), 617 "binary-%s-lintian" % packagename, do_lintian(deb_filename)) + "\n" 618 619 result += foldable_output("contents of %s" % (filename), "binary-%s-contents" % packagename, 620 do_command(["dpkg", "-c", deb_filename])) + "\n" 621 622 if is_a_udeb: 623 result += foldable_output("skipping copyright for udeb", 624 "binary-%s-copyright" % packagename, "") + "\n" 625 else: 626 result += foldable_output("copyright of %s" % (filename), 627 "binary-%s-copyright" % packagename, get_copyright(deb_filename)) + "\n" 628 629 return result
630 631 # Read a file, strip the signature and return the modified contents as 632 # a string. 633 634
635 -def strip_pgp_signature(filename):
636 with open(filename, 'rb') as f: 637 data = f.read() 638 signedfile = SignedFile(data, keyrings=(), require_signature=False) 639 return signedfile.contents.decode()
640 641
642 -def display_changes(suite, changes_filename):
643 global printed 644 changes = read_changes_or_dsc(suite, changes_filename) 645 printed.copyrights = {} 646 return foldable_output(changes_filename, "changes", changes, norow=True)
647 648
649 -def check_changes(changes_filename):
650 try: 651 changes = utils.parse_changes(changes_filename) 652 except UnicodeDecodeError: 653 utils.warn("Encoding problem with changes file %s" % (changes_filename)) 654 output = display_changes(changes['distribution'], changes_filename) 655 656 files = utils.build_file_list(changes) 657 for f in files.keys(): 658 if f.endswith(".deb") or f.endswith(".udeb"): 659 output += check_deb(changes['distribution'], f) 660 if f.endswith(".dsc"): 661 output += check_dsc(changes['distribution'], f) 662 # else: => byhand 663 return output
664 665
666 -def main():
667 global Cnf, db_files, waste, excluded 668 669 # Cnf = utils.get_conf() 670 671 Arguments = [('h', "help", "Examine-Package::Options::Help"), 672 ('H', "html-output", "Examine-Package::Options::Html-Output"), 673 ] 674 for i in ["Help", "Html-Output", "partial-html"]: 675 key = "Examine-Package::Options::%s" % i 676 if key not in Cnf: 677 Cnf[key] = "" 678 679 args = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) 680 Options = Cnf.subtree("Examine-Package::Options") 681 682 if Options["Help"]: 683 usage() 684 685 if Options["Html-Output"]: 686 global use_html 687 use_html = True 688 689 for f in args: 690 try: 691 if not Options["Html-Output"]: 692 # Pipe output for each argument through less 693 less_cmd = ("less", "-r", "-") 694 less_process = subprocess.Popen(less_cmd, stdin=subprocess.PIPE, bufsize=0, text=True) 695 less_fd = less_process.stdin 696 # -R added to display raw control chars for colour 697 my_fd = less_fd 698 else: 699 my_fd = sys.stdout 700 701 try: 702 if f.endswith(".changes"): 703 my_fd.write(check_changes(f)) 704 elif f.endswith(".deb") or f.endswith(".udeb"): 705 # default to unstable when we don't have a .changes file 706 # perhaps this should be a command line option? 707 my_fd.write(check_deb('unstable', f)) 708 elif f.endswith(".dsc"): 709 my_fd.write(check_dsc('unstable', f)) 710 else: 711 utils.fubar("Unrecognised file type: '%s'." % (f)) 712 finally: 713 my_fd.write(output_package_relations()) 714 if not Options["Html-Output"]: 715 # Reset stdout here so future less invocations aren't FUBAR 716 less_fd.close() 717 less_process.wait() 718 except OSError as e: 719 if e.errno == errno.EPIPE: 720 utils.warn("[examine-package] Caught EPIPE; skipping.") 721 pass 722 else: 723 raise 724 except KeyboardInterrupt: 725 utils.warn("[examine-package] Caught C-c; skipping.") 726 pass
727 728
729 -def get_lintian_version():
730 if not hasattr(get_lintian_version, '_version'): 731 # eg. "Lintian v2.5.100" 732 val = subprocess.check_output(('lintian', '--version'), text=True) 733 get_lintian_version._version = val.split(' v')[-1].strip() 734 735 return get_lintian_version._version
736 737 738 ####################################################################################### 739 740 if __name__ == '__main__': 741 main() 742