Coverage for dak/examine

1#! /usr/bin/env python3

3"""

4Script to automate some parts of checking NEW packages

6Most functions are written in a functional programming style. They

7return a string avoiding the side effect of directly printing the string

8to stdout. Those functions can be used in multithreaded parts of dak.

10@contact: Debian FTP Master <ftpmaster@debian.org>

11@copyright: 2000, 2001, 2002, 2003, 2006 James Troup <james@nocrew.org>

12@copyright: 2009 Joerg Jaspert <joerg@debian.org>

13@license: GNU General Public License version 2 or later

14"""

16# This program is free software; you can redistribute it and/or modify

17# it under the terms of the GNU General Public License as published by

18# the Free Software Foundation; either version 2 of the License, or

19# (at your option) any later version.

21# This program is distributed in the hope that it will be useful,

22# but WITHOUT ANY WARRANTY; without even the implied warranty of

23# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

24# GNU General Public License for more details.

26# You should have received a copy of the GNU General Public License

27# along with this program; if not, write to the Free Software

28# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

30################################################################################

32# <Omnic> elmo wrote docs?!!?!?!?!?!?!

33# <aj> as if he wasn't scary enough before!!

34# * aj imagines a little red furry toy sitting hunched over a computer

35# tapping furiously and giggling to himself

36# <aj> eventually he stops, and his heads slowly spins around and you

37# see this really evil grin and then he sees you, and picks up a

38# knife from beside the keyboard and throws it at you, and as you

39# breathe your last breath, he starts giggling again

40# <aj> but i should be telling this to my psychiatrist, not you guys,

41# right? :)

43################################################################################

45import errno

46import hashlib

47import html

48import os

49import re

50import subprocess

51import sys

52import tarfile

53import tempfile

54import threading

56import apt_pkg

58from daklib import utils

59from daklib.config import Config

60from daklib.dbconn import DBConn, get_component_by_package_suite

61from daklib.gpg import SignedFile

62from daklib.regexes import (

63 re_contrib,

64 re_file_binary,

65 re_localhost,

66 re_newlinespace,

67 re_nonfree,

68 re_spacestrip,

69 re_version,

70)

72################################################################################

74Cnf = None

75Cnf = utils.get_conf()

77printed = threading.local()

78printed.copyrights = {}

79package_relations = {} #: Store relations of packages for later output

81# default is to not output html.

82use_html = False

84################################################################################

87def usage(exit_code=0):

88 print(

89 """Usage: dak examine-package [PACKAGE]...

90Check NEW package(s).

92 -h, --help show this help and exit

93 -H, --html-output output html page with inspection result

94 -f, --file-name filename for the html page

96PACKAGE can be a .changes, .dsc, .deb or .udeb filename."""

97 )

99 sys.exit(exit_code)

100

101

102################################################################################

103# probably xml.sax.saxutils would work as well

104

105

106def escape_if_needed(s):

107 if use_html:

108 return html.escape(s)

109 else:

110 return s

111

112

113def headline(s, level=2, bodyelement=None):

114 if use_html:

115 if bodyelement: 115 ↛ 123line 115 didn't jump to line 123, because the condition on line 115 was never false

116 return """<thead>

117 <tr><th colspan="2" class="title" onclick="toggle('%(bodyelement)s', 'table-row-group', 'table-row-group')">%(title)s <span class="toggle-msg">(click to toggle)</span></th></tr>

118 </thead>\n""" % {

119 "bodyelement": bodyelement,

120 "title": html.escape(os.path.basename(s), quote=False),

121 }

122 else:

123 return "<h%d>%s</h%d>\n" % (level, html.escape(s, quote=False), level)

124 else:

125 return "---- %s ----\n" % (s)

126

127

128# Colour definitions, 'end' isn't really for use

129

130ansi_colours = {

131 "main": "\033[36m",

132 "contrib": "\033[33m",

133 "nonfree": "\033[31m",

134 "provides": "\033[35m",

135 "arch": "\033[32m",

136 "end": "\033[0m",

137 "bold": "\033[1m",

138 "maintainer": "\033[32m",

139 "distro": "\033[1m\033[41m",

140 "error": "\033[1m\033[41m",

141}

142

143html_colours = {

144 "main": ('<span style="color: green">', "</span>"),

145 "contrib": ('<span style="color: orange">', "</span>"),

146 "nonfree": ('<span style="color: red">', "</span>"),

147 "provides": ('<span style="color: magenta">', "</span>"),

148 "arch": ('<span style="color: green">', "</span>"),

149 "bold": ('<span style="font-weight: bold">', "</span>"),

150 "maintainer": ('<span style="color: green">', "</span>"),

151 "distro": ('<span style="font-weight: bold; background-color: red">', "</span>"),

152 "error": ('<span style="font-weight: bold; background-color: red">', "</span>"),

153}

154

155

156def colour_output(s, colour):

157 if use_html:

158 return "%s%s%s" % (

159 html_colours[colour][0],

160 html.escape(s, quote=False),

161 html_colours[colour][1],

162 )

163 else:

164 return "%s%s%s" % (ansi_colours[colour], s, ansi_colours["end"])

165

166

167def escaped_text(s, strip=False):

168 if use_html:

169 if strip: 169 ↛ 170line 169 didn't jump to line 170, because the condition on line 169 was never true

170 s = s.strip()

171 return "<pre>%s</pre>" % (s)

172 else:

173 return s

174

175

176def formatted_text(s, strip=False):

177 if use_html:

178 if strip:

179 s = s.strip()

180 return "<pre>%s</pre>" % (html.escape(s, quote=False))

181 else:

182 return s

183

184

185def output_row(s):

186 if use_html:

187 return """<tr><td>""" + s + """</td></tr>"""

188 else:

189 return s

190

191

192def format_field(k, v):

193 if use_html:

194 return """<tr><td class="key">%s:</td><td class="val">%s</td></tr>""" % (k, v)

195 else:

196 return "%s: %s" % (k, v)

197

198

199def foldable_output(title, elementnameprefix, content, norow=False):

200 d = {"elementnameprefix": elementnameprefix}

201 result = ""

202 if use_html:

203 result += (

204 """<div id="%(elementnameprefix)s-wrap"><a name="%(elementnameprefix)s"></a>

205 <table class="infobox rfc822">\n"""

206 % d

207 )

208 result += headline(title, bodyelement="%(elementnameprefix)s-body" % d)

209 if use_html:

210 result += (

211 """ <tbody id="%(elementnameprefix)s-body" class="infobody">\n""" % d

212 )

213 if norow:

214 result += content + "\n"

215 else:

216 result += output_row(content) + "\n"

217 if use_html:

218 result += """</tbody></table></div>"""

219 return result

220

221

222################################################################################

223

224

225def get_depends_parts(depend):

226 v_match = re_version.match(depend)

227 if v_match: 227 ↛ 230line 227 didn't jump to line 230, because the condition on line 227 was never false

228 d_parts = {"name": v_match.group(1), "version": v_match.group(2)}

229 else:

230 d_parts = {"name": depend, "version": ""}

231 return d_parts

232

233

234def get_or_list(depend):

235 or_list = depend.split("|")

236 return or_list

237

238

239def get_comma_list(depend):

240 dep_list = depend.split(",")

241 return dep_list

242

243

244def split_depends(d_str):

245 # creates a list of lists of dictionaries of depends (package,version relation)

246

247 d_str = re_spacestrip.sub("", d_str)

248 depends_tree = []

249 # first split depends string up amongs comma delimiter

250 dep_list = get_comma_list(d_str)

251 d = 0

252 while d < len(dep_list):

253 # put depends into their own list

254 depends_tree.append([dep_list[d]])

255 d += 1

256 d = 0

257 while d < len(depends_tree):

258 k = 0

259 # split up Or'd depends into a multi-item list

260 depends_tree[d] = get_or_list(depends_tree[d][0])

261 while k < len(depends_tree[d]):

262 # split depends into {package, version relation}

263 depends_tree[d][k] = get_depends_parts(depends_tree[d][k])

264 k += 1

265 d += 1

266 return depends_tree

267

268

269def read_control(filename):

270 recommends = []

271 predepends = []

272 depends = []

273 section = ""

274 maintainer = ""

275 arch = ""

276

277 try:

278 extracts = utils.deb_extract_control(filename)

279 control = apt_pkg.TagSection(extracts)

280 except:

281 print(formatted_text("can't parse control info"))

282 raise

283

284 control_keys = list(control.keys())

285

286 if "Pre-Depends" in control: 286 ↛ 287line 286 didn't jump to line 287, because the condition on line 286 was never true

287 predepends_str = control["Pre-Depends"]

288 predepends = split_depends(predepends_str)

289

290 if "Depends" in control: 290 ↛ 291line 290 didn't jump to line 291, because the condition on line 290 was never true

291 depends_str = control["Depends"]

292 # create list of dependancy lists

293 depends = split_depends(depends_str)

294

295 if "Recommends" in control: 295 ↛ 296line 295 didn't jump to line 296, because the condition on line 295 was never true

296 recommends_str = control["Recommends"]

297 recommends = split_depends(recommends_str)

298

299 if "Section" in control: 299 ↛ 313line 299 didn't jump to line 313, because the condition on line 299 was never false

300 section_str = control["Section"]

301

302 c_match = re_contrib.search(section_str)

303 nf_match = re_nonfree.search(section_str)

304 if c_match: 304 ↛ 306line 304 didn't jump to line 306, because the condition on line 304 was never true

305 # contrib colour

306 section = colour_output(section_str, "contrib")

307 elif nf_match: 307 ↛ 309line 307 didn't jump to line 309, because the condition on line 307 was never true

308 # non-free colour

309 section = colour_output(section_str, "nonfree")

310 else:

311 # main

312 section = colour_output(section_str, "main")

313 if "Architecture" in control: 313 ↛ 317line 313 didn't jump to line 317, because the condition on line 313 was never false

314 arch_str = control["Architecture"]

315 arch = colour_output(arch_str, "arch")

316

317 if "Maintainer" in control: 317 ↛ 326line 317 didn't jump to line 326, because the condition on line 317 was never false

318 maintainer = control["Maintainer"]

319 localhost = re_localhost.search(maintainer)

320 if localhost: 320 ↛ 322line 320 didn't jump to line 322, because the condition on line 320 was never true

321 # highlight bad email

322 maintainer = colour_output(maintainer, "maintainer")

323 else:

324 maintainer = escape_if_needed(maintainer)

325

326 return (

327 control,

328 control_keys,

329 section,

330 predepends,

331 depends,

332 recommends,

333 arch,

334 maintainer,

335 )

336

337

338def read_changes_or_dsc(suite, filename, session=None):

339 dsc = {}

340

341 try:

342 dsc = utils.parse_changes(filename, dsc_file=True)

343 except:

344 return formatted_text("can't parse .dsc control info")

345

346 filecontents = strip_pgp_signature(filename)

347 keysinorder = []

348 for line in filecontents.split("\n"):

349 m = re.match(r"([-a-zA-Z0-9]*):", line)

350 if m:

351 keysinorder.append(m.group(1))

352

353 for k in list(dsc.keys()):

354 if k in ("build-depends", "build-depends-indep"):

355 dsc[k] = create_depends_string(suite, split_depends(dsc[k]), session)

356 elif k == "architecture":

357 if dsc["architecture"] != "any": 357 ↛ 353line 357 didn't jump to line 353, because the condition on line 357 was never false

358 dsc["architecture"] = colour_output(dsc["architecture"], "arch")

359 elif k == "distribution":

360 if dsc["distribution"] not in ("unstable", "experimental"): 360 ↛ 361line 360 didn't jump to line 361, because the condition on line 360 was never true

361 dsc["distribution"] = colour_output(dsc["distribution"], "distro")

362 elif k in ("files", "changes", "description"):

363 if use_html:

364 dsc[k] = formatted_text(dsc[k], strip=True)

365 else:

366 dsc[k] = (

367 "\n" + "\n".join(" " + x for x in dsc[k].split("\n"))

368 ).rstrip()

369 else:

370 dsc[k] = escape_if_needed(dsc[k])

371

372 filecontents = (

373 "\n".join(

374 format_field(x, dsc[x.lower()])

375 for x in keysinorder

376 if not x.lower().startswith("checksums-")

377 )

378 + "\n"

379 )

380 return filecontents

381

382

383def get_provides(suite):

384 provides = set()

385 session = DBConn().session()

386 query = """SELECT DISTINCT value

387 FROM binaries_metadata m

388 JOIN bin_associations b

389 ON b.bin = m.bin_id

390 WHERE key_id = (

391 SELECT key_id

392 FROM metadata_keys

393 WHERE key = 'Provides' )

394 AND b.suite = (

395 SELECT id

396 FROM suite

397 WHERE suite_name = :suite

398 OR codename = :suite)"""

399 for p in session.execute(query, {"suite": suite}): 399 ↛ 400line 399 didn't jump to line 400, because the loop on line 399 never started

400 for e in p:

401 for i in e.split(","):

402 provides.add(i.strip())

403 session.close()

404 return provides

405

406

407def create_depends_string(suite, depends_tree, session=None):

408 result = ""

409 if suite == "experimental": 409 ↛ 410line 409 didn't jump to line 410, because the condition on line 409 was never true

410 suite_list = ["experimental", "unstable"]

411 else:

412 suite_list = [suite]

413

414 provides = set()

415 comma_count = 1

416 for item in depends_tree:

417 if comma_count >= 2: 417 ↛ 418line 417 didn't jump to line 418, because the condition on line 417 was never true

418 result += ", "

419 or_count = 1

420 for d in item:

421 if or_count >= 2: 421 ↛ 422line 421 didn't jump to line 422, because the condition on line 421 was never true

422 result += " | "

423 # doesn't do version lookup yet.

424

425 component = get_component_by_package_suite(

426 d["name"], suite_list, session=session

427 )

428 if component is not None: 428 ↛ 429line 428 didn't jump to line 429, because the condition on line 428 was never true

429 adepends = d["name"]

430 if d["version"] != "":

431 adepends += " (%s)" % (d["version"])

432

433 if component == "contrib":

434 result += colour_output(adepends, "contrib")

435 elif component in ("non-free-firmware", "non-free"):

436 result += colour_output(adepends, "nonfree")

437 else:

438 result += colour_output(adepends, "main")

439 else:

440 adepends = d["name"]

441 if d["version"] != "": 441 ↛ 443line 441 didn't jump to line 443, because the condition on line 441 was never false

442 adepends += " (%s)" % (d["version"])

443 if not provides: 443 ↛ 445line 443 didn't jump to line 445, because the condition on line 443 was never false

444 provides = get_provides(suite)

445 if d["name"] in provides: 445 ↛ 446line 445 didn't jump to line 446, because the condition on line 445 was never true

446 result += colour_output(adepends, "provides")

447 else:

448 result += colour_output(adepends, "bold")

449 or_count += 1

450 comma_count += 1

451 return result

452

453

454def output_package_relations():

455 """

456 Output the package relations, if there is more than one package checked in this run.

457 """

458

459 if len(package_relations) < 2: 459 ↛ 464line 459 didn't jump to line 464, because the condition on line 459 was never false

460 # Only list something if we have more than one binary to compare

461 package_relations.clear()

462 result = ""

463 else:

464 to_print = ""

465 for package in package_relations:

466 for relation in package_relations[package]:

467 to_print += "%-15s: (%s) %s\n" % (

468 package,

469 relation,

470 package_relations[package][relation],

471 )

472

473 package_relations.clear()

474 result = foldable_output("Package relations", "relations", to_print)

475 package_relations.clear()

476 return result

477

478

479def output_deb_info(suite, filename, packagename, session=None):

480 (

481 control,

482 control_keys,

483 section,

484 predepends,

485 depends,

486 recommends,

487 arch,

488 maintainer,

489 ) = read_control(filename)

490

491 if control == "": 491 ↛ 492line 491 didn't jump to line 492, because the condition on line 491 was never true

492 return formatted_text("no control info")

493 to_print = ""

494 if packagename not in package_relations: 494 ↛ 496line 494 didn't jump to line 496, because the condition on line 494 was never false

495 package_relations[packagename] = {}

496 for key in control_keys:

497 if key == "Source": 497 ↛ 498line 497 didn't jump to line 498, because the condition on line 497 was never true

498 field_value = escape_if_needed(control.find(key))

499 if use_html:

500 field_value = '<a href="https://tracker.debian.org/pkg/{0}" rel="nofollow">{0}</a>'.format(

501 field_value

502 )

503 elif key == "Pre-Depends": 503 ↛ 504line 503 didn't jump to line 504, because the condition on line 503 was never true

504 field_value = create_depends_string(suite, predepends, session)

505 package_relations[packagename][key] = field_value

506 elif key == "Depends": 506 ↛ 507line 506 didn't jump to line 507, because the condition on line 506 was never true

507 field_value = create_depends_string(suite, depends, session)

508 package_relations[packagename][key] = field_value

509 elif key == "Recommends": 509 ↛ 510line 509 didn't jump to line 510, because the condition on line 509 was never true

510 field_value = create_depends_string(suite, recommends, session)

511 package_relations[packagename][key] = field_value

512 elif key == "Section":

513 field_value = section

514 elif key == "Architecture":

515 field_value = arch

516 elif key == "Maintainer":

517 field_value = maintainer

518 elif key in ("Homepage", "Vcs-Browser"): 518 ↛ 519line 518 didn't jump to line 519, because the condition on line 518 was never true

519 field_value = escape_if_needed(control.find(key))

520 if use_html:

521 field_value = '<a href="%s" rel="nofollow">%s</a>' % (

522 field_value,

523 field_value,

524 )

525 elif key == "Description":

526 if use_html:

527 field_value = formatted_text(control.find(key), strip=True)

528 else:

529 desc = control.find(key)

530 desc = re_newlinespace.sub("\n ", desc)

531 field_value = escape_if_needed(desc)

532 else:

533 field_value = escape_if_needed(control.find(key))

534 to_print += " " + format_field(key, field_value) + "\n"

535 return to_print

536

537

538def do_command(command, escaped=False):

539 result = subprocess.run(command, stdout=subprocess.PIPE, text=True)

540 if escaped:

541 return escaped_text(result.stdout)

542 else:

543 return formatted_text(result.stdout)

544

545

546def do_lintian(filename):

547 cnf = Config()

548 cmd = []

549

550 user = cnf.get("Dinstall::UnprivUser") or None

551 if user is not None: 551 ↛ 552line 551 didn't jump to line 552, because the condition on line 551 was never true

552 cmd.extend(["sudo", "-H", "-u", user])

553

554 color = "always"

555 if use_html:

556 color = "html"

557

558 cmd.extend(["lintian", "--show-overrides", "--color", color, "--", filename])

559

560 try:

561 return do_command(cmd, escaped=True)

562 except OSError as e:

563 return colour_output("Running lintian failed: %s" % (e), "error")

564

565

566def extract_one_file_from_deb(deb_filename, match):

567 with tempfile.TemporaryFile() as tmpfh:

568 dpkg_cmd = ("dpkg-deb", "--fsys-tarfile", deb_filename)

569 subprocess.check_call(dpkg_cmd, stdout=tmpfh)

570

571 tmpfh.seek(0)

572 with tarfile.open(fileobj=tmpfh, mode="r") as tar:

573 matched_member = None

574 for member in tar: 574 ↛ 579line 574 didn't jump to line 579, because the loop on line 574 didn't complete

575 if member.isfile() and match.match(member.name):

576 matched_member = member

577 break

578

579 if not matched_member: 579 ↛ 580line 579 didn't jump to line 580, because the condition on line 579 was never true

580 return None, None

581

582 fh = tar.extractfile(matched_member)

583 matched_data = fh.read()

584 fh.close()

585

586 return matched_member.name, matched_data

587

588

589def get_copyright(deb_filename):

590 global printed

591

592 re_copyright = re.compile(r"\./usr(/share)?/doc/(?P<package>[^/]+)/copyright")

593 cright_path, cright = extract_one_file_from_deb(deb_filename, re_copyright)

594

595 if not cright_path: 595 ↛ 596line 595 didn't jump to line 596, because the condition on line 595 was never true

596 return formatted_text(

597 "WARNING: No copyright found, please check package manually."

598 )

599

600 package = re_file_binary.match(os.path.basename(deb_filename)).group("package")

601 doc_directory = re_copyright.match(cright_path).group("package")

602 if package != doc_directory: 602 ↛ 603line 602 didn't jump to line 603, because the condition on line 602 was never true

603 return formatted_text(

604 "WARNING: wrong doc directory (expected %s, got %s)."

605 % (package, doc_directory)

606 )

607

608 copyrightmd5 = hashlib.md5(cright).hexdigest()

609

610 res = ""

611 if copyrightmd5 in printed.copyrights and printed.copyrights[ 611 ↛ 614line 611 didn't jump to line 614, because the condition on line 611 was never true

612 copyrightmd5

613 ] != "%s (%s)" % (package, os.path.basename(deb_filename)):

614 res += formatted_text(

615 "NOTE: Copyright is the same as %s.\n\n"

616 % (printed.copyrights[copyrightmd5])

617 )

618 else:

619 printed.copyrights[copyrightmd5] = "%s (%s)" % (

620 package,

621 os.path.basename(deb_filename),

622 )

623 return res + formatted_text(cright.decode())

624

625

626def get_readme_source(dsc_filename):

627 with tempfile.TemporaryDirectory(prefix="dak-examine-package") as tempdir:

628 targetdir = os.path.join(tempdir, "source")

629

630 cmd = ("dpkg-source", "--no-check", "--no-copy", "-x", dsc_filename, targetdir)

631 try:

632 subprocess.check_output(cmd, stderr=subprocess.STDOUT)

633 except subprocess.CalledProcessError as e:

634 res = "How is education supposed to make me feel smarter? Besides, every time I learn something new, it pushes some\n old stuff out of my brain. Remember when I took that home winemaking course, and I forgot how to drive?\n"

635 res += "Error, couldn't extract source, WTF?\n"

636 res += "'dpkg-source -x' failed. return code: %s.\n\n" % (e.returncode)

637 res += e.output

638 return res

639

640 path = os.path.join(targetdir, "debian/README.source")