Coverage for dak/examine

1#! /usr/bin/env python3

3"""

4Script to automate some parts of checking NEW packages

6Most functions are written in a functional programming style. They

7return a string avoiding the side effect of directly printing the string

8to stdout. Those functions can be used in multithreaded parts of dak.

10@contact: Debian FTP Master <ftpmaster@debian.org>

11@copyright: 2000, 2001, 2002, 2003, 2006 James Troup <james@nocrew.org>

12@copyright: 2009 Joerg Jaspert <joerg@debian.org>

13@license: GNU General Public License version 2 or later

14"""

16# This program is free software; you can redistribute it and/or modify

17# it under the terms of the GNU General Public License as published by

18# the Free Software Foundation; either version 2 of the License, or

19# (at your option) any later version.

21# This program is distributed in the hope that it will be useful,

22# but WITHOUT ANY WARRANTY; without even the implied warranty of

23# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

24# GNU General Public License for more details.

26# You should have received a copy of the GNU General Public License

27# along with this program; if not, write to the Free Software

28# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

30################################################################################

32# <Omnic> elmo wrote docs?!!?!?!?!?!?!

33# <aj> as if he wasn't scary enough before!!

34# * aj imagines a little red furry toy sitting hunched over a computer

35# tapping furiously and giggling to himself

36# <aj> eventually he stops, and his heads slowly spins around and you

37# see this really evil grin and then he sees you, and picks up a

38# knife from beside the keyboard and throws it at you, and as you

39# breathe your last breath, he starts giggling again

40# <aj> but i should be telling this to my psychiatrist, not you guys,

41# right? :)

43################################################################################

45import errno

46import hashlib

47import html

48import os

49import re

50import sys

51import apt_pkg

52import shutil

53import subprocess

54import tarfile

55import tempfile

56import threading

58from daklib import utils

59from daklib.config import Config

60from daklib.dbconn import DBConn, get_component_by_package_suite

61from daklib.gpg import SignedFile

62from daklib.regexes import re_version, re_spacestrip, \

63 re_contrib, re_nonfree, re_localhost, re_newlinespace, \

64 re_package, re_doc_directory, re_file_binary

66################################################################################

68Cnf = None

69Cnf = utils.get_conf()

71printed = threading.local()

72printed.copyrights = {}

73package_relations = {} #: Store relations of packages for later output

75# default is to not output html.

76use_html = False

78################################################################################

81def usage(exit_code=0):

82 print("""Usage: dak examine-package [PACKAGE]...

83Check NEW package(s).

85 -h, --help show this help and exit

86 -H, --html-output output html page with inspection result

87 -f, --file-name filename for the html page

89PACKAGE can be a .changes, .dsc, .deb or .udeb filename.""")

91 sys.exit(exit_code)

93################################################################################

94# probably xml.sax.saxutils would work as well

97def escape_if_needed(s):

98 if use_html:

99 return html.escape(s)

100 else:

101 return s

102

103

104def headline(s, level=2, bodyelement=None):

105 if use_html:

106 if bodyelement: 106 ↛ 111line 106 didn't jump to line 111, because the condition on line 106 was never false

107 return """<thead>

108 <tr><th colspan="2" class="title" onclick="toggle('%(bodyelement)s', 'table-row-group', 'table-row-group')">%(title)s <span class="toggle-msg">(click to toggle)</span></th></tr>

109 </thead>\n""" % {"bodyelement": bodyelement, "title": html.escape(os.path.basename(s), quote=False)}

110 else:

111 return "<h%d>%s</h%d>\n" % (level, html.escape(s, quote=False), level)

112 else:

113 return "---- %s ----\n" % (s)

114

115

116# Colour definitions, 'end' isn't really for use

117

118ansi_colours = {

119 'main': "\033[36m",

120 'contrib': "\033[33m",

121 'nonfree': "\033[31m",

122 'provides': "\033[35m",

123 'arch': "\033[32m",

124 'end': "\033[0m",

125 'bold': "\033[1m",

126 'maintainer': "\033[32m",

127 'distro': "\033[1m\033[41m",

128 'error': "\033[1m\033[41m",

129}

130

131html_colours = {

132 'main': ('<span style="color: green">', "</span>"),

133 'contrib': ('<span style="color: orange">', "</span>"),

134 'nonfree': ('<span style="color: red">', "</span>"),

135 'provides': ('<span style="color: magenta">', "</span>"),

136 'arch': ('<span style="color: green">', "</span>"),

137 'bold': ('<span style="font-weight: bold">', "</span>"),

138 'maintainer': ('<span style="color: green">', "</span>"),

139 'distro': ('<span style="font-weight: bold; background-color: red">', "</span>"),

140 'error': ('<span style="font-weight: bold; background-color: red">', "</span>"),

141}

142

143

144def colour_output(s, colour):

145 if use_html:

146 return ("%s%s%s" % (html_colours[colour][0], html.escape(s, quote=False), html_colours[colour][1]))

147 else:

148 return ("%s%s%s" % (ansi_colours[colour], s, ansi_colours['end']))

149

150

151def escaped_text(s, strip=False):

152 if use_html:

153 if strip: 153 ↛ 154line 153 didn't jump to line 154, because the condition on line 153 was never true

154 s = s.strip()

155 return "<pre>%s</pre>" % (s)

156 else:

157 return s

158

159

160def formatted_text(s, strip=False):

161 if use_html:

162 if strip:

163 s = s.strip()

164 return "<pre>%s</pre>" % (html.escape(s, quote=False))

165 else:

166 return s

167

168

169def output_row(s):

170 if use_html:

171 return """<tr><td>""" + s + """</td></tr>"""

172 else:

173 return s

174

175

176def format_field(k, v):

177 if use_html:

178 return """<tr><td class="key">%s:</td><td class="val">%s</td></tr>""" % (k, v)

179 else:

180 return "%s: %s" % (k, v)

181

182

183def foldable_output(title, elementnameprefix, content, norow=False):

184 d = {'elementnameprefix': elementnameprefix}

185 result = ''

186 if use_html:

187 result += """<div id="%(elementnameprefix)s-wrap"><a name="%(elementnameprefix)s"></a>

188 <table class="infobox rfc822">\n""" % d

189 result += headline(title, bodyelement="%(elementnameprefix)s-body" % d)

190 if use_html:

191 result += """ <tbody id="%(elementnameprefix)s-body" class="infobody">\n""" % d

192 if norow:

193 result += content + "\n"

194 else:

195 result += output_row(content) + "\n"

196 if use_html:

197 result += """</tbody></table></div>"""

198 return result

199

200################################################################################

201

202

203def get_depends_parts(depend):

204 v_match = re_version.match(depend)

205 if v_match: 205 ↛ 208line 205 didn't jump to line 208, because the condition on line 205 was never false

206 d_parts = {'name': v_match.group(1), 'version': v_match.group(2)}

207 else:

208 d_parts = {'name': depend, 'version': ''}

209 return d_parts

210

211

212def get_or_list(depend):

213 or_list = depend.split("|")

214 return or_list

215

216

217def get_comma_list(depend):

218 dep_list = depend.split(",")

219 return dep_list

220

221

222def split_depends(d_str):

223 # creates a list of lists of dictionaries of depends (package,version relation)

224

225 d_str = re_spacestrip.sub('', d_str)

226 depends_tree = []

227 # first split depends string up amongs comma delimiter

228 dep_list = get_comma_list(d_str)

229 d = 0

230 while d < len(dep_list):

231 # put depends into their own list

232 depends_tree.append([dep_list[d]])

233 d += 1

234 d = 0

235 while d < len(depends_tree):

236 k = 0

237 # split up Or'd depends into a multi-item list

238 depends_tree[d] = get_or_list(depends_tree[d][0])

239 while k < len(depends_tree[d]):

240 # split depends into {package, version relation}

241 depends_tree[d][k] = get_depends_parts(depends_tree[d][k])

242 k += 1

243 d += 1

244 return depends_tree

245

246

247def read_control(filename):

248 recommends = []

249 predepends = []

250 depends = []

251 section = ''

252 maintainer = ''

253 arch = ''

254

255 try:

256 extracts = utils.deb_extract_control(filename)

257 control = apt_pkg.TagSection(extracts)

258 except:

259 print(formatted_text("can't parse control info"))

260 raise

261

262 control_keys = list(control.keys())

263

264 if "Pre-Depends" in control: 264 ↛ 265line 264 didn't jump to line 265, because the condition on line 264 was never true

265 predepends_str = control["Pre-Depends"]

266 predepends = split_depends(predepends_str)

267

268 if "Depends" in control: 268 ↛ 269line 268 didn't jump to line 269, because the condition on line 268 was never true

269 depends_str = control["Depends"]

270 # create list of dependancy lists

271 depends = split_depends(depends_str)

272

273 if "Recommends" in control: 273 ↛ 274line 273 didn't jump to line 274, because the condition on line 273 was never true

274 recommends_str = control["Recommends"]

275 recommends = split_depends(recommends_str)

276

277 if "Section" in control: 277 ↛ 291line 277 didn't jump to line 291, because the condition on line 277 was never false

278 section_str = control["Section"]

279

280 c_match = re_contrib.search(section_str)

281 nf_match = re_nonfree.search(section_str)

282 if c_match: 282 ↛ 284line 282 didn't jump to line 284, because the condition on line 282 was never true

283 # contrib colour

284 section = colour_output(section_str, 'contrib')

285 elif nf_match: 285 ↛ 287line 285 didn't jump to line 287, because the condition on line 285 was never true

286 # non-free colour

287 section = colour_output(section_str, 'nonfree')

288 else:

289 # main

290 section = colour_output(section_str, 'main')

291 if "Architecture" in control: 291 ↛ 295line 291 didn't jump to line 295, because the condition on line 291 was never false

292 arch_str = control["Architecture"]

293 arch = colour_output(arch_str, 'arch')

294

295 if "Maintainer" in control: 295 ↛ 304line 295 didn't jump to line 304, because the condition on line 295 was never false

296 maintainer = control["Maintainer"]

297 localhost = re_localhost.search(maintainer)

298 if localhost: 298 ↛ 300line 298 didn't jump to line 300, because the condition on line 298 was never true

299 # highlight bad email

300 maintainer = colour_output(maintainer, 'maintainer')

301 else:

302 maintainer = escape_if_needed(maintainer)

303

304 return (control, control_keys, section, predepends, depends, recommends, arch, maintainer)

305

306

307def read_changes_or_dsc(suite, filename, session=None):

308 dsc = {}

309

310 with open(filename) as dsc_file:

311 try:

312 dsc = utils.parse_changes(filename, dsc_file=True)

313 except:

314 return formatted_text("can't parse .dsc control info")

315

316 filecontents = strip_pgp_signature(filename)

317 keysinorder = []

318 for l in filecontents.split('\n'):

319 m = re.match(r'([-a-zA-Z0-9]*):', l)

320 if m:

321 keysinorder.append(m.group(1))

322

323 for k in list(dsc.keys()):

324 if k in ("build-depends", "build-depends-indep"):

325 dsc[k] = create_depends_string(suite, split_depends(dsc[k]), session)

326 elif k == "architecture":

327 if (dsc["architecture"] != "any"): 327 ↛ 323line 327 didn't jump to line 323, because the condition on line 327 was never false

328 dsc['architecture'] = colour_output(dsc["architecture"], 'arch')

329 elif k == "distribution":

330 if dsc["distribution"] not in ('unstable', 'experimental'): 330 ↛ 331line 330 didn't jump to line 331, because the condition on line 330 was never true

331 dsc['distribution'] = colour_output(dsc["distribution"], 'distro')

332 elif k in ("files", "changes", "description"):

333 if use_html:

334 dsc[k] = formatted_text(dsc[k], strip=True)

335 else:

336 dsc[k] = ('\n' + '\n'.join(' ' + x for x in dsc[k].split('\n'))).rstrip()

337 else:

338 dsc[k] = escape_if_needed(dsc[k])

339

340 filecontents = '\n'.join(format_field(x, dsc[x.lower()])

341 for x in keysinorder if not x.lower().startswith('checksums-')

342 ) + '\n'

343 return filecontents

344

345

346def get_provides(suite):

347 provides = set()

348 session = DBConn().session()

349 query = '''SELECT DISTINCT value

350 FROM binaries_metadata m

351 JOIN bin_associations b

352 ON b.bin = m.bin_id

353 WHERE key_id = (

354 SELECT key_id

355 FROM metadata_keys

356 WHERE key = 'Provides' )

357 AND b.suite = (

358 SELECT id

359 FROM suite

360 WHERE suite_name = :suite

361 OR codename = :suite)'''

362 for p in session.execute(query, {'suite': suite}): 362 ↛ 363line 362 didn't jump to line 363, because the loop on line 362 never started

363 for e in p:

364 for i in e.split(','):

365 provides.add(i.strip())

366 session.close()

367 return provides

368

369

370def create_depends_string(suite, depends_tree, session=None):

371 result = ""

372 if suite == 'experimental': 372 ↛ 373line 372 didn't jump to line 373, because the condition on line 372 was never true

373 suite_list = ['experimental', 'unstable']

374 else:

375 suite_list = [suite]

376

377 provides = set()

378 comma_count = 1

379 for l in depends_tree:

380 if (comma_count >= 2): 380 ↛ 381line 380 didn't jump to line 381, because the condition on line 380 was never true

381 result += ", "

382 or_count = 1

383 for d in l:

384 if (or_count >= 2): 384 ↛ 385line 384 didn't jump to line 385, because the condition on line 384 was never true

385 result += " | "

386 # doesn't do version lookup yet.

387

388 component = get_component_by_package_suite(d['name'], suite_list,

389 session=session)

390 if component is not None: 390 ↛ 391line 390 didn't jump to line 391, because the condition on line 390 was never true

391 adepends = d['name']

392 if d['version'] != '':

393 adepends += " (%s)" % (d['version'])

394

395 if component == "contrib":

396 result += colour_output(adepends, "contrib")

397 elif component in ("non-free-firmware", "non-free"):

398 result += colour_output(adepends, "nonfree")

399 else:

400 result += colour_output(adepends, "main")

401 else:

402 adepends = d['name']

403 if d['version'] != '': 403 ↛ 405line 403 didn't jump to line 405, because the condition on line 403 was never false

404 adepends += " (%s)" % (d['version'])

405 if not provides: 405 ↛ 407line 405 didn't jump to line 407, because the condition on line 405 was never false

406 provides = get_provides(suite)

407 if d['name'] in provides: 407 ↛ 408line 407 didn't jump to line 408, because the condition on line 407 was never true

408 result += colour_output(adepends, "provides")

409 else:

410 result += colour_output(adepends, "bold")

411 or_count += 1

412 comma_count += 1

413 return result

414

415

416def output_package_relations():

417 """

418 Output the package relations, if there is more than one package checked in this run.

419 """

420

421 if len(package_relations) < 2: 421 ↛ 426line 421 didn't jump to line 426, because the condition on line 421 was never false

422 # Only list something if we have more than one binary to compare

423 package_relations.clear()

424 result = ""

425 else:

426 to_print = ""

427 for package in package_relations:

428 for relation in package_relations[package]:

429 to_print += "%-15s: (%s) %s\n" % (package, relation, package_relations[package][relation])

430

431 package_relations.clear()

432 result = foldable_output("Package relations", "relations", to_print)

433 package_relations.clear()

434 return result

435

436

437def output_deb_info(suite, filename, packagename, session=None):

438 (control, control_keys, section, predepends, depends, recommends, arch, maintainer) = read_control(filename)

439

440 if control == '': 440 ↛ 441line 440 didn't jump to line 441, because the condition on line 440 was never true

441 return formatted_text("no control info")

442 to_print = ""

443 if packagename not in package_relations: 443 ↛ 445line 443 didn't jump to line 445, because the condition on line 443 was never false

444 package_relations[packagename] = {}

445 for key in control_keys:

446 if key == 'Source': 446 ↛ 447line 446 didn't jump to line 447, because the condition on line 446 was never true

447 field_value = escape_if_needed(control.find(key))

448 if use_html:

449 field_value = '<a href="https://tracker.debian.org/pkg/{0}" rel="nofollow">{0}</a>'.format(

450 field_value)

451 elif key == 'Pre-Depends': 451 ↛ 452line 451 didn't jump to line 452, because the condition on line 451 was never true

452 field_value = create_depends_string(suite, predepends, session)

453 package_relations[packagename][key] = field_value

454 elif key == 'Depends': 454 ↛ 455line 454 didn't jump to line 455, because the condition on line 454 was never true

455 field_value = create_depends_string(suite, depends, session)

456 package_relations[packagename][key] = field_value

457 elif key == 'Recommends': 457 ↛ 458line 457 didn't jump to line 458, because the condition on line 457 was never true

458 field_value = create_depends_string(suite, recommends, session)

459 package_relations[packagename][key] = field_value

460 elif key == 'Section':

461 field_value = section

462 elif key == 'Architecture':

463 field_value = arch

464 elif key == 'Maintainer':

465 field_value = maintainer

466 elif key in ('Homepage', 'Vcs-Browser'): 466 ↛ 467line 466 didn't jump to line 467, because the condition on line 466 was never true

467 field_value = escape_if_needed(control.find(key))

468 if use_html:

469 field_value = '<a href="%s" rel="nofollow">%s</a>' % \

470 (field_value, field_value)

471 elif key == 'Description':

472 if use_html:

473 field_value = formatted_text(control.find(key), strip=True)

474 else:

475 desc = control.find(key)

476 desc = re_newlinespace.sub('\n ', desc)

477 field_value = escape_if_needed(desc)

478 else:

479 field_value = escape_if_needed(control.find(key))

480 to_print += " " + format_field(key, field_value) + '\n'

481 return to_print

482

483

484def do_command(command, escaped=False):

485 result = subprocess.run(command, stdout=subprocess.PIPE, text=True)

486 if escaped:

487 return escaped_text(result.stdout)

488 else:

489 return formatted_text(result.stdout)

490

491

492def do_lintian(filename):

493 cnf = Config()

494 cmd = []

495

496 user = cnf.get('Dinstall::UnprivUser') or None

497 if user is not None: 497 ↛ 498line 497 didn't jump to line 498, because the condition on line 497 was never true

498 cmd.extend(['sudo', '-H', '-u', user])

499

500 color = 'always'

501 if use_html:

502 color = 'html'

503

504 cmd.extend(['lintian', '--show-overrides', '--color', color, "--", filename])

505

506 try:

507 return do_command(cmd, escaped=True)

508 except OSError as e:

509 return (colour_output("Running lintian failed: %s" % (e), "error"))

510

511

512def extract_one_file_from_deb(deb_filename, match):

513 with tempfile.TemporaryFile() as tmpfh:

514 dpkg_cmd = ('dpkg-deb', '--fsys-tarfile', deb_filename)

515 subprocess.check_call(dpkg_cmd, stdout=tmpfh)

516

517 tmpfh.seek(0)

518 with tarfile.open(fileobj=tmpfh, mode="r") as tar:

519 matched_member = None

520 for member in tar: 520 ↛ 525line 520 didn't jump to line 525, because the loop on line 520 didn't complete

521 if member.isfile() and match.match(member.name):

522 matched_member = member

523 break

524

525 if not matched_member: 525 ↛ 526line 525 didn't jump to line 526, because the condition on line 525 was never true

526 return None, None

527

528 fh = tar.extractfile(matched_member)

529 matched_data = fh.read()

530 fh.close()

531

532 return matched_member.name, matched_data

533

534

535def get_copyright(deb_filename):

536 global printed

537

538 re_copyright = re.compile(r"\./usr(/share)?/doc/(?P<package>[^/]+)/copyright")

539 cright_path, cright = extract_one_file_from_deb(deb_filename, re_copyright)

540

541 if not cright_path: 541 ↛ 542line 541 didn't jump to line 542, because the condition on line 541 was never true

542 return formatted_text("WARNING: No copyright found, please check package manually.")

543

544 package = re_file_binary.match(os.path.basename(deb_filename)).group('package')

545 doc_directory = re_copyright.match(cright_path).group('package')

546 if package != doc_directory: 546 ↛ 547line 546 didn't jump to line 547, because the condition on line 546 was never true

547 return formatted_text("WARNING: wrong doc directory (expected %s, got %s)." % (package, doc_directory))

548

549 copyrightmd5 = hashlib.md5(cright).hexdigest()

550

551 res = ""

552 if copyrightmd5 in printed.copyrights and printed.copyrights[copyrightmd5] != "%s (%s)" % (package, os.path.basename(deb_filename)): 552 ↛ 553line 552 didn't jump to line 553, because the condition on line 552 was never true

553 res += formatted_text("NOTE: Copyright is the same as %s.\n\n" %

554 (printed.copyrights[copyrightmd5]))

555 else:

556 printed.copyrights[copyrightmd5] = "%s (%s)" % (package, os.path.basename(deb_filename))

557 return res + formatted_text(cright.decode())

558

559

560def get_readme_source(dsc_filename):

561 with tempfile.TemporaryDirectory(prefix="dak-examine-package") as tempdir:

562 targetdir = os.path.join(tempdir, "source")

563

564 cmd = ('dpkg-source', '--no-check', '--no-copy', '-x', dsc_filename, targetdir)

565 try:

566 subprocess.check_output(cmd, stderr=subprocess.STDOUT)

567 except subprocess.CalledProcessError as e:

568 res = "How is education supposed to make me feel smarter? Besides, every time I learn something new, it pushes some\n old stuff out of my brain. Remember when I took that home winemaking course, and I forgot how to drive?\n"

569 res += "Error, couldn't extract source, WTF?\n"

570 res += "'dpkg-source -x' failed. return code: %s.\n\n" % (e.returncode)

571 res += e.output

572 return res

573

574 path = os.path.join(targetdir, 'debian/README.source')