Coverage for dak/check_archive.py: 14%

1"""Various different sanity checks

3@contact: Debian FTP Master <ftpmaster@debian.org>

5@license: GNU General Public License version 2 or later

6"""

8# This program is free software; you can redistribute it and/or modify

9# it under the terms of the GNU General Public License as published by

10# the Free Software Foundation; either version 2 of the License, or

11# (at your option) any later version.

13# This program is distributed in the hope that it will be useful,

14# but WITHOUT ANY WARRANTY; without even the implied warranty of

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

16# GNU General Public License for more details.

18# You should have received a copy of the GNU General Public License

19# along with this program; if not, write to the Free Software

20# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

22################################################################################

24# And, lo, a great and menacing voice rose from the depths, and with

25# great wrath and vehemence it's voice boomed across the

26# land... ``hehehehehehe... that *tickles*''

27# -- aj on IRC

29################################################################################

31import errno

32import os

33import stat

34import sys

35import time

36from collections.abc import Iterable

37from typing import NoReturn, cast

39import apt_pkg

40from sqlalchemy import select, sql

41from sqlalchemy.engine import CursorResult

43from daklib import utils

44from daklib.config import Config

45from daklib.dak_exceptions import InvalidDscError

46from daklib.dbconn import (

47 Archive,

48 ArchiveFile,

49 DBConn,

50 DBSource,

51 DSCFile,

52 PoolFile,

53 get_component_names,

54 get_or_set_metadatakey,

55 get_suite,

56 get_suite_architectures,

57)

59################################################################################

61db_files: dict = {} #: Cache of filenames as known by the database

62waste = 0.0 #: How many bytes are "wasted" by files not referenced in database

63excluded: dict = {} #: List of files which are excluded from files check

64current_file: str | None = None

65future_files: dict[str, int] = {}

66current_time = time.time() #: now()

68################################################################################

71def usage(exit_code=0) -> NoReturn:

72 print(

73 """Usage: dak check-archive MODE

74Run various sanity checks of the archive and/or database.

76 -h, --help show this help and exit.

78The following MODEs are available:

80 checksums - validate the checksums stored in the database

81 files - check files in the database against what's in the archive

82 dsc-syntax - validate the syntax of .dsc files in the archive

83 missing-overrides - check for missing overrides

84 source-in-one-dir - ensure the source for each package is in one directory

85 timestamps - check for future timestamps in .deb's

86 files-in-dsc - ensure each .dsc references appropriate Files

87 validate-indices - ensure files mentioned in Packages & Sources exist

88 files-not-symlinks - check files in the database aren't symlinks

89 validate-builddeps - validate build-dependencies of .dsc files in the archive

90 add-missing-source-checksums - add missing checksums for source packages

91"""

92 )

93 sys.exit(exit_code)

96################################################################################

99def check_files() -> None:

100 """

101 Prepare the dictionary of existing filenames, then walk through the archive

102 pool/ directory to compare it.

103 """

104 session = DBConn().session()

105

106 query = """

107 SELECT archive.name, suite.suite_name, f.filename

108 FROM binaries b

109 JOIN bin_associations ba ON b.id = ba.bin

110 JOIN suite ON ba.suite = suite.id

111 JOIN archive ON suite.archive_id = archive.id

112 JOIN files f ON b.file = f.id

113 WHERE NOT EXISTS (SELECT 1 FROM files_archive_map af

114 WHERE af.archive_id = suite.archive_id

115 AND af.file_id = b.file)

116 ORDER BY archive.name, suite.suite_name, f.filename

117 """

118 for row in session.execute(sql.text(query)):

119 print("MISSING-ARCHIVE-FILE {0} {1} {2}".format(*row))

120

121 query = """

122 SELECT archive.name, suite.suite_name, f.filename

123 FROM source s

124 JOIN src_associations sa ON s.id = sa.source

125 JOIN suite ON sa.suite = suite.id

126 JOIN archive ON suite.archive_id = archive.id

127 JOIN dsc_files df ON s.id = df.source

128 JOIN files f ON df.file = f.id

129 WHERE NOT EXISTS (SELECT 1 FROM files_archive_map af

130 WHERE af.archive_id = suite.archive_id

131 AND af.file_id = df.file)

132 ORDER BY archive.name, suite.suite_name, f.filename

133 """

134 for row in session.execute(sql.text(query)):

135 print("MISSING-ARCHIVE-FILE {0} {1} {2}".format(*row))

136

137 archive_files = session.scalars(

138 select(ArchiveFile)

139 .join(ArchiveFile.archive)

140 .join(ArchiveFile.file)

141 .order_by(Archive.archive_name, PoolFile.filename)

142 )

143

144 expected_files = set()

145 for af in archive_files:

146 path = af.path

147 expected_files.add(af.path)

148 if not os.path.exists(path):

149 print(

150 "MISSING-FILE {0} {1} {2}".format(

151 af.archive.archive_name, af.file.filename, path

152 )

153 )

154

155 archives = session.scalars(select(Archive).order_by(Archive.archive_name))

156

157 for a in archives:

158 top = os.path.join(a.path, "pool")

159 for dirpath, dirnames, filenames in os.walk(top):

160 for fn in filenames:

161 path = os.path.join(dirpath, fn)

162 if path in expected_files:

163 continue

164 print("UNEXPECTED-FILE {0} {1}".format(a.archive_name, path))

165

166

167################################################################################

168

169

170def check_dscs() -> None:

171 """

172 Parse every .dsc file in the archive and check for it's validity.

173 """

174

175 count = 0

176

177 for src in (

178 DBConn()

179 .session()

180 .scalars(select(DBSource).order_by(DBSource.source, DBSource.version))

181 ):

182 f = src.poolfile.fullpath

183 try:

184 utils.parse_changes(f, signing_rules=1, dsc_file=True)

185 except InvalidDscError:

186 utils.warn("syntax error in .dsc file %s" % f)

187 count += 1

188 except UnicodeDecodeError:

189 utils.warn("found invalid dsc file (%s), not properly utf-8 encoded" % f)

190 count += 1

191 except OSError as e:

192 if e.errno == errno.ENOENT:

193 utils.warn("missing dsc file (%s)" % f)

194 count += 1

195 else:

196 raise

197 except Exception as e:

198 utils.warn("miscellaneous error parsing dsc file (%s): %s" % (f, str(e)))

199 count += 1

200

201 if count:

202 utils.warn("Found %s invalid .dsc files." % (count))

203

204

205################################################################################

206

207

208def check_override() -> None:

209 """

210 Check for missing overrides in stable and unstable.

211 """

212 session = DBConn().session()

213

214 for suite_name in ["stable", "unstable"]:

215 print(suite_name)

216 print("-" * len(suite_name))

217 print()

218 suite = get_suite(suite_name)

219 assert suite is not None

220 q = session.execute(

221 sql.text(

222 """

223SELECT DISTINCT b.package FROM binaries b, bin_associations ba

224 WHERE b.id = ba.bin AND ba.suite = :suiteid AND NOT EXISTS

225 (SELECT 1 FROM override o WHERE o.suite = :suiteid AND o.package = b.package)"""

226 ),

227 {"suiteid": suite.suite_id},

228 )

229

230 for j in q.fetchall():

231 print(j[0])

232

233 q = session.execute(

234 sql.text(

235 """

236SELECT DISTINCT s.source FROM source s, src_associations sa

237 WHERE s.id = sa.source AND sa.suite = :suiteid AND NOT EXISTS

238 (SELECT 1 FROM override o WHERE o.suite = :suiteid and o.package = s.source)"""

239 ),

240 {"suiteid": suite.suite_id},

241 )

242 for j in q.fetchall():

243 print(j[0])

244

245

246################################################################################

247

248

249def check_source_in_one_dir() -> None:

250 """

251 Ensure that the source files for any given package is all in one

252 directory so that 'apt-get source' works...

253 """

254

255 cnf = Config()

256

257 # Not the most enterprising method, but hey...

258 broken_count = 0

259

260 session = DBConn().session()

261

262 for s in session.scalars(select(DBSource)):

263 first_path = ""

264 first_filename = ""

265 broken = False

266

267 qf = select(PoolFile).join(DSCFile).where(DSCFile.source_id == s.source_id)

268 for f in session.scalars(qf):

269 # 0: path

270 # 1: filename

271 filename = os.path.join(cnf["Dir::Root"], f.filename)

272 path = os.path.dirname(filename)

273

274 if first_path == "":

275 first_path = path

276 first_filename = filename

277 elif first_path != path:

278 symlink = path + "/" + os.path.basename(first_filename)

279 if not os.path.exists(symlink):

280 broken = True

281 print(

282 "WOAH, we got a live one here... %s [%s] {%s}"

283 % (filename, s.source_id, symlink)

284 )

285 if broken:

286 broken_count += 1

287

288 print(

289 "Found %d source packages where the source is not all in one directory."

290 % (broken_count)

291 )

292

293

294################################################################################

295

296

297def check_checksums() -> None:

298 """

299 Validate all files

300 """

301 print("Getting file information from database...")

302 q = DBConn().session().scalars(select(PoolFile))

303

304 print("Checking file checksums & sizes...")

305 for f in q:

306 filename = f.fullpath

307

308 try:

309 fi = open(filename)

310 except:

311 utils.warn("can't open '%s'." % (filename))

312 continue

313

314 size = os.stat(filename)[stat.ST_SIZE]

315 if size != f.filesize:

316 utils.warn(

317 "**WARNING** size mismatch for '%s' ('%s' [current] vs. '%s' [db])."

318 % (filename, size, f.filesize)

319 )

320

321 md5sum = apt_pkg.md5sum(fi)

322 if md5sum != f.md5sum:

323 utils.warn(

324 "**WARNING** md5sum mismatch for '%s' ('%s' [current] vs. '%s' [db])."

325 % (filename, md5sum, f.md5sum)

326 )

327

328 fi.seek(0)

329 sha1sum = apt_pkg.sha1sum(fi) # type: ignore[attr-defined]

330 if sha1sum != f.sha1sum:

331 utils.warn(

332 "**WARNING** sha1sum mismatch for '%s' ('%s' [current] vs. '%s' [db])."

333 % (filename, sha1sum, f.sha1sum)

334 )

335

336 fi.seek(0)

337 sha256sum = apt_pkg.sha256sum(fi) # type: ignore[attr-defined]

338 if sha256sum != f.sha256sum:

339 utils.warn(

340 "**WARNING** sha256sum mismatch for '%s' ('%s' [current] vs. '%s' [db])."

341 % (filename, sha256sum, f.sha256sum)

342 )

343 fi.close()

344

345 print("Done.")

346

347

348################################################################################

349#

350

351

352def Ent(Kind, Name, Link, Mode, UID, GID, Size, MTime: int, Major, Minor) -> None:

353 global future_files

354 assert current_file is not None

355

356 if MTime > current_time:

357 future_files[current_file] = MTime

358 print(

359 "%s: %s '%s','%s',%u,%u,%u,%u,%u,%u,%u"

360 % (

361 current_file,

362 Kind,

363 Name,

364 Link,

365 Mode,

366 UID,

367 GID,

368 Size,

369 MTime,

370 Major,

371 Minor,

372 )

373 )

374

375

376def check_timestamps() -> None:

377 """

378 Check all files for timestamps in the future; common from hardware

379 (e.g. alpha) which have far-future dates as their default dates.

380 """

381 return

382

383 # global current_file

384 #

385 # q = DBConn().session().query(PoolFile).filter(PoolFile.filename.like(".deb$"))

386 #

387 # db_files.clear()

388 # count = 0

389 #

390 # for pf in q.all():

391 # filename = os.path.abspath(os.path.join(pf.location.path, pf.filename))

392 # if os.access(filename, os.R_OK):

393 # with open(filename) as f:

394 # current_file = filename

395 # print("Processing %s." % (filename), file=sys.stderr)

396 # apt_inst.debExtract(f, Ent, "control.tar.gz")

397 # f.seek(0)

398 # apt_inst.debExtract(f, Ent, "data.tar.gz")

399 # count += 1

400 #

401 # print("Checked %d files (out of %d)." % (count, len(db_files)))

402

403

404################################################################################

405

406

407def check_files_in_dsc() -> None:

408 """

409 Ensure each .dsc lists appropriate files in its Files field (according

410 to the format announced in its Format field).

411 """

412 return

413

414 # count = 0

415 #

416 # print("Building list of database files...")

417 # q = DBConn().session().query(PoolFile).filter(PoolFile.filename.like(".dsc$"))

418 #

419 # if q.count() > 0:

420 # print("Checking %d files..." % q.count())

421 # else:

422 # print("No files to check.")

423 #

424 # cnf = Config()

425 # for pf in q.all():

426 # filename = os.path.abspath(os.path.join(cnf["Dir::Root"], pf.filename))

427 #

428 # try:

429 # # NB: don't enforce .dsc syntax

430 # dsc = utils.parse_changes(filename, dsc_file=True)

431 # except:

432 # utils.fubar("error parsing .dsc file '%s'." % (filename))

433 #

434 # reasons = utils.check_dsc_files(filename, dsc)

435 # for r in reasons:

436 # utils.warn(r)

437 #

438 # if len(reasons) > 0:

439 # count += 1

440 #

441 # if count:

442 # utils.warn("Found %s invalid .dsc files." % (count))

443

444

445################################################################################

446

447

448def validate_sources(suite: str, component: str) -> None:

449 """

450 Ensure files mentioned in Sources exist

451 """

452 cnf = Config()

453 filename = "%s/dists/%s/%s/source/Sources" % (cnf["Dir::Root"], suite, component)

454 filename = utils.find_possibly_compressed_file(filename)

455 print("Processing %s..." % (filename))

456 with apt_pkg.TagFile(filename) as Sources:

457 while Sources.step(): # type: ignore[attr-defined]

458 section: apt_pkg.TagSection = Sources.section # type: ignore[attr-defined]

459 source = section.find("Package")

460 directory = section.find("Directory")

461 files = section.find("Files")

462 for i in files.split("\n"):

463 (md5, size, name) = i.split()

464 filename = "%s/%s/%s" % (cnf["Dir::Root"], directory, name)

465 if not os.path.exists(filename):

466 if directory.find("potato") == -1:

467 print("W: %s missing." % (filename))

468 else:

469 pool_location = utils.poolify(source)

470 pool_filename = "%s/%s/%s" % (

471 cnf["Dir::Pool"],

472 pool_location,

473 name,

474 )

475 if not os.path.exists(pool_filename):

476 print("E: %s missing (%s)." % (filename, pool_filename))

477 else:

478 # Create symlink

479 pool_filename = os.path.normpath(pool_filename)

480 filename = os.path.normpath(filename)

481 src = utils.clean_symlink(

482 pool_filename, filename, cnf["Dir::Root"]

483 )

484 print("Symlinking: %s -> %s" % (filename, src))

485

486

487########################################

488

489

490def validate_packages(suite: str, component: str, architecture: str) -> None:

491 """

492 Ensure files mentioned in Packages exist

493 """

494 cnf = Config()

495 filename = "%s/dists/%s/%s/binary-%s/Packages" % (

496 cnf["Dir::Root"],

497 suite,

498 component,

499 architecture,

500 )

501 filename = utils.find_possibly_compressed_file(filename)

502 print("Processing %s..." % (filename))

503 with apt_pkg.TagFile(filename) as Packages:

504 while Packages.step(): # type: ignore[attr-defined]

505 section: apt_pkg.TagSection = Packages.section # type: ignore[attr-defined]

506 filename = "%s/%s" % (cnf["Dir::Root"], section.find("Filename"))

507 if not os.path.exists(filename):

508 print("W: %s missing." % (filename))

509

510

511########################################

512

513

514def check_indices_files_exist() -> None:

515 """

516 Ensure files mentioned in Packages & Sources exist

517 """

518 for suite in ["stable", "testing", "unstable"]:

519 for component in get_component_names():

520 architectures = get_suite_architectures(suite)

521 for arch in [i.arch_string.lower() for i in architectures]:

522 if arch == "source":

523 validate_sources(suite, component)

524 elif arch == "all":

525 continue

526 else:

527 validate_packages(suite, component, arch)

528

529

530################################################################################

531

532

533def check_files_not_symlinks() -> None:

534 """

535 Check files in the database aren't symlinks

536 """

537 return

538

539 # print("Building list of database files... ", end=" ")

540 # q = DBConn().session().query(PoolFile).filter(PoolFile.filename.like(".dsc$"))

541 #

542 # for pf in q.all():

543 # filename = os.path.abspath(os.path.join(pf.location.path, pf.filename))

544 # if os.access(filename, os.R_OK) == 0:

545 # utils.warn("%s: doesn't exist." % (filename))

546 # else:

547 # if os.path.islink(filename):

548 # utils.warn("%s: is a symlink." % (filename))

549

550

551################################################################################

552

553

554def chk_bd_process_dir(dirname: str, filenames: Iterable[str]) -> None:

555 for name in filenames:

556 if not name.endswith(".dsc"):

557 continue

558 filename = os.path.abspath(dirname + "/" + name)

559 dsc = utils.parse_changes(filename, dsc_file=True)

560 for field_name in ["build-depends", "build-depends-indep"]:

561 field = dsc.get(field_name)

562 if field:

563 try:

564 apt_pkg.parse_src_depends(field)

565 except:

566 print("E: [%s] %s: %s" % (filename, field_name, field))

567

568

569################################################################################

570

571

572def check_build_depends() -> None:

573 """Validate build-dependencies of .dsc files in the archive"""

574 cnf = Config()

575 for dirpath, dirnames, filenames in os.walk(cnf["Dir::Root"]):

576 chk_bd_process_dir(dirpath, filenames)

577

578

579################################################################################

580

581

582_add_missing_source_checksums_query = R"""

583INSERT INTO source_metadata

584 (src_id, key_id, value)

585SELECT

586 s.id,

587 :checksum_key,

588 E'\n' ||

589 (SELECT STRING_AGG(' ' || tmp.checksum || ' ' || tmp.size || ' ' || tmp.basename, E'\n' ORDER BY tmp.basename)

590 FROM

591 (SELECT

592 CASE :checksum_type

593 WHEN 'Files' THEN f.md5sum

594 WHEN 'Checksums-Sha1' THEN f.sha1sum

595 WHEN 'Checksums-Sha256' THEN f.sha256sum

596 END AS checksum,

597 f.size,

598 SUBSTRING(f.filename FROM E'/([^/]*)\\Z') AS basename

599 FROM files f JOIN dsc_files ON f.id = dsc_files.file

600 WHERE dsc_files.source = s.id AND f.id != s.file

601 ) AS tmp

602 )

603

604 FROM

605 source s

606 WHERE NOT EXISTS (SELECT 1 FROM source_metadata md WHERE md.src_id=s.id AND md.key_id = :checksum_key);

607"""

608

609

610def add_missing_source_checksums() -> None:

611 """Add missing source checksums to source_metadata"""

612 session = DBConn().session()

613 for checksum in ["Files", "Checksums-Sha1", "Checksums-Sha256"]:

614 checksum_key = get_or_set_metadatakey(checksum, session).key_id

615 rows = cast(

616 CursorResult,

617 session.execute(

618 sql.text(_add_missing_source_checksums_query),

619 {"checksum_key": checksum_key, "checksum_type": checksum},

620 ),

621 ).rowcount

622 if rows > 0:

623 print("Added {0} missing entries for {1}".format(rows, checksum))

624 session.commit()

625

626

627################################################################################

628

629

630def main() -> None:

631 global db_files, waste, excluded

632

633 cnf = Config()

634

635 Arguments = [("h", "help", "Check-Archive::Options::Help")]

636 for i in ["help"]:

637 key = "Check-Archive::Options::%s" % i

638 if key not in cnf: 638 ↛ 636line 638 didn't jump to line 636 because the condition on line 638 was always true

639 cnf[key] = ""

640

641 args = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) # type: ignore[attr-defined]

642

643 Options = cnf.subtree("Check-Archive::Options")

644 if Options["Help"]: 644 ↛ 647line 644 didn't jump to line 647 because the condition on line 644 was always true

645 usage()

646

647 if len(args) < 1:

648 utils.warn("dak check-archive requires at least one argument")

649 usage(1)

650 elif len(args) > 1:

651 utils.warn("dak check-archive accepts only one argument")

652 usage(1)

653 mode = args[0].lower()

654

655 # Initialize DB

656 DBConn()

657

658 if mode == "checksums":

659 check_checksums()

660 elif mode == "files":

661 check_files()

662 elif mode == "dsc-syntax":

663 check_dscs()

664 elif mode == "missing-overrides":

665 check_override()

666 elif mode == "source-in-one-dir":

667 check_source_in_one_dir()

668 elif mode == "timestamps":

669 check_timestamps()

670 elif mode == "files-in-dsc":

671 check_files_in_dsc()

672 elif mode == "validate-indices":

673 check_indices_files_exist()

674 elif mode == "files-not-symlinks":

675 check_files_not_symlinks()

676 elif mode == "validate-builddeps":

677 check_build_depends()

678 elif mode == "add-missing-source-checksums":

679 add_missing_source_checksums()

680 else:

681 utils.warn("unknown mode '%s'" % (mode))

682 usage(1)