Coverage for dak/generate_releases.py: 88%

1#! /usr/bin/env python3

3"""

4Create all the Release files

6@contact: Debian FTPMaster <ftpmaster@debian.org>

7@copyright: 2011 Joerg Jaspert <joerg@debian.org>

8@copyright: 2011 Mark Hymers <mhy@debian.org>

9@license: GNU General Public License version 2 or later

11"""

13# This program is free software; you can redistribute it and/or modify

14# it under the terms of the GNU General Public License as published by

15# the Free Software Foundation; either version 2 of the License, or

16# (at your option) any later version.

18# This program is distributed in the hope that it will be useful,

19# but WITHOUT ANY WARRANTY; without even the implied warranty of

20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

21# GNU General Public License for more details.

23# You should have received a copy of the GNU General Public License

24# along with this program; if not, write to the Free Software

25# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

27################################################################################

29# <mhy> I wish they wouldnt leave biscuits out, thats just tempting. Damnit.

31################################################################################

33import bz2

34import errno

35import gzip

36import os

37import os.path

38import subprocess

39import sys

40import time

41from collections.abc import Callable

42from typing import TYPE_CHECKING, Literal, NoReturn, Protocol, cast

44import apt_pkg

45from sqlalchemy import sql

46from sqlalchemy.orm import object_session

48import daklib.gpg

49from daklib import daklog, utils

50from daklib.config import Config

51from daklib.dakmultiprocessing import PROC_STATUS_SUCCESS, DakProcessPool

52from daklib.dbconn import Archive, DBConn, Suite, get_suite, get_suite_architectures

53from daklib.regexes import (

54 re_gensubrelease,

55 re_includeinrelease_byhash,

56 re_includeinrelease_plain,

57)

59if TYPE_CHECKING:

60 from sqlalchemy.orm import Session

62################################################################################

63Logger = None #: Our logging object

65################################################################################

68def usage(exit_code=0) -> NoReturn:

69 """Usage information"""

71 print(

72 """Usage: dak generate-releases [OPTIONS]

73Generate the Release files

75 -a, --archive=ARCHIVE process suites in ARCHIVE

76 -s, --suite=SUITE(s) process this suite

77 Default: All suites not marked 'untouchable'

78 -f, --force Allow processing of untouchable suites

79 CAREFUL: Only to be used at (point) release time!

80 -h, --help show this help and exit

81 -q, --quiet Don't output progress

83SUITE can be a space separated list, e.g.

84 --suite=unstable testing

85 """

86 )

87 sys.exit(exit_code)

90########################################################################

93def sign_release_dir(suite: Suite, dirname: str) -> None:

94 cnf = Config()

96 if "Dinstall::SigningHomedir" in cnf: 96 ↛ exitline 96 didn't return from function 'sign_release_dir' because the condition on line 96 was always true

97 args = {

98 "keyids": suite.signingkeys or [],

99 "pubring": cnf.get("Dinstall::SigningPubKeyring") or None,

100 "homedir": cnf.get("Dinstall::SigningHomedir") or None,

101 "passphrase_file": cnf.get("Dinstall::SigningPassphraseFile") or None,

102 }

103

104 relname = os.path.join(dirname, "Release")

105

106 dest = os.path.join(dirname, "Release.gpg")

107 if os.path.exists(dest):

108 os.unlink(dest)

109

110 inlinedest = os.path.join(dirname, "InRelease")

111 if os.path.exists(inlinedest):

112 os.unlink(inlinedest)

113

114 with open(relname, "r") as stdin:

115 with open(dest, "w") as stdout:

116 daklib.gpg.sign(stdin, stdout, inline=False, **args) # type: ignore[arg-type]

117 stdin.seek(0)

118 with open(inlinedest, "w") as stdout:

119 daklib.gpg.sign(stdin, stdout, inline=True, **args) # type: ignore[arg-type]

120

121

122class _Reader(Protocol):

123 def read(self) -> bytes: ... # noqa: E704 123 ↛ exitline 123 didn't jump to line 123 because

124

125

126class XzFile:

127 def __init__(self, filename: str, mode="r"):

128 self.filename = filename

129

130 def read(self) -> bytes:

131 with open(self.filename, "rb") as stdin:

132 return subprocess.check_output(["xz", "-d"], stdin=stdin)

133

134

135class ZstdFile:

136 def __init__(self, filename: str, mode="r"):

137 self.filename = filename

138

139 def read(self) -> bytes:

140 with open(self.filename, "rb") as stdin:

141 return subprocess.check_output(["zstd", "--decompress"], stdin=stdin)

142

143

144class HashFunc:

145 def __init__(self, release_field: str, func: Callable[[bytes], str], db_name: str):

146 self.release_field = release_field

147 self.func = func

148 self.db_name = db_name

149

150

151RELEASE_HASHES = [

152 HashFunc("MD5Sum", apt_pkg.md5sum, "md5sum"),

153 HashFunc("SHA1", apt_pkg.sha1sum, "sha1"), # type: ignore[attr-defined]

154 HashFunc("SHA256", apt_pkg.sha256sum, "sha256"), # type: ignore[attr-defined]

155]

156

157

158class ReleaseWriter:

159 def __init__(self, suite: Suite):

160 self.suite = suite

161

162 def suite_path(self) -> str:

163 """

164 Absolute path to the suite-specific files.

165 """

166 suite_suffix = utils.suite_suffix(self.suite.suite_name)

167

168 return os.path.join(

169 self.suite.archive.path, "dists", self.suite.suite_name, suite_suffix

170 )

171

172 def suite_release_path(self) -> str:

173 """

174 Absolute path where Release files are physically stored.

175 This should be a path that sorts after the dists/ directory.

176 """

177 suite_suffix = utils.suite_suffix(self.suite.suite_name)

178

179 return os.path.join(

180 self.suite.archive.path,

181 "zzz-dists",

182 self.suite.codename or self.suite.suite_name,

183 suite_suffix,

184 )

185

186 def create_release_symlinks(self) -> None:

187 """

188 Create symlinks for Release files.

189 This creates the symlinks for Release files in the `suite_path`

190 to the actual files in `suite_release_path`.

191 """

192 relpath = os.path.relpath(self.suite_release_path(), self.suite_path())

193 for f in ("Release", "Release.gpg", "InRelease"):

194 source = os.path.join(relpath, f)

195 dest = os.path.join(self.suite_path(), f)

196 if os.path.lexists(dest):

197 if not os.path.islink(dest): 197 ↛ 198line 197 didn't jump to line 198 because the condition on line 197 was never true

198 os.unlink(dest)

199 elif os.readlink(dest) == source: 199 ↛ 202line 199 didn't jump to line 202 because the condition on line 199 was always true

200 continue

201 else:

202 os.unlink(dest)

203 os.symlink(source, dest)

204

205 def create_output_directories(self) -> None:

206 for path in (self.suite_path(), self.suite_release_path()):

207 try:

208 os.makedirs(path)

209 except OSError as e:

210 if e.errno != errno.EEXIST: 210 ↛ 211line 210 didn't jump to line 211 because the condition on line 210 was never true

211 raise

212

213 def _update_hashfile_table(

214 self,

215 session: "Session",

216 fileinfo: dict[str, dict[str, str | int]],

217 hashes: list[HashFunc],

218 ) -> None:

219 # Mark all by-hash files as obsolete. We will undo that for the ones

220 # we still reference later.

221 query = """

222 UPDATE hashfile SET unreferenced = CURRENT_TIMESTAMP

223 WHERE suite_id = :id AND unreferenced IS NULL"""

224 session.execute(sql.text(query), {"id": self.suite.suite_id})

225

226 query = "SELECT path FROM hashfile WHERE suite_id = :id"

227 q = session.execute(sql.text(query), {"id": self.suite.suite_id})

228 known_hashfiles = set(row[0] for row in q)

229 updated = set()

230 new = set()

231

232 # Update the hashfile table with new or updated files

233 for filename in fileinfo:

234 if not os.path.lexists(filename): 234 ↛ 236line 234 didn't jump to line 236 because the condition on line 234 was never true

235 # probably an uncompressed index we didn't generate

236 continue

237 byhashdir = os.path.join(os.path.dirname(filename), "by-hash")

238 for h in hashes:

239 field = h.release_field

240 hashfile = os.path.join(

241 byhashdir, field, cast(str, fileinfo[filename][field])

242 )

243 if hashfile in known_hashfiles:

244 updated.add(hashfile)

245 else:

246 new.add(hashfile)

247

248 if updated:

249 session.execute(

250 sql.text(

251 """

252 UPDATE hashfile SET unreferenced = NULL

253 WHERE path = ANY(:p) AND suite_id = :id"""

254 ),

255 {"p": list(updated), "id": self.suite.suite_id},

256 )

257 if new:

258 session.execute(

259 sql.text(

260 """

261 INSERT INTO hashfile (path, suite_id)

262 VALUES (:p, :id)"""

263 ),

264 [{"p": hashfile, "id": self.suite.suite_id} for hashfile in new],

265 )

266

267 session.commit()

268

269 def _make_byhash_links(

270 self, fileinfo: dict[str, dict[str, str | int]], hashes: list[HashFunc]

271 ) -> None:

272 # Create hardlinks in by-hash directories

273 for filename in fileinfo:

274 if not os.path.lexists(filename): 274 ↛ 276line 274 didn't jump to line 276 because the condition on line 274 was never true

275 # probably an uncompressed index we didn't generate

276 continue

277

278 for h in hashes:

279 field = h.release_field

280 hashfile = os.path.join(

281 os.path.dirname(filename),

282 "by-hash",

283 field,

284 cast(str, fileinfo[filename][field]),

285 )

286 try:

287 os.makedirs(os.path.dirname(hashfile))

288 except OSError as exc:

289 if exc.errno != errno.EEXIST: 289 ↛ 290line 289 didn't jump to line 290 because the condition on line 289 was never true

290 raise

291 try:

292 os.link(filename, hashfile)

293 except OSError as exc:

294 if exc.errno != errno.EEXIST: 294 ↛ 295line 294 didn't jump to line 295 because the condition on line 294 was never true

295 raise

296

297 def _make_byhash_base_symlink(

298 self, fileinfo: dict[str, dict[str, str | int]], hashes: list[HashFunc]

299 ) -> None:

300 # Create symlinks to files in by-hash directories

301 for filename in fileinfo:

302 if not os.path.lexists(filename): 302 ↛ 304line 302 didn't jump to line 304 because the condition on line 302 was never true

303 # probably an uncompressed index we didn't generate

304 continue

305

306 besthash = hashes[-1]

307 field = besthash.release_field

308 hashfilebase = os.path.join(

309 "by-hash", field, cast(str, fileinfo[filename][field])

310 )

311 hashfile = os.path.join(os.path.dirname(filename), hashfilebase)

312

313 assert os.path.exists(hashfile), "by-hash file {} is missing".format(

314 hashfile

315 )

316

317 os.unlink(filename)

318 os.symlink(hashfilebase, filename)

319

320 def generate_release_files(self) -> None:

321 """

322 Generate Release files for the given suite

323 """

324

325 suite = self.suite

326 session = object_session(suite)

327 assert session is not None

328

329 # Attribs contains a tuple of field names and the database names to use to

330 # fill them in

331 attribs = (

332 ("Origin", "origin"),

333 ("Label", "label"),

334 ("Suite", "release_suite_output"),

335 ("Version", "version"),

336 ("Codename", "codename"),

337 ("Changelogs", "changelog_url"),

338 )

339

340 # A "Sub" Release file has slightly different fields

341 subattribs = (

342 ("Archive", "suite_name"),

343 ("Origin", "origin"),

344 ("Label", "label"),

345 ("Version", "version"),

346 )

347

348 # Boolean stuff. If we find it true in database, write out "yes" into the release file

349 boolattrs = (

350 ("NotAutomatic", "notautomatic"),

351 ("ButAutomaticUpgrades", "butautomaticupgrades"),

352 ("Acquire-By-Hash", "byhash"),

353 )

354

355 cnf = Config()

356 cnf_suite_suffix = cnf.get("Dinstall::SuiteSuffix", "").rstrip("/")

357

358 suite_suffix = utils.suite_suffix(suite.suite_name)

359

360 self.create_output_directories()

361 self.create_release_symlinks()

362

363 outfile = os.path.join(self.suite_release_path(), "Release")

364 out = open(outfile + ".new", "w")

365

366 for key, dbfield in attribs:

367 # Hack to skip NULL Version fields as we used to do this

368 # We should probably just always ignore anything which is None

369 if key in ("Version", "Changelogs") and getattr(suite, dbfield) is None:

370 continue

371

372 out.write("%s: %s\n" % (key, getattr(suite, dbfield)))

373

374 out.write(

375 "Date: %s\n"

376 % (time.strftime("%a, %d %b %Y %H:%M:%S UTC", time.gmtime(time.time())))

377 )

378

379 if suite.validtime: 379 ↛ 391line 379 didn't jump to line 391 because the condition on line 379 was always true

380 validtime = float(suite.validtime)

381 out.write(

382 "Valid-Until: %s\n"

383 % (

384 time.strftime(

385 "%a, %d %b %Y %H:%M:%S UTC",

386 time.gmtime(time.time() + validtime),

387 )

388 )

389 )

390

391 for key, dbfield in boolattrs:

392 if getattr(suite, dbfield, False):

393 out.write("%s: yes\n" % (key))

394

395 skip_arch_all = True

396 if (

397 suite.separate_contents_architecture_all

398 or suite.separate_packages_architecture_all

399 ):

400 # According to the Repository format specification:

401 # https://wiki.debian.org/DebianRepository/Format#No-Support-for-Architecture-all

402 #

403 # Clients are not expected to support Packages-all without Contents-all. At the

404 # time of writing, it is not possible to set separate_packages_architecture_all.

405 # However, we add this little assert to stop the bug early.

406 #

407 # If you are here because the assert failed, you probably want to see "update123.py"

408 # and its advice on updating the CHECK constraint.

409 assert suite.separate_contents_architecture_all

410 skip_arch_all = False

411

412 if not suite.separate_packages_architecture_all: 412 ↛ 415line 412 didn't jump to line 415 because the condition on line 412 was always true

413 out.write("No-Support-for-Architecture-all: Packages\n")

414

415 architectures = get_suite_architectures(

416 suite.suite_name, skipall=skip_arch_all, skipsrc=True, session=session

417 )

418

419 out.write(

420 "Architectures: %s\n" % (" ".join(a.arch_string for a in architectures))

421 )

422

423 components = [c.component_name for c in suite.components]

424

425 out.write("Components: %s\n" % (" ".join(components)))

426

427 # For exact compatibility with old g-r, write out Description here instead

428 # of with the rest of the DB fields above

429 if getattr(suite, "description") is not None: 429 ↛ 430line 429 didn't jump to line 430 because the condition on line 429 was never true

430 out.write("Description: %s\n" % suite.description)

431

432 for comp in components:

433 for dirpath, dirnames, filenames in os.walk(

434 os.path.join(self.suite_path(), comp), topdown=True

435 ):

436 if not re_gensubrelease.match(dirpath):

437 continue

438

439 subfile = os.path.join(dirpath, "Release")

440 subrel = open(subfile + ".new", "w")

441

442 for key, dbfield in subattribs:

443 if getattr(suite, dbfield) is not None:

444 subrel.write("%s: %s\n" % (key, getattr(suite, dbfield)))

445

446 for key, dbfield in boolattrs:

447 if getattr(suite, dbfield, False):

448 subrel.write("%s: yes\n" % (key))

449

450 subrel.write("Component: %s%s\n" % (suite_suffix, comp))

451

452 # Urgh, but until we have all the suite/component/arch stuff in the DB,

453 # this'll have to do

454 arch = os.path.split(dirpath)[-1]

455 if arch.startswith("binary-"):

456 arch = arch[7:]

457

458 subrel.write("Architecture: %s\n" % (arch))

459 subrel.close()

460

461 os.rename(subfile + ".new", subfile)

462

463 # Now that we have done the groundwork, we want to get off and add the files with

464 # their checksums to the main Release file

465 oldcwd = os.getcwd()

466

467 os.chdir(self.suite_path())

468

469 assert suite.checksums is not None

470 hashes = [x for x in RELEASE_HASHES if x.db_name in suite.checksums]

471

472 fileinfo: dict[str, dict[str, str | int]] = {}

473 fileinfo_byhash: dict[str, dict[str, str | int]] = {}

474

475 uncompnotseen: dict[str, tuple[Callable[[str, Literal["r"]], _Reader], str]] = (

476 {}

477 )

478

479 for dirpath, dirnames, filenames in os.walk(

480 ".", followlinks=True, topdown=True

481 ):

482 # SuiteSuffix deprecation:

483 # components on security-master are updates/{main,contrib,non-free}, but

484 # we want dists/${suite}/main. Until we can rename the components,

485 # we cheat by having an updates -> . symlink. This should not be visited.

486 if cnf_suite_suffix: 486 ↛ 487line 486 didn't jump to line 487 because the condition on line 486 was never true

487 path = os.path.join(dirpath, cnf_suite_suffix)

488 try:

489 target = os.readlink(path)

490 if target == ".":

491 dirnames.remove(cnf_suite_suffix)

492 except (OSError, ValueError):

493 pass

494 for entry in filenames:

495 if dirpath == "." and entry in ["Release", "Release.gpg", "InRelease"]:

496 continue

497

498 filename = os.path.join(dirpath.lstrip("./"), entry)

499

500 if re_includeinrelease_byhash.match(entry):

501 fileinfo[filename] = fileinfo_byhash[filename] = {}

502 elif re_includeinrelease_plain.match(entry): 502 ↛ 503line 502 didn't jump to line 503 because the condition on line 502 was never true

503 fileinfo[filename] = {}

504 # Skip things we don't want to include

505 else:

506 continue

507

508 with open(filename, "rb") as fd:

509 contents = fd.read()

510

511 # If we find a file for which we have a compressed version and

512 # haven't yet seen the uncompressed one, store the possibility

513 # for future use

514 if entry.endswith(".gz") and filename[:-3] not in uncompnotseen:

515 uncompnotseen[filename[:-3]] = (gzip.GzipFile, filename)

516 elif entry.endswith(".bz2") and filename[:-4] not in uncompnotseen: 516 ↛ 517line 516 didn't jump to line 517 because the condition on line 516 was never true

517 uncompnotseen[filename[:-4]] = (bz2.BZ2File, filename)

518 elif entry.endswith(".xz") and filename[:-3] not in uncompnotseen:

519 uncompnotseen[filename[:-3]] = (XzFile, filename)

520 elif entry.endswith(".zst") and filename[:-3] not in uncompnotseen: 520 ↛ 521line 520 didn't jump to line 521 because the condition on line 520 was never true

521 uncompnotseen[filename[:-3]] = (ZstdFile, filename)

522

523 fileinfo[filename]["len"] = len(contents)

524

525 for hf in hashes:

526 fileinfo[filename][hf.release_field] = hf.func(contents)

527

528 for filename, reader in uncompnotseen.items():

529 # If we've already seen the uncompressed file, we don't

530 # need to do anything again

531 if filename in fileinfo: 531 ↛ 532line 531 didn't jump to line 532 because the condition on line 531 was never true

532 continue

533

534 fileinfo[filename] = {}

535

536 # File handler is reader[0], filename of compressed file is reader[1]

537 contents = reader[0](reader[1], "r").read()

538

539 fileinfo[filename]["len"] = len(contents)

540

541 for hf in hashes:

542 fileinfo[filename][hf.release_field] = hf.func(contents)

543

544 for field in sorted(h.release_field for h in hashes):

545 out.write("%s:\n" % field)

546 for filename in sorted(fileinfo.keys()):

547 out.write(

548 " %s %8d %s\n"

549 % (

550 fileinfo[filename][field],

551 cast(int, fileinfo[filename]["len"]),

552 filename,

553 )

554 )

555

556 out.close()

557 os.rename(outfile + ".new", outfile)

558

559 self._update_hashfile_table(session, fileinfo_byhash, hashes)

560 self._make_byhash_links(fileinfo_byhash, hashes)

561 self._make_byhash_base_symlink(fileinfo_byhash, hashes)

562

563 sign_release_dir(suite, os.path.dirname(outfile))

564

565 os.chdir(oldcwd)

566

567

568def main() -> None:

569 global Logger

570

571 cnf = Config()

572

573 for i in ["Help", "Suite", "Force", "Quiet"]:

574 key = "Generate-Releases::Options::%s" % i

575 if key not in cnf: 575 ↛ 573line 575 didn't jump to line 573 because the condition on line 575 was always true

576 cnf[key] = ""

577

578 Arguments = [

579 ("h", "help", "Generate-Releases::Options::Help"),

580 ("a", "archive", "Generate-Releases::Options::Archive", "HasArg"),

581 ("s", "suite", "Generate-Releases::Options::Suite"),

582 ("f", "force", "Generate-Releases::Options::Force"),

583 ("q", "quiet", "Generate-Releases::Options::Quiet"),

584 ("o", "option", "", "ArbItem"),

585 ]

586

587 suite_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) # type: ignore[attr-defined]

588 Options = cnf.subtree("Generate-Releases::Options")

589

590 if Options["Help"]:

591 usage()

592

593 Logger = daklog.Logger("generate-releases")

594 pool = DakProcessPool()

595

596 session = DBConn().session()

597

598 if Options["Suite"]:

599 suites = []

600 for s in suite_names:

601 suite = get_suite(s.lower(), session)

602 if suite: 602 ↛ 605line 602 didn't jump to line 605 because the condition on line 602 was always true

603 suites.append(suite)

604 else:

605 print("cannot find suite %s" % s)

606 Logger.log(["cannot find suite %s" % s])

607 else:

608 query = session.query(Suite).filter(Suite.untouchable == False) # noqa:E712

609 if "Archive" in Options: 609 ↛ 614line 609 didn't jump to line 614 because the condition on line 609 was always true

610 archive_names = utils.split_args(Options["Archive"])

611 query = query.join(Suite.archive).filter(

612 Archive.archive_name.in_(archive_names)

613 )

614 suites = query.all()

615

616 for s in suites:

617 # Setup a multiprocessing Pool. As many workers as we have CPU cores.

618 if s.untouchable and not Options["Force"]: 618 ↛ 619line 618 didn't jump to line 619 because the condition on line 618 was never true

619 print("Skipping %s (untouchable)" % s.suite_name)

620 continue

621

622 if not Options["Quiet"]: 622 ↛ 624line 622 didn't jump to line 624 because the condition on line 622 was always true

623 print("Processing %s" % s.suite_name)

624 Logger.log(["Processing release file for Suite: %s" % (s.suite_name)])

625 pool.apply_async(generate_helper, (s.suite_id,))

626

627 # No more work will be added to our pool, close it and then wait for all to finish

628 pool.close()

629 pool.join()

630

631 retcode = pool.overall_status()

632

633 if retcode > 0: 633 ↛ 635line 633 didn't jump to line 635 because the condition on line 633 was never true

634 # TODO: CENTRAL FUNCTION FOR THIS / IMPROVE LOGGING

635 Logger.log(

636 [

637 "Release file generation broken: %s"

638 % (",".join([str(x[1]) for x in pool.results]))

639 ]

640 )

641

642 Logger.close()

643

644 sys.exit(retcode)

645

646

647def generate_helper(suite_id: int) -> tuple[int, str]:

648 """

649 This function is called in a new subprocess.

650 """

651 session = DBConn().session()

652 suite = session.get_one(Suite, suite_id)

653

654 # We allow the process handler to catch and deal with any exceptions

655 rw = ReleaseWriter(suite)

656 rw.generate_release_files()

657

658 return (PROC_STATUS_SUCCESS, "Release file written for %s" % suite.suite_name)

659

660

661#######################################################################################

662

663

664if __name__ == "__main__":

665 main()