Coverage for dak/generate_releases.py: 87%

1"""

2Create all the Release files

4@contact: Debian FTPMaster <ftpmaster@debian.org>

5@copyright: 2011 Joerg Jaspert <joerg@debian.org>

6@copyright: 2011 Mark Hymers <mhy@debian.org>

7@license: GNU General Public License version 2 or later

9"""

11# This program is free software; you can redistribute it and/or modify

12# it under the terms of the GNU General Public License as published by

13# the Free Software Foundation; either version 2 of the License, or

14# (at your option) any later version.

16# This program is distributed in the hope that it will be useful,

17# but WITHOUT ANY WARRANTY; without even the implied warranty of

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

19# GNU General Public License for more details.

21# You should have received a copy of the GNU General Public License

22# along with this program; if not, write to the Free Software

23# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

25################################################################################

27# <mhy> I wish they wouldnt leave biscuits out, thats just tempting. Damnit.

29################################################################################

31import bz2

32import errno

33import gzip

34import os

35import os.path

36import subprocess

37import sys

38import time

39from collections.abc import Callable

40from typing import TYPE_CHECKING, Literal, NoReturn, Protocol, cast

42import apt_pkg

43from sqlalchemy import select, sql

44from sqlalchemy.orm import object_session

46import daklib.gpg

47from daklib import daklog, utils

48from daklib.config import Config

49from daklib.dakmultiprocessing import PROC_STATUS_SUCCESS, DakProcessPool

50from daklib.dbconn import Archive, DBConn, Suite, get_suite, get_suite_architectures

51from daklib.regexes import (

52 re_gensubrelease,

53 re_includeinrelease_byhash,

54 re_includeinrelease_plain,

55)

57if TYPE_CHECKING:

58 from sqlalchemy.orm import Session

60################################################################################

61Logger = None #: Our logging object

63################################################################################

66def usage(exit_code=0) -> NoReturn:

67 """Usage information"""

69 print(

70 """Usage: dak generate-releases [OPTIONS]

71Generate the Release files

73 -a, --archive=ARCHIVE process suites in ARCHIVE

74 -s, --suite=SUITE(s) process this suite

75 Default: All suites not marked 'untouchable'

76 -f, --force Allow processing of untouchable suites

77 CAREFUL: Only to be used at (point) release time!

78 -h, --help show this help and exit

79 -q, --quiet Don't output progress

81SUITE can be a space separated list, e.g.

82 --suite=unstable testing

83 """

84 )

85 sys.exit(exit_code)

88########################################################################

91def sign_release_dir(suite: Suite, dirname: str) -> None:

92 cnf = Config()

94 if "Dinstall::SigningHomedir" in cnf: 94 ↛ exitline 94 didn't return from function 'sign_release_dir' because the condition on line 94 was always true

95 args = {

96 "keyids": suite.signingkeys or [],

97 "pubring": cnf.get("Dinstall::SigningPubKeyring") or None,

98 "homedir": cnf.get("Dinstall::SigningHomedir") or None,

99 "passphrase_file": cnf.get("Dinstall::SigningPassphraseFile") or None,

100 }

101

102 relname = os.path.join(dirname, "Release")

103

104 dest = os.path.join(dirname, "Release.gpg")

105 if os.path.exists(dest):

106 os.unlink(dest)

107

108 inlinedest = os.path.join(dirname, "InRelease")

109 if os.path.exists(inlinedest):

110 os.unlink(inlinedest)

111

112 with open(relname, "r") as stdin:

113 with open(dest, "w") as stdout:

114 daklib.gpg.sign(stdin, stdout, inline=False, **args) # type: ignore[arg-type]

115 stdin.seek(0)

116 with open(inlinedest, "w") as stdout:

117 daklib.gpg.sign(stdin, stdout, inline=True, **args) # type: ignore[arg-type]

118

119

120class _Reader(Protocol):

121 def read(self) -> bytes: ... # noqa: E704 121 ↛ exitline 121 didn't jump to line 121 because

122

123

124class XzFile:

125 def __init__(self, filename: str, mode="r"):

126 self.filename = filename

127

128 def read(self) -> bytes:

129 with open(self.filename, "rb") as stdin:

130 return subprocess.check_output(["xz", "-d"], stdin=stdin)

131

132

133class ZstdFile:

134 def __init__(self, filename: str, mode="r"):

135 self.filename = filename

136

137 def read(self) -> bytes:

138 with open(self.filename, "rb") as stdin:

139 return subprocess.check_output(["zstd", "--decompress"], stdin=stdin)

140

141

142class HashFunc:

143 def __init__(self, release_field: str, func: Callable[[bytes], str], db_name: str):

144 self.release_field = release_field

145 self.func = func

146 self.db_name = db_name

147

148

149RELEASE_HASHES = [

150 HashFunc("MD5Sum", apt_pkg.md5sum, "md5sum"),

151 HashFunc("SHA1", apt_pkg.sha1sum, "sha1"), # type: ignore[attr-defined]

152 HashFunc("SHA256", apt_pkg.sha256sum, "sha256"), # type: ignore[attr-defined]

153]

154

155

156class ReleaseWriter:

157 def __init__(self, suite: Suite):

158 self.suite = suite

159

160 def suite_path(self) -> str:

161 """

162 Absolute path to the suite-specific files.

163 """

164 suite_suffix = utils.suite_suffix(self.suite.suite_name)

165

166 return os.path.join(

167 self.suite.archive.path, "dists", self.suite.suite_name, suite_suffix

168 )

169

170 def suite_release_path(self) -> str:

171 """

172 Absolute path where Release files are physically stored.

173 This should be a path that sorts after the dists/ directory.

174 """

175 suite_suffix = utils.suite_suffix(self.suite.suite_name)

176

177 return os.path.join(

178 self.suite.archive.path,

179 "zzz-dists",

180 self.suite.codename or self.suite.suite_name,

181 suite_suffix,

182 )

183

184 def create_release_symlinks(self) -> None:

185 """

186 Create symlinks for Release files.

187 This creates the symlinks for Release files in the `suite_path`

188 to the actual files in `suite_release_path`.

189 """

190 relpath = os.path.relpath(self.suite_release_path(), self.suite_path())

191 for f in ("Release", "Release.gpg", "InRelease"):

192 source = os.path.join(relpath, f)

193 dest = os.path.join(self.suite_path(), f)

194 if os.path.lexists(dest):

195 if not os.path.islink(dest): 195 ↛ 196line 195 didn't jump to line 196 because the condition on line 195 was never true

196 os.unlink(dest)

197 elif os.readlink(dest) == source: 197 ↛ 200line 197 didn't jump to line 200 because the condition on line 197 was always true

198 continue

199 else:

200 os.unlink(dest)

201 os.symlink(source, dest)

202

203 def create_output_directories(self) -> None:

204 for path in (self.suite_path(), self.suite_release_path()):

205 try:

206 os.makedirs(path)

207 except OSError as e:

208 if e.errno != errno.EEXIST: 208 ↛ 209line 208 didn't jump to line 209 because the condition on line 208 was never true

209 raise

210

211 def _update_hashfile_table(

212 self,

213 session: "Session",

214 fileinfo: dict[str, dict[str, str | int]],

215 hashes: list[HashFunc],

216 ) -> None:

217 # Mark all by-hash files as obsolete. We will undo that for the ones

218 # we still reference later.

219 query = """

220 UPDATE hashfile SET unreferenced = CURRENT_TIMESTAMP

221 WHERE suite_id = :id AND unreferenced IS NULL"""

222 session.execute(sql.text(query), {"id": self.suite.suite_id})

223

224 query = "SELECT path FROM hashfile WHERE suite_id = :id"

225 q = session.execute(sql.text(query), {"id": self.suite.suite_id})

226 known_hashfiles = {row[0] for row in q}

227 updated = set()

228 new = set()

229

230 # Update the hashfile table with new or updated files

231 for filename in fileinfo:

232 if not os.path.lexists(filename): 232 ↛ 234line 232 didn't jump to line 234 because the condition on line 232 was never true

233 # probably an uncompressed index we didn't generate

234 continue

235 byhashdir = os.path.join(os.path.dirname(filename), "by-hash")

236 for h in hashes:

237 field = h.release_field

238 hashfile = os.path.join(

239 byhashdir, field, cast(str, fileinfo[filename][field])

240 )

241 if hashfile in known_hashfiles:

242 updated.add(hashfile)

243 else:

244 new.add(hashfile)

245

246 if updated:

247 session.execute(

248 sql.text(

249 """

250 UPDATE hashfile SET unreferenced = NULL

251 WHERE path = ANY(:p) AND suite_id = :id"""

252 ),

253 {"p": list(updated), "id": self.suite.suite_id},

254 )

255 if new:

256 session.execute(

257 sql.text(

258 """

259 INSERT INTO hashfile (path, suite_id)

260 VALUES (:p, :id)"""

261 ),

262 [{"p": hashfile, "id": self.suite.suite_id} for hashfile in new],

263 )

264

265 session.commit()

266

267 def _make_byhash_links(

268 self, fileinfo: dict[str, dict[str, str | int]], hashes: list[HashFunc]

269 ) -> None:

270 # Create hardlinks in by-hash directories

271 for filename in fileinfo:

272 if not os.path.lexists(filename): 272 ↛ 274line 272 didn't jump to line 274 because the condition on line 272 was never true

273 # probably an uncompressed index we didn't generate

274 continue

275

276 for h in hashes:

277 field = h.release_field

278 hashfile = os.path.join(

279 os.path.dirname(filename),

280 "by-hash",

281 field,

282 cast(str, fileinfo[filename][field]),

283 )

284 try:

285 os.makedirs(os.path.dirname(hashfile))

286 except OSError as exc:

287 if exc.errno != errno.EEXIST: 287 ↛ 288line 287 didn't jump to line 288 because the condition on line 287 was never true

288 raise

289 try:

290 os.link(filename, hashfile)

291 except OSError as exc:

292 if exc.errno != errno.EEXIST: 292 ↛ 293line 292 didn't jump to line 293 because the condition on line 292 was never true

293 raise

294

295 def _make_byhash_base_symlink(

296 self, fileinfo: dict[str, dict[str, str | int]], hashes: list[HashFunc]

297 ) -> None:

298 # Create symlinks to files in by-hash directories

299 for filename in fileinfo:

300 if not os.path.lexists(filename): 300 ↛ 302line 300 didn't jump to line 302 because the condition on line 300 was never true

301 # probably an uncompressed index we didn't generate

302 continue

303

304 besthash = hashes[-1]

305 field = besthash.release_field

306 hashfilebase = os.path.join(

307 "by-hash", field, cast(str, fileinfo[filename][field])

308 )

309 hashfile = os.path.join(os.path.dirname(filename), hashfilebase)

310

311 assert os.path.exists(hashfile), "by-hash file {} is missing".format(

312 hashfile

313 )

314

315 os.unlink(filename)

316 os.symlink(hashfilebase, filename)

317

318 def generate_release_files(self) -> None:

319 """

320 Generate Release files for the given suite

321 """

322

323 suite = self.suite

324 session = object_session(suite)

325 assert session is not None

326

327 # Attribs contains a tuple of field names and the database names to use to

328 # fill them in

329 attribs = (

330 ("Origin", "origin"),

331 ("Label", "label"),

332 ("Suite", "release_suite_output"),

333 ("Version", "version"),

334 ("Codename", "codename"),

335 ("Changelogs", "changelog_url"),

336 )

337

338 # A "Sub" Release file has slightly different fields

339 subattribs = (

340 ("Archive", "suite_name"),

341 ("Origin", "origin"),

342 ("Label", "label"),

343 ("Version", "version"),

344 )

345

346 # Boolean stuff. If we find it true in database, write out "yes" into the release file

347 boolattrs = (

348 ("NotAutomatic", "notautomatic"),

349 ("ButAutomaticUpgrades", "butautomaticupgrades"),

350 ("Acquire-By-Hash", "byhash"),

351 )

352

353 cnf = Config()

354 cnf_suite_suffix = cnf.get("Dinstall::SuiteSuffix", "").rstrip("/")

355

356 suite_suffix = utils.suite_suffix(suite.suite_name)

357

358 self.create_output_directories()

359 self.create_release_symlinks()

360

361 outfile = os.path.join(self.suite_release_path(), "Release")

362 out = open(outfile + ".new", "w")

363

364 for key, dbfield in attribs:

365 # Hack to skip NULL Version fields as we used to do this

366 # We should probably just always ignore anything which is None

367 if key in ("Version", "Changelogs") and getattr(suite, dbfield) is None:

368 continue

369

370 out.write("%s: %s\n" % (key, getattr(suite, dbfield)))

371

372 out.write(

373 "Date: %s\n"

374 % (time.strftime("%a, %d %b %Y %H:%M:%S UTC", time.gmtime(time.time())))

375 )

376

377 if suite.validtime: 377 ↛ 389line 377 didn't jump to line 389 because the condition on line 377 was always true

378 validtime = float(suite.validtime)

379 out.write(

380 "Valid-Until: %s\n"

381 % (

382 time.strftime(

383 "%a, %d %b %Y %H:%M:%S UTC",

384 time.gmtime(time.time() + validtime),

385 )

386 )

387 )

388

389 for key, dbfield in boolattrs:

390 if getattr(suite, dbfield, False):

391 out.write("%s: yes\n" % (key))

392

393 skip_arch_all = True

394 if (

395 suite.separate_contents_architecture_all

396 or suite.separate_packages_architecture_all

397 ):

398 # According to the Repository format specification:

399 # https://wiki.debian.org/DebianRepository/Format#No-Support-for-Architecture-all

400 #

401 # Clients are not expected to support Packages-all without Contents-all. At the

402 # time of writing, it is not possible to set separate_packages_architecture_all.

403 # However, we add this little assert to stop the bug early.

404 #

405 # If you are here because the assert failed, you probably want to see "update123.py"

406 # and its advice on updating the CHECK constraint.

407 assert suite.separate_contents_architecture_all

408 skip_arch_all = False

409

410 if not suite.separate_packages_architecture_all: 410 ↛ 413line 410 didn't jump to line 413 because the condition on line 410 was always true

411 out.write("No-Support-for-Architecture-all: Packages\n")

412

413 architectures = get_suite_architectures(

414 suite.suite_name, skipall=skip_arch_all, skipsrc=True, session=session

415 )

416

417 out.write(

418 "Architectures: %s\n" % (" ".join(a.arch_string for a in architectures))

419 )

420

421 components = [c.component_name for c in suite.components]

422

423 out.write("Components: %s\n" % (" ".join(components)))

424

425 # For exact compatibility with old g-r, write out Description here instead

426 # of with the rest of the DB fields above

427 if suite.description is not None: 427 ↛ 428line 427 didn't jump to line 428 because the condition on line 427 was never true

428 out.write("Description: %s\n" % suite.description)

429

430 for comp in components:

431 for dirpath, dirnames, filenames in os.walk(

432 os.path.join(self.suite_path(), comp), topdown=True

433 ):

434 if not re_gensubrelease.match(dirpath):

435 continue

436

437 subfile = os.path.join(dirpath, "Release")

438 subrel = open(subfile + ".new", "w")

439

440 for key, dbfield in subattribs:

441 if getattr(suite, dbfield) is not None:

442 subrel.write("%s: %s\n" % (key, getattr(suite, dbfield)))

443

444 for key, dbfield in boolattrs:

445 if getattr(suite, dbfield, False):

446 subrel.write("%s: yes\n" % (key))

447

448 subrel.write("Component: %s%s\n" % (suite_suffix, comp))

449

450 # Urgh, but until we have all the suite/component/arch stuff in the DB,

451 # this'll have to do

452 arch = os.path.split(dirpath)[-1]

453 arch = arch.removeprefix("binary-")

454

455 subrel.write("Architecture: %s\n" % (arch))

456 subrel.close()

457

458 os.rename(subfile + ".new", subfile)

459

460 # Now that we have done the groundwork, we want to get off and add the files with

461 # their checksums to the main Release file

462 oldcwd = os.getcwd()

463

464 os.chdir(self.suite_path())

465

466 assert suite.checksums is not None

467 hashes = [x for x in RELEASE_HASHES if x.db_name in suite.checksums]

468

469 fileinfo: dict[str, dict[str, str | int]] = {}

470 fileinfo_byhash: dict[str, dict[str, str | int]] = {}

471

472 uncompnotseen: dict[str, tuple[Callable[[str, Literal["r"]], _Reader], str]] = (

473 {}

474 )

475

476 for dirpath, dirnames, filenames in os.walk(

477 ".", followlinks=True, topdown=True

478 ):

479 # SuiteSuffix deprecation:

480 # components on security-master are updates/{main,contrib,non-free}, but

481 # we want dists/${suite}/main. Until we can rename the components,

482 # we cheat by having an updates -> . symlink. This should not be visited.

483 if cnf_suite_suffix: 483 ↛ 484line 483 didn't jump to line 484 because the condition on line 483 was never true

484 path = os.path.join(dirpath, cnf_suite_suffix)

485 try:

486 target = os.readlink(path)

487 if target == ".":

488 dirnames.remove(cnf_suite_suffix)

489 except (OSError, ValueError):

490 pass

491 for entry in filenames:

492 if dirpath == "." and entry in ["Release", "Release.gpg", "InRelease"]:

493 continue

494

495 filename = os.path.join(dirpath.lstrip("./"), entry)

496

497 if re_includeinrelease_byhash.match(entry):

498 fileinfo[filename] = fileinfo_byhash[filename] = {}

499 elif re_includeinrelease_plain.match(entry): 499 ↛ 500line 499 didn't jump to line 500 because the condition on line 499 was never true

500 fileinfo[filename] = {}

501 # Skip things we don't want to include

502 else:

503 continue

504

505 with open(filename, "rb") as fd:

506 contents = fd.read()

507

508 # If we find a file for which we have a compressed version and

509 # haven't yet seen the uncompressed one, store the possibility

510 # for future use

511 if entry.endswith(".gz") and filename[:-3] not in uncompnotseen:

512 uncompnotseen[filename[:-3]] = (gzip.GzipFile, filename)

513 elif entry.endswith(".bz2") and filename[:-4] not in uncompnotseen: 513 ↛ 514line 513 didn't jump to line 514 because the condition on line 513 was never true

514 uncompnotseen[filename[:-4]] = (bz2.BZ2File, filename)

515 elif entry.endswith(".xz") and filename[:-3] not in uncompnotseen:

516 uncompnotseen[filename[:-3]] = (XzFile, filename)

517 elif entry.endswith(".zst") and filename[:-3] not in uncompnotseen: 517 ↛ 518line 517 didn't jump to line 518 because the condition on line 517 was never true

518 uncompnotseen[filename[:-3]] = (ZstdFile, filename)

519

520 fileinfo[filename]["len"] = len(contents)

521

522 for hf in hashes:

523 fileinfo[filename][hf.release_field] = hf.func(contents)

524

525 for filename, reader in uncompnotseen.items():

526 # If we've already seen the uncompressed file, we don't

527 # need to do anything again

528 if filename in fileinfo: 528 ↛ 529line 528 didn't jump to line 529 because the condition on line 528 was never true

529 continue

530

531 fileinfo[filename] = {}

532

533 # File handler is reader[0], filename of compressed file is reader[1]

534 contents = reader[0](reader[1], "r").read()

535

536 fileinfo[filename]["len"] = len(contents)

537

538 for hf in hashes:

539 fileinfo[filename][hf.release_field] = hf.func(contents)

540

541 for field in sorted(h.release_field for h in hashes):

542 out.write("%s:\n" % field)

543 for filename in sorted(fileinfo.keys()):

544 out.write(

545 " %s %8d %s\n"

546 % (

547 fileinfo[filename][field],

548 cast(int, fileinfo[filename]["len"]),

549 filename,

550 )

551 )

552

553 out.close()

554 os.rename(outfile + ".new", outfile)

555

556 self._update_hashfile_table(session, fileinfo_byhash, hashes)

557 self._make_byhash_links(fileinfo_byhash, hashes)

558 self._make_byhash_base_symlink(fileinfo_byhash, hashes)

559

560 sign_release_dir(suite, os.path.dirname(outfile))

561

562 os.chdir(oldcwd)

563

564

565def main() -> None:

566 global Logger

567

568 cnf = Config()

569

570 for i in ["Help", "Suite", "Force", "Quiet"]:

571 key = "Generate-Releases::Options::%s" % i

572 if key not in cnf: 572 ↛ 570line 572 didn't jump to line 570 because the condition on line 572 was always true

573 cnf[key] = ""

574

575 Arguments = [

576 ("h", "help", "Generate-Releases::Options::Help"),

577 ("a", "archive", "Generate-Releases::Options::Archive", "HasArg"),

578 ("s", "suite", "Generate-Releases::Options::Suite"),

579 ("f", "force", "Generate-Releases::Options::Force"),

580 ("q", "quiet", "Generate-Releases::Options::Quiet"),

581 ("o", "option", "", "ArbItem"),

582 ]

583

584 suite_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) # type: ignore[attr-defined]

585 Options = cnf.subtree("Generate-Releases::Options")

586

587 if Options["Help"]:

588 usage()

589

590 Logger = daklog.Logger("generate-releases")

591 pool = DakProcessPool()

592

593 session = DBConn().session()

594

595 if Options["Suite"]:

596 suites = []

597 for s in suite_names:

598 suite = get_suite(s.lower(), session)

599 if suite: 599 ↛ 602line 599 didn't jump to line 602 because the condition on line 599 was always true

600 suites.append(suite)

601 else:

602 print("cannot find suite %s" % s)

603 Logger.log(["cannot find suite %s" % s])

604 else:

605 query = select(Suite).where(~Suite.untouchable)

606 if "Archive" in Options: 606 ↛ 611line 606 didn't jump to line 611 because the condition on line 606 was always true

607 archive_names = utils.split_args(Options["Archive"])

608 query = query.join(Suite.archive).where(

609 Archive.archive_name.in_(archive_names)

610 )

611 suites = list(session.scalars(query))

612

613 for s in suites:

614 # Setup a multiprocessing Pool. As many workers as we have CPU cores.

615 if s.untouchable and not Options["Force"]: 615 ↛ 616line 615 didn't jump to line 616 because the condition on line 615 was never true

616 print("Skipping %s (untouchable)" % s.suite_name)

617 continue

618

619 if not Options["Quiet"]: 619 ↛ 621line 619 didn't jump to line 621 because the condition on line 619 was always true

620 print("Processing %s" % s.suite_name)

621 Logger.log(["Processing release file for Suite: %s" % (s.suite_name)])

622 pool.apply_async(generate_helper, (s.suite_id,))

623

624 # No more work will be added to our pool, close it and then wait for all to finish

625 pool.close()

626 pool.join()

627

628 retcode = pool.overall_status()

629

630 if retcode > 0: 630 ↛ 632line 630 didn't jump to line 632 because the condition on line 630 was never true

631 # TODO: CENTRAL FUNCTION FOR THIS / IMPROVE LOGGING

632 Logger.log(

633 [

634 "Release file generation broken: %s"

635 % (",".join([str(x[1]) for x in pool.results]))

636 ]

637 )

638

639 Logger.close()

640

641 sys.exit(retcode)

642

643

644def generate_helper(suite_id: int) -> tuple[int, str]:

645 """

646 This function is called in a new subprocess.

647 """

648 session = DBConn().session()

649 suite = session.get_one(Suite, suite_id)

650

651 # We allow the process handler to catch and deal with any exceptions

652 rw = ReleaseWriter(suite)

653 rw.generate_release_files()

654

655 return (PROC_STATUS_SUCCESS, "Release file written for %s" % suite.suite_name)