Coverage for dak/generate_releases.py: 87%

1#! /usr/bin/env python3

3"""

4Create all the Release files

6@contact: Debian FTPMaster <ftpmaster@debian.org>

7@copyright: 2011 Joerg Jaspert <joerg@debian.org>

8@copyright: 2011 Mark Hymers <mhy@debian.org>

9@license: GNU General Public License version 2 or later

11"""

13# This program is free software; you can redistribute it and/or modify

14# it under the terms of the GNU General Public License as published by

15# the Free Software Foundation; either version 2 of the License, or

16# (at your option) any later version.

18# This program is distributed in the hope that it will be useful,

19# but WITHOUT ANY WARRANTY; without even the implied warranty of

20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

21# GNU General Public License for more details.

23# You should have received a copy of the GNU General Public License

24# along with this program; if not, write to the Free Software

25# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

27################################################################################

29# <mhy> I wish they wouldnt leave biscuits out, thats just tempting. Damnit.

31################################################################################

33import bz2

34import errno

35import gzip

36import os

37import os.path

38import subprocess

39import sys

40import time

42import apt_pkg

43from sqlalchemy.orm import object_session

45import daklib.gpg

46from daklib import daklog, utils

47from daklib.config import Config

48from daklib.dakmultiprocessing import PROC_STATUS_SUCCESS, DakProcessPool

49from daklib.dbconn import Archive, DBConn, Suite, get_suite, get_suite_architectures

50from daklib.regexes import (

51 re_gensubrelease,

52 re_includeinrelease_byhash,

53 re_includeinrelease_plain,

54)

56################################################################################

57Logger = None #: Our logging object

59################################################################################

62def usage(exit_code=0):

63 """Usage information"""

65 print(

66 """Usage: dak generate-releases [OPTIONS]

67Generate the Release files

69 -a, --archive=ARCHIVE process suites in ARCHIVE

70 -s, --suite=SUITE(s) process this suite

71 Default: All suites not marked 'untouchable'

72 -f, --force Allow processing of untouchable suites

73 CAREFUL: Only to be used at (point) release time!

74 -h, --help show this help and exit

75 -q, --quiet Don't output progress

77SUITE can be a space separated list, e.g.

78 --suite=unstable testing

79 """

80 )

81 sys.exit(exit_code)

84########################################################################

87def sign_release_dir(suite, dirname):

88 cnf = Config()

90 if "Dinstall::SigningHomedir" in cnf: 90 ↛ exitline 90 didn't return from function 'sign_release_dir', because the condition on line 90 was never false

91 args = {

92 "keyids": suite.signingkeys or [],

93 "pubring": cnf.get("Dinstall::SigningPubKeyring") or None,

94 "homedir": cnf.get("Dinstall::SigningHomedir") or None,

95 "passphrase_file": cnf.get("Dinstall::SigningPassphraseFile") or None,

96 }

98 relname = os.path.join(dirname, "Release")

100 dest = os.path.join(dirname, "Release.gpg")

101 if os.path.exists(dest):

102 os.unlink(dest)

103

104 inlinedest = os.path.join(dirname, "InRelease")

105 if os.path.exists(inlinedest):

106 os.unlink(inlinedest)

107

108 with open(relname, "r") as stdin:

109 with open(dest, "w") as stdout:

110 daklib.gpg.sign(stdin, stdout, inline=False, **args)

111 stdin.seek(0)

112 with open(inlinedest, "w") as stdout:

113 daklib.gpg.sign(stdin, stdout, inline=True, **args)

114

115

116class XzFile:

117 def __init__(self, filename, mode="r"):

118 self.filename = filename

119

120 def read(self):

121 with open(self.filename, "rb") as stdin:

122 return subprocess.check_output(["xz", "-d"], stdin=stdin)

123

124

125class ZstdFile:

126 def __init__(self, filename, mode="r"):

127 self.filename = filename

128

129 def read(self):

130 with open(self.filename, "rb") as stdin:

131 return subprocess.check_output(["zstd", "--decompress"], stdin=stdin)

132

133

134class HashFunc:

135 def __init__(self, release_field, func, db_name):

136 self.release_field = release_field

137 self.func = func

138 self.db_name = db_name

139

140

141RELEASE_HASHES = [

142 HashFunc("MD5Sum", apt_pkg.md5sum, "md5sum"),

143 HashFunc("SHA1", apt_pkg.sha1sum, "sha1"),

144 HashFunc("SHA256", apt_pkg.sha256sum, "sha256"),

145]

146

147

148class ReleaseWriter:

149 def __init__(self, suite):

150 self.suite = suite

151

152 def suite_path(self):

153 """

154 Absolute path to the suite-specific files.

155 """

156 suite_suffix = utils.suite_suffix(self.suite.suite_name)

157

158 return os.path.join(

159 self.suite.archive.path, "dists", self.suite.suite_name, suite_suffix

160 )

161

162 def suite_release_path(self):

163 """

164 Absolute path where Release files are physically stored.

165 This should be a path that sorts after the dists/ directory.

166 """

167 suite_suffix = utils.suite_suffix(self.suite.suite_name)

168

169 return os.path.join(

170 self.suite.archive.path,

171 "zzz-dists",

172 self.suite.codename or self.suite.suite_name,

173 suite_suffix,

174 )

175

176 def create_release_symlinks(self):

177 """

178 Create symlinks for Release files.

179 This creates the symlinks for Release files in the `suite_path`

180 to the actual files in `suite_release_path`.

181 """

182 relpath = os.path.relpath(self.suite_release_path(), self.suite_path())

183 for f in ("Release", "Release.gpg", "InRelease"):

184 source = os.path.join(relpath, f)

185 dest = os.path.join(self.suite_path(), f)

186 if os.path.lexists(dest):

187 if not os.path.islink(dest): 187 ↛ 188line 187 didn't jump to line 188, because the condition on line 187 was never true

188 os.unlink(dest)

189 elif os.readlink(dest) == source: 189 ↛ 192line 189 didn't jump to line 192, because the condition on line 189 was never false

190 continue

191 else:

192 os.unlink(dest)

193 os.symlink(source, dest)

194

195 def create_output_directories(self):

196 for path in (self.suite_path(), self.suite_release_path()):

197 try:

198 os.makedirs(path)

199 except OSError as e:

200 if e.errno != errno.EEXIST: 200 ↛ 201line 200 didn't jump to line 201, because the condition on line 200 was never true

201 raise

202

203 def _update_hashfile_table(self, session, fileinfo, hashes):

204 # Mark all by-hash files as obsolete. We will undo that for the ones

205 # we still reference later.

206 query = """

207 UPDATE hashfile SET unreferenced = CURRENT_TIMESTAMP

208 WHERE suite_id = :id AND unreferenced IS NULL"""

209 session.execute(query, {"id": self.suite.suite_id})

210

211 query = "SELECT path FROM hashfile WHERE suite_id = :id"

212 q = session.execute(query, {"id": self.suite.suite_id})

213 known_hashfiles = set(row[0] for row in q)

214 updated = set()

215 new = set()

216

217 # Update the hashfile table with new or updated files

218 for filename in fileinfo:

219 if not os.path.lexists(filename): 219 ↛ 221line 219 didn't jump to line 221, because the condition on line 219 was never true

220 # probably an uncompressed index we didn't generate

221 continue

222 byhashdir = os.path.join(os.path.dirname(filename), "by-hash")

223 for h in hashes:

224 field = h.release_field

225 hashfile = os.path.join(byhashdir, field, fileinfo[filename][field])

226 if hashfile in known_hashfiles:

227 updated.add(hashfile)

228 else:

229 new.add(hashfile)

230

231 if updated:

232 session.execute(

233 """

234 UPDATE hashfile SET unreferenced = NULL

235 WHERE path = ANY(:p) AND suite_id = :id""",

236 {"p": list(updated), "id": self.suite.suite_id},

237 )

238 if new:

239 session.execute(

240 """

241 INSERT INTO hashfile (path, suite_id)

242 VALUES (:p, :id)""",

243 [{"p": hashfile, "id": self.suite.suite_id} for hashfile in new],

244 )

245

246 session.commit()

247

248 def _make_byhash_links(self, fileinfo, hashes):

249 # Create hardlinks in by-hash directories

250 for filename in fileinfo:

251 if not os.path.lexists(filename): 251 ↛ 253line 251 didn't jump to line 253, because the condition on line 251 was never true

252 # probably an uncompressed index we didn't generate

253 continue

254

255 for h in hashes:

256 field = h.release_field

257 hashfile = os.path.join(

258 os.path.dirname(filename),

259 "by-hash",

260 field,

261 fileinfo[filename][field],

262 )

263 try:

264 os.makedirs(os.path.dirname(hashfile))

265 except OSError as exc:

266 if exc.errno != errno.EEXIST: 266 ↛ 267line 266 didn't jump to line 267, because the condition on line 266 was never true

267 raise

268 try:

269 os.link(filename, hashfile)

270 except OSError as exc:

271 if exc.errno != errno.EEXIST: 271 ↛ 272line 271 didn't jump to line 272, because the condition on line 271 was never true

272 raise

273

274 def _make_byhash_base_symlink(self, fileinfo, hashes):

275 # Create symlinks to files in by-hash directories

276 for filename in fileinfo:

277 if not os.path.lexists(filename): 277 ↛ 279line 277 didn't jump to line 279, because the condition on line 277 was never true

278 # probably an uncompressed index we didn't generate

279 continue

280

281 besthash = hashes[-1]

282 field = besthash.release_field

283 hashfilebase = os.path.join("by-hash", field, fileinfo[filename][field])

284 hashfile = os.path.join(os.path.dirname(filename), hashfilebase)

285

286 assert os.path.exists(hashfile), "by-hash file {} is missing".format(

287 hashfile

288 )

289

290 os.unlink(filename)

291 os.symlink(hashfilebase, filename)

292

293 def generate_release_files(self):

294 """

295 Generate Release files for the given suite

296 """

297

298 suite = self.suite

299 session = object_session(suite)

300

301 # Attribs contains a tuple of field names and the database names to use to

302 # fill them in

303 attribs = (

304 ("Origin", "origin"),

305 ("Label", "label"),

306 ("Suite", "release_suite_output"),

307 ("Version", "version"),

308 ("Codename", "codename"),

309 ("Changelogs", "changelog_url"),

310 )

311

312 # A "Sub" Release file has slightly different fields

313 subattribs = (

314 ("Archive", "suite_name"),

315 ("Origin", "origin"),

316 ("Label", "label"),

317 ("Version", "version"),

318 )

319

320 # Boolean stuff. If we find it true in database, write out "yes" into the release file

321 boolattrs = (

322 ("NotAutomatic", "notautomatic"),

323 ("ButAutomaticUpgrades", "butautomaticupgrades"),

324 ("Acquire-By-Hash", "byhash"),

325 )

326

327 cnf = Config()

328 cnf_suite_suffix = cnf.get("Dinstall::SuiteSuffix", "").rstrip("/")

329

330 suite_suffix = utils.suite_suffix(suite.suite_name)

331

332 self.create_output_directories()

333 self.create_release_symlinks()

334

335 outfile = os.path.join(self.suite_release_path(), "Release")

336 out = open(outfile + ".new", "w")

337

338 for key, dbfield in attribs:

339 # Hack to skip NULL Version fields as we used to do this

340 # We should probably just always ignore anything which is None

341 if key in ("Version", "Changelogs") and getattr(suite, dbfield) is None:

342 continue

343

344 out.write("%s: %s\n" % (key, getattr(suite, dbfield)))

345

346 out.write(

347 "Date: %s\n"

348 % (time.strftime("%a, %d %b %Y %H:%M:%S UTC", time.gmtime(time.time())))

349 )

350

351 if suite.validtime: 351 ↛ 363line 351 didn't jump to line 363, because the condition on line 351 was never false

352 validtime = float(suite.validtime)

353 out.write(

354 "Valid-Until: %s\n"

355 % (

356 time.strftime(

357 "%a, %d %b %Y %H:%M:%S UTC",

358 time.gmtime(time.time() + validtime),

359 )

360 )

361 )

362

363 for key, dbfield in boolattrs:

364 if getattr(suite, dbfield, False):

365 out.write("%s: yes\n" % (key))

366

367 skip_arch_all = True

368 if (

369 suite.separate_contents_architecture_all

370 or suite.separate_packages_architecture_all

371 ):

372 # According to the Repository format specification:

373 # https://wiki.debian.org/DebianRepository/Format#No-Support-for-Architecture-all

374 #

375 # Clients are not expected to support Packages-all without Contents-all. At the

376 # time of writing, it is not possible to set separate_packages_architecture_all.

377 # However, we add this little assert to stop the bug early.

378 #

379 # If you are here because the assert failed, you probably want to see "update123.py"

380 # and its advice on updating the CHECK constraint.

381 assert suite.separate_contents_architecture_all

382 skip_arch_all = False

383

384 if not suite.separate_packages_architecture_all: 384 ↛ 387line 384 didn't jump to line 387, because the condition on line 384 was never false

385 out.write("No-Support-for-Architecture-all: Packages\n")

386

387 architectures = get_suite_architectures(

388 suite.suite_name, skipall=skip_arch_all, skipsrc=True, session=session

389 )

390

391 out.write(

392 "Architectures: %s\n" % (" ".join(a.arch_string for a in architectures))

393 )

394

395 components = [c.component_name for c in suite.components]

396

397 out.write("Components: %s\n" % (" ".join(components)))

398

399 # For exact compatibility with old g-r, write out Description here instead

400 # of with the rest of the DB fields above

401 if getattr(suite, "description") is not None: 401 ↛ 402line 401 didn't jump to line 402, because the condition on line 401 was never true

402 out.write("Description: %s\n" % suite.description)

403

404 for comp in components:

405 for dirpath, dirnames, filenames in os.walk(

406 os.path.join(self.suite_path(), comp), topdown=True

407 ):

408 if not re_gensubrelease.match(dirpath):

409 continue

410

411 subfile = os.path.join(dirpath, "Release")

412 subrel = open(subfile + ".new", "w")

413

414 for key, dbfield in subattribs:

415 if getattr(suite, dbfield) is not None:

416 subrel.write("%s: %s\n" % (key, getattr(suite, dbfield)))

417

418 for key, dbfield in boolattrs:

419 if getattr(suite, dbfield, False):

420 subrel.write("%s: yes\n" % (key))

421

422 subrel.write("Component: %s%s\n" % (suite_suffix, comp))

423

424 # Urgh, but until we have all the suite/component/arch stuff in the DB,

425 # this'll have to do

426 arch = os.path.split(dirpath)[-1]

427 if arch.startswith("binary-"):

428 arch = arch[7:]

429

430 subrel.write("Architecture: %s\n" % (arch))

431 subrel.close()

432

433 os.rename(subfile + ".new", subfile)

434

435 # Now that we have done the groundwork, we want to get off and add the files with

436 # their checksums to the main Release file

437 oldcwd = os.getcwd()

438

439 os.chdir(self.suite_path())

440

441 hashes = [x for x in RELEASE_HASHES if x.db_name in suite.checksums]

442

443 fileinfo = {}

444 fileinfo_byhash = {}

445

446 uncompnotseen = {}

447

448 for dirpath, dirnames, filenames in os.walk(

449 ".", followlinks=True, topdown=True

450 ):

451 # SuiteSuffix deprecation:

452 # components on security-master are updates/{main,contrib,non-free}, but

453 # we want dists/${suite}/main. Until we can rename the components,

454 # we cheat by having an updates -> . symlink. This should not be visited.

455 if cnf_suite_suffix: 455 ↛ 456line 455 didn't jump to line 456, because the condition on line 455 was never true

456 path = os.path.join(dirpath, cnf_suite_suffix)

457 try:

458 target = os.readlink(path)

459 if target == ".":

460 dirnames.remove(cnf_suite_suffix)

461 except (OSError, ValueError):

462 pass

463 for entry in filenames:

464 if dirpath == "." and entry in ["Release", "Release.gpg", "InRelease"]:

465 continue

466

467 filename = os.path.join(dirpath.lstrip("./"), entry)

468

469 if re_includeinrelease_byhash.match(entry):

470 fileinfo[filename] = fileinfo_byhash[filename] = {}

471 elif re_includeinrelease_plain.match(entry): 471 ↛ 472line 471 didn't jump to line 472, because the condition on line 471 was never true

472 fileinfo[filename] = {}

473 # Skip things we don't want to include

474 else:

475 continue

476

477 with open(filename, "rb") as fd:

478 contents = fd.read()

479

480 # If we find a file for which we have a compressed version and

481 # haven't yet seen the uncompressed one, store the possibility

482 # for future use

483 if entry.endswith(".gz") and filename[:-3] not in uncompnotseen:

484 uncompnotseen[filename[:-3]] = (gzip.GzipFile, filename)

485 elif entry.endswith(".bz2") and filename[:-4] not in uncompnotseen: 485 ↛ 486line 485 didn't jump to line 486, because the condition on line 485 was never true

486 uncompnotseen[filename[:-4]] = (bz2.BZ2File, filename)

487 elif entry.endswith(".xz") and filename[:-3] not in uncompnotseen:

488 uncompnotseen[filename[:-3]] = (XzFile, filename)

489 elif entry.endswith(".zst") and filename[:-3] not in uncompnotseen: 489 ↛ 490line 489 didn't jump to line 490, because the condition on line 489 was never true

490 uncompnotseen[filename[:-3]] = (ZstdFile, filename)

491

492 fileinfo[filename]["len"] = len(contents)

493

494 for hf in hashes:

495 fileinfo[filename][hf.release_field] = hf.func(contents)

496

497 for filename, comp in uncompnotseen.items():

498 # If we've already seen the uncompressed file, we don't

499 # need to do anything again

500 if filename in fileinfo: 500 ↛ 501line 500 didn't jump to line 501, because the condition on line 500 was never true

501 continue

502

503 fileinfo[filename] = {}

504

505 # File handler is comp[0], filename of compressed file is comp[1]

506 contents = comp[0](comp[1], "r").read()

507

508 fileinfo[filename]["len"] = len(contents)

509

510 for hf in hashes:

511 fileinfo[filename][hf.release_field] = hf.func(contents)

512

513 for field in sorted(h.release_field for h in hashes):

514 out.write("%s:\n" % field)

515 for filename in sorted(fileinfo.keys()):

516 out.write(

517 " %s %8d %s\n"

518 % (fileinfo[filename][field], fileinfo[filename]["len"], filename)

519 )

520

521 out.close()

522 os.rename(outfile + ".new", outfile)

523

524 self._update_hashfile_table(session, fileinfo_byhash, hashes)

525 self._make_byhash_links(fileinfo_byhash, hashes)

526 self._make_byhash_base_symlink(fileinfo_byhash, hashes)

527

528 sign_release_dir(suite, os.path.dirname(outfile))

530 os.chdir(oldcwd)

532 return

535def main():

536 global Logger

537

538 cnf = Config()

539

540 for i in ["Help", "Suite", "Force", "Quiet"]:

541 key = "Generate-Releases::Options::%s" % i

542 if key not in cnf: 542 ↛ 540line 542 didn't jump to line 540, because the condition on line 542 was never false

543 cnf[key] = ""

544

545 Arguments = [

546 ("h", "help", "Generate-Releases::Options::Help"),

547 ("a", "archive", "Generate-Releases::Options::Archive", "HasArg"),

548 ("s", "suite", "Generate-Releases::Options::Suite"),

549 ("f", "force", "Generate-Releases::Options::Force"),

550 ("q", "quiet", "Generate-Releases::Options::Quiet"),

551 ("o", "option", "", "ArbItem"),

552 ]

553

554 suite_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)

555 Options = cnf.subtree("Generate-Releases::Options")

556

557 if Options["Help"]:

558 usage()

559

560 Logger = daklog.Logger("generate-releases")

561 pool = DakProcessPool()

562

563 session = DBConn().session()

564

565 if Options["Suite"]:

566 suites = []

567 for s in suite_names:

568 suite = get_suite(s.lower(), session)

569 if suite: 569 ↛ 572line 569 didn't jump to line 572, because the condition on line 569 was never false

570 suites.append(suite)

571 else:

572 print("cannot find suite %s" % s)

573 Logger.log(["cannot find suite %s" % s])

574 else:

575 query = session.query(Suite).filter(Suite.untouchable == False) # noqa:E712

576 if "Archive" in Options: 576 ↛ 581line 576 didn't jump to line 581, because the condition on line 576 was never false

577 archive_names = utils.split_args(Options["Archive"])

578 query = query.join(Suite.archive).filter(

579 Archive.archive_name.in_(archive_names)

580 )

581 suites = query.all()

582

583 for s in suites:

584 # Setup a multiprocessing Pool. As many workers as we have CPU cores.

585 if s.untouchable and not Options["Force"]: 585 ↛ 586line 585 didn't jump to line 586, because the condition on line 585 was never true

586 print("Skipping %s (untouchable)" % s.suite_name)

587 continue

588

589 if not Options["Quiet"]: 589 ↛ 591line 589 didn't jump to line 591, because the condition on line 589 was never false

590 print("Processing %s" % s.suite_name)

591 Logger.log(["Processing release file for Suite: %s" % (s.suite_name)])

592 pool.apply_async(generate_helper, (s.suite_id,))

593

594 # No more work will be added to our pool, close it and then wait for all to finish

595 pool.close()

596 pool.join()

597

598 retcode = pool.overall_status()

599

600 if retcode > 0: 600 ↛ 602line 600 didn't jump to line 602, because the condition on line 600 was never true

601 # TODO: CENTRAL FUNCTION FOR THIS / IMPROVE LOGGING

602 Logger.log(

603 [

604 "Release file generation broken: %s"

605 % (",".join([str(x[1]) for x in pool.results]))

606 ]

607 )

608

609 Logger.close()

610

611 sys.exit(retcode)

612

613

614def generate_helper(suite_id):

615 """

616 This function is called in a new subprocess.

617 """

618 session = DBConn().session()

619 suite = Suite.get(suite_id, session)

620

621 # We allow the process handler to catch and deal with any exceptions

622 rw = ReleaseWriter(suite)

623 rw.generate_release_files()

624

625 return (PROC_STATUS_SUCCESS, "Release file written for %s" % suite.suite_name)

626

627

628#######################################################################################

629

630

631if __name__ == "__main__":

632 main()