dak.generate_releases

150 - def __init__(self, suite):

151 self.suite = suite

152

153 - def suite_path(self):

154 """ 155 Absolute path to the suite-specific files. 156 """ 157 suite_suffix = utils.suite_suffix(self.suite.suite_name) 158 159 return os.path.join( 160 self.suite.archive.path, "dists", self.suite.suite_name, suite_suffix 161 )

162

163 - def suite_release_path(self):

164 """ 165 Absolute path where Release files are physically stored. 166 This should be a path that sorts after the dists/ directory. 167 """ 168 suite_suffix = utils.suite_suffix(self.suite.suite_name) 169 170 return os.path.join( 171 self.suite.archive.path, 172 "zzz-dists", 173 self.suite.codename or self.suite.suite_name, 174 suite_suffix, 175 )

176

177 - def create_release_symlinks(self):

178 """ 179 Create symlinks for Release files. 180 This creates the symlinks for Release files in the `suite_path` 181 to the actual files in `suite_release_path`. 182 """ 183 relpath = os.path.relpath(self.suite_release_path(), self.suite_path()) 184 for f in ("Release", "Release.gpg", "InRelease"): 185 source = os.path.join(relpath, f) 186 dest = os.path.join(self.suite_path(), f) 187 if os.path.lexists(dest): 188 if not os.path.islink(dest): 189 os.unlink(dest) 190 elif os.readlink(dest) == source: 191 continue 192 else: 193 os.unlink(dest) 194 os.symlink(source, dest)

195

196 - def create_output_directories(self):

197 for path in (self.suite_path(), self.suite_release_path()): 198 try: 199 os.makedirs(path) 200 except OSError as e: 201 if e.errno != errno.EEXIST: 202 raise

203

204 - def _update_hashfile_table(self, session, fileinfo, hashes):

205 # Mark all by-hash files as obsolete. We will undo that for the ones 206 # we still reference later. 207 query = """ 208 UPDATE hashfile SET unreferenced = CURRENT_TIMESTAMP 209 WHERE suite_id = :id AND unreferenced IS NULL""" 210 session.execute(query, {"id": self.suite.suite_id}) 211 212 query = "SELECT path FROM hashfile WHERE suite_id = :id" 213 q = session.execute(query, {"id": self.suite.suite_id}) 214 known_hashfiles = set(row[0] for row in q) 215 updated = set() 216 new = set() 217 218 # Update the hashfile table with new or updated files 219 for filename in fileinfo: 220 if not os.path.lexists(filename): 221 # probably an uncompressed index we didn't generate 222 continue 223 byhashdir = os.path.join(os.path.dirname(filename), "by-hash") 224 for h in hashes: 225 field = h.release_field 226 hashfile = os.path.join(byhashdir, field, fileinfo[filename][field]) 227 if hashfile in known_hashfiles: 228 updated.add(hashfile) 229 else: 230 new.add(hashfile) 231 232 if updated: 233 session.execute( 234 """ 235 UPDATE hashfile SET unreferenced = NULL 236 WHERE path = ANY(:p) AND suite_id = :id""", 237 {"p": list(updated), "id": self.suite.suite_id}, 238 ) 239 if new: 240 session.execute( 241 """ 242 INSERT INTO hashfile (path, suite_id) 243 VALUES (:p, :id)""", 244 [{"p": hashfile, "id": self.suite.suite_id} for hashfile in new], 245 ) 246 247 session.commit()

248

249 - def _make_byhash_links(self, fileinfo, hashes):

250 # Create hardlinks in by-hash directories 251 for filename in fileinfo: 252 if not os.path.lexists(filename): 253 # probably an uncompressed index we didn't generate 254 continue 255 256 for h in hashes: 257 field = h.release_field 258 hashfile = os.path.join( 259 os.path.dirname(filename), 260 "by-hash", 261 field, 262 fileinfo[filename][field], 263 ) 264 try: 265 os.makedirs(os.path.dirname(hashfile)) 266 except OSError as exc: 267 if exc.errno != errno.EEXIST: 268 raise 269 try: 270 os.link(filename, hashfile) 271 except OSError as exc: 272 if exc.errno != errno.EEXIST: 273 raise

274

275 - def _make_byhash_base_symlink(self, fileinfo, hashes):

276 # Create symlinks to files in by-hash directories 277 for filename in fileinfo: 278 if not os.path.lexists(filename): 279 # probably an uncompressed index we didn't generate 280 continue 281 282 besthash = hashes[-1] 283 field = besthash.release_field 284 hashfilebase = os.path.join("by-hash", field, fileinfo[filename][field]) 285 hashfile = os.path.join(os.path.dirname(filename), hashfilebase) 286 287 assert os.path.exists(hashfile), "by-hash file {} is missing".format( 288 hashfile 289 ) 290 291 os.unlink(filename) 292 os.symlink(hashfilebase, filename)

293

294 - def generate_release_files(self):

295 """ 296 Generate Release files for the given suite 297 """ 298 299 suite = self.suite 300 session = object_session(suite) 301 302 # Attribs contains a tuple of field names and the database names to use to 303 # fill them in 304 attribs = ( 305 ("Origin", "origin"), 306 ("Label", "label"), 307 ("Suite", "release_suite_output"), 308 ("Version", "version"), 309 ("Codename", "codename"), 310 ("Changelogs", "changelog_url"), 311 ) 312 313 # A "Sub" Release file has slightly different fields 314 subattribs = ( 315 ("Archive", "suite_name"), 316 ("Origin", "origin"), 317 ("Label", "label"), 318 ("Version", "version"), 319 ) 320 321 # Boolean stuff. If we find it true in database, write out "yes" into the release file 322 boolattrs = ( 323 ("NotAutomatic", "notautomatic"), 324 ("ButAutomaticUpgrades", "butautomaticupgrades"), 325 ("Acquire-By-Hash", "byhash"), 326 ) 327 328 cnf = Config() 329 cnf_suite_suffix = cnf.get("Dinstall::SuiteSuffix", "").rstrip("/") 330 331 suite_suffix = utils.suite_suffix(suite.suite_name) 332 333 self.create_output_directories() 334 self.create_release_symlinks() 335 336 outfile = os.path.join(self.suite_release_path(), "Release") 337 out = open(outfile + ".new", "w") 338 339 for key, dbfield in attribs: 340 # Hack to skip NULL Version fields as we used to do this 341 # We should probably just always ignore anything which is None 342 if key in ("Version", "Changelogs") and getattr(suite, dbfield) is None: 343 continue 344 345 out.write("%s: %s\n" % (key, getattr(suite, dbfield))) 346 347 out.write( 348 "Date: %s\n" 349 % (time.strftime("%a, %d %b %Y %H:%M:%S UTC", time.gmtime(time.time()))) 350 ) 351 352 if suite.validtime: 353 validtime = float(suite.validtime) 354 out.write( 355 "Valid-Until: %s\n" 356 % ( 357 time.strftime( 358 "%a, %d %b %Y %H:%M:%S UTC", 359 time.gmtime(time.time() + validtime), 360 ) 361 ) 362 ) 363 364 for key, dbfield in boolattrs: 365 if getattr(suite, dbfield, False): 366 out.write("%s: yes\n" % (key)) 367 368 skip_arch_all = True 369 if ( 370 suite.separate_contents_architecture_all 371 or suite.separate_packages_architecture_all 372 ): 373 # According to the Repository format specification: 374 # https://wiki.debian.org/DebianRepository/Format#No-Support-for-Architecture-all 375 # 376 # Clients are not expected to support Packages-all without Contents-all. At the 377 # time of writing, it is not possible to set separate_packages_architecture_all. 378 # However, we add this little assert to stop the bug early. 379 # 380 # If you are here because the assert failed, you probably want to see "update123.py" 381 # and its advice on updating the CHECK constraint. 382 assert suite.separate_contents_architecture_all 383 skip_arch_all = False 384 385 if not suite.separate_packages_architecture_all: 386 out.write("No-Support-for-Architecture-all: Packages\n") 387 388 architectures = get_suite_architectures( 389 suite.suite_name, skipall=skip_arch_all, skipsrc=True, session=session 390 ) 391 392 out.write( 393 "Architectures: %s\n" % (" ".join(a.arch_string for a in architectures)) 394 ) 395 396 components = [c.component_name for c in suite.components] 397 398 out.write("Components: %s\n" % (" ".join(components))) 399 400 # For exact compatibility with old g-r, write out Description here instead 401 # of with the rest of the DB fields above 402 if getattr(suite, "description") is not None: 403 out.write("Description: %s\n" % suite.description) 404 405 for comp in components: 406 for dirpath, dirnames, filenames in os.walk( 407 os.path.join(self.suite_path(), comp), topdown=True 408 ): 409 if not re_gensubrelease.match(dirpath): 410 continue 411 412 subfile = os.path.join(dirpath, "Release") 413 subrel = open(subfile + ".new", "w") 414 415 for key, dbfield in subattribs: 416 if getattr(suite, dbfield) is not None: 417 subrel.write("%s: %s\n" % (key, getattr(suite, dbfield))) 418 419 for key, dbfield in boolattrs: 420 if getattr(suite, dbfield, False): 421 subrel.write("%s: yes\n" % (key)) 422 423 subrel.write("Component: %s%s\n" % (suite_suffix, comp)) 424 425 # Urgh, but until we have all the suite/component/arch stuff in the DB, 426 # this'll have to do 427 arch = os.path.split(dirpath)[-1] 428 if arch.startswith("binary-"): 429 arch = arch[7:] 430 431 subrel.write("Architecture: %s\n" % (arch)) 432 subrel.close() 433 434 os.rename(subfile + ".new", subfile) 435 436 # Now that we have done the groundwork, we want to get off and add the files with 437 # their checksums to the main Release file 438 oldcwd = os.getcwd() 439 440 os.chdir(self.suite_path()) 441 442 hashes = [x for x in RELEASE_HASHES if x.db_name in suite.checksums] 443 444 fileinfo = {} 445 fileinfo_byhash = {} 446 447 uncompnotseen = {} 448 449 for dirpath, dirnames, filenames in os.walk( 450 ".", followlinks=True, topdown=True 451 ): 452 # SuiteSuffix deprecation: 453 # components on security-master are updates/{main,contrib,non-free}, but 454 # we want dists/${suite}/main. Until we can rename the components, 455 # we cheat by having an updates -> . symlink. This should not be visited. 456 if cnf_suite_suffix: 457 path = os.path.join(dirpath, cnf_suite_suffix) 458 try: 459 target = os.readlink(path) 460 if target == ".": 461 dirnames.remove(cnf_suite_suffix) 462 except (OSError, ValueError): 463 pass 464 for entry in filenames: 465 if dirpath == "." and entry in ["Release", "Release.gpg", "InRelease"]: 466 continue 467 468 filename = os.path.join(dirpath.lstrip("./"), entry) 469 470 if re_includeinrelease_byhash.match(entry): 471 fileinfo[filename] = fileinfo_byhash[filename] = {} 472 elif re_includeinrelease_plain.match(entry): 473 fileinfo[filename] = {} 474 # Skip things we don't want to include 475 else: 476 continue 477 478 with open(filename, "rb") as fd: 479 contents = fd.read() 480 481 # If we find a file for which we have a compressed version and 482 # haven't yet seen the uncompressed one, store the possibility 483 # for future use 484 if entry.endswith(".gz") and filename[:-3] not in uncompnotseen: 485 uncompnotseen[filename[:-3]] = (gzip.GzipFile, filename) 486 elif entry.endswith(".bz2") and filename[:-4] not in uncompnotseen: 487 uncompnotseen[filename[:-4]] = (bz2.BZ2File, filename) 488 elif entry.endswith(".xz") and filename[:-3] not in uncompnotseen: 489 uncompnotseen[filename[:-3]] = (XzFile, filename) 490 elif entry.endswith(".zst") and filename[:-3] not in uncompnotseen: 491 uncompnotseen[filename[:-3]] = (ZstdFile, filename) 492 493 fileinfo[filename]["len"] = len(contents) 494 495 for hf in hashes: 496 fileinfo[filename][hf.release_field] = hf.func(contents) 497 498 for filename, comp in uncompnotseen.items(): 499 # If we've already seen the uncompressed file, we don't 500 # need to do anything again 501 if filename in fileinfo: 502 continue 503 504 fileinfo[filename] = {} 505 506 # File handler is comp[0], filename of compressed file is comp[1] 507 contents = comp[0](comp[1], "r").read() 508 509 fileinfo[filename]["len"] = len(contents) 510 511 for hf in hashes: 512 fileinfo[filename][hf.release_field] = hf.func(contents) 513 514 for field in sorted(h.release_field for h in hashes): 515 out.write("%s:\n" % field) 516 for filename in sorted(fileinfo.keys()): 517 out.write( 518 " %s %8d %s\n" 519 % (fileinfo[filename][field], fileinfo[filename]["len"], filename) 520 ) 521 522 out.close() 523 os.rename(outfile + ".new", outfile) 524 525 self._update_hashfile_table(session, fileinfo_byhash, hashes) 526 self._make_byhash_links(fileinfo_byhash, hashes) 527 self._make_byhash_base_symlink(fileinfo_byhash, hashes) 528 529 sign_release_dir(suite, os.path.dirname(outfile)) 530 531 os.chdir(oldcwd) 532 533 return

Source Code for Module dak.generate_releases