dak.generate_releases

142 - def __init__(self, suite):

143 self.suite = suite

144

145 - def suite_path(self):

146 """ 147 Absolute path to the suite-specific files. 148 """ 149 suite_suffix = utils.suite_suffix(self.suite.suite_name) 150 151 return os.path.join(self.suite.archive.path, 'dists', 152 self.suite.suite_name, suite_suffix)

153

154 - def suite_release_path(self):

155 """ 156 Absolute path where Release files are physically stored. 157 This should be a path that sorts after the dists/ directory. 158 """ 159 cnf = Config() 160 suite_suffix = utils.suite_suffix(self.suite.suite_name) 161 162 return os.path.join(self.suite.archive.path, 'zzz-dists', 163 self.suite.codename or self.suite.suite_name, suite_suffix)

164

165 - def create_release_symlinks(self):

166 """ 167 Create symlinks for Release files. 168 This creates the symlinks for Release files in the `suite_path` 169 to the actual files in `suite_release_path`. 170 """ 171 relpath = os.path.relpath(self.suite_release_path(), self.suite_path()) 172 for f in ("Release", "Release.gpg", "InRelease"): 173 source = os.path.join(relpath, f) 174 dest = os.path.join(self.suite_path(), f) 175 if os.path.lexists(dest): 176 if not os.path.islink(dest): 177 os.unlink(dest) 178 elif os.readlink(dest) == source: 179 continue 180 else: 181 os.unlink(dest) 182 os.symlink(source, dest)

183

184 - def create_output_directories(self):

185 for path in (self.suite_path(), self.suite_release_path()): 186 try: 187 os.makedirs(path) 188 except OSError as e: 189 if e.errno != errno.EEXIST: 190 raise

191

192 - def _update_hashfile_table(self, session, fileinfo, hashes):

193 # Mark all by-hash files as obsolete. We will undo that for the ones 194 # we still reference later. 195 query = """ 196 UPDATE hashfile SET unreferenced = CURRENT_TIMESTAMP 197 WHERE suite_id = :id AND unreferenced IS NULL""" 198 session.execute(query, {'id': self.suite.suite_id}) 199 200 query = "SELECT path FROM hashfile WHERE suite_id = :id" 201 q = session.execute(query, {'id': self.suite.suite_id}) 202 known_hashfiles = set(row[0] for row in q) 203 updated = set() 204 new = set() 205 206 # Update the hashfile table with new or updated files 207 for filename in fileinfo: 208 if not os.path.lexists(filename): 209 # probably an uncompressed index we didn't generate 210 continue 211 byhashdir = os.path.join(os.path.dirname(filename), 'by-hash') 212 for h in hashes: 213 field = h.release_field 214 hashfile = os.path.join(byhashdir, field, fileinfo[filename][field]) 215 if hashfile in known_hashfiles: 216 updated.add(hashfile) 217 else: 218 new.add(hashfile) 219 220 if updated: 221 session.execute(""" 222 UPDATE hashfile SET unreferenced = NULL 223 WHERE path = ANY(:p) AND suite_id = :id""", 224 {'p': list(updated), 'id': self.suite.suite_id}) 225 if new: 226 session.execute(""" 227 INSERT INTO hashfile (path, suite_id) 228 VALUES (:p, :id)""", 229 [{'p': hashfile, 'id': self.suite.suite_id} for hashfile in new]) 230 231 session.commit()

232

233 - def _make_byhash_links(self, fileinfo, hashes):

234 # Create hardlinks in by-hash directories 235 for filename in fileinfo: 236 if not os.path.lexists(filename): 237 # probably an uncompressed index we didn't generate 238 continue 239 240 for h in hashes: 241 field = h.release_field 242 hashfile = os.path.join(os.path.dirname(filename), 'by-hash', field, fileinfo[filename][field]) 243 try: 244 os.makedirs(os.path.dirname(hashfile)) 245 except OSError as exc: 246 if exc.errno != errno.EEXIST: 247 raise 248 try: 249 os.link(filename, hashfile) 250 except OSError as exc: 251 if exc.errno != errno.EEXIST: 252 raise

253

254 - def _make_byhash_base_symlink(self, fileinfo, hashes):

255 # Create symlinks to files in by-hash directories 256 for filename in fileinfo: 257 if not os.path.lexists(filename): 258 # probably an uncompressed index we didn't generate 259 continue 260 261 besthash = hashes[-1] 262 field = besthash.release_field 263 hashfilebase = os.path.join('by-hash', field, fileinfo[filename][field]) 264 hashfile = os.path.join(os.path.dirname(filename), hashfilebase) 265 266 assert os.path.exists(hashfile), 'by-hash file {} is missing'.format(hashfile) 267 268 os.unlink(filename) 269 os.symlink(hashfilebase, filename)

270

271 - def generate_release_files(self):

272 """ 273 Generate Release files for the given suite 274 """ 275 276 suite = self.suite 277 session = object_session(suite) 278 279 # Attribs contains a tuple of field names and the database names to use to 280 # fill them in 281 attribs = (('Origin', 'origin'), 282 ('Label', 'label'), 283 ('Suite', 'release_suite_output'), 284 ('Version', 'version'), 285 ('Codename', 'codename'), 286 ('Changelogs', 'changelog_url'), 287 ) 288 289 # A "Sub" Release file has slightly different fields 290 subattribs = (('Archive', 'suite_name'), 291 ('Origin', 'origin'), 292 ('Label', 'label'), 293 ('Version', 'version')) 294 295 # Boolean stuff. If we find it true in database, write out "yes" into the release file 296 boolattrs = (('NotAutomatic', 'notautomatic'), 297 ('ButAutomaticUpgrades', 'butautomaticupgrades'), 298 ('Acquire-By-Hash', 'byhash'), 299 ) 300 301 cnf = Config() 302 cnf_suite_suffix = cnf.get("Dinstall::SuiteSuffix", "").rstrip("/") 303 304 suite_suffix = utils.suite_suffix(suite.suite_name) 305 306 self.create_output_directories() 307 self.create_release_symlinks() 308 309 outfile = os.path.join(self.suite_release_path(), "Release") 310 out = open(outfile + ".new", "w") 311 312 for key, dbfield in attribs: 313 # Hack to skip NULL Version fields as we used to do this 314 # We should probably just always ignore anything which is None 315 if key in ("Version", "Changelogs") and getattr(suite, dbfield) is None: 316 continue 317 318 out.write("%s: %s\n" % (key, getattr(suite, dbfield))) 319 320 out.write("Date: %s\n" % (time.strftime("%a, %d %b %Y %H:%M:%S UTC", time.gmtime(time.time())))) 321 322 if suite.validtime: 323 validtime = float(suite.validtime) 324 out.write("Valid-Until: %s\n" % (time.strftime("%a, %d %b %Y %H:%M:%S UTC", time.gmtime(time.time() + validtime)))) 325 326 for key, dbfield in boolattrs: 327 if getattr(suite, dbfield, False): 328 out.write("%s: yes\n" % (key)) 329 330 skip_arch_all = True 331 if suite.separate_contents_architecture_all or suite.separate_packages_architecture_all: 332 # According to the Repository format specification: 333 # https://wiki.debian.org/DebianRepository/Format#No-Support-for-Architecture-all 334 # 335 # Clients are not expected to support Packages-all without Contents-all. At the 336 # time of writing, it is not possible to set separate_packages_architecture_all. 337 # However, we add this little assert to stop the bug early. 338 # 339 # If you are here because the assert failed, you probably want to see "update123.py" 340 # and its advice on updating the CHECK constraint. 341 assert suite.separate_contents_architecture_all 342 skip_arch_all = False 343 344 if not suite.separate_packages_architecture_all: 345 out.write("No-Support-for-Architecture-all: Packages\n") 346 347 architectures = get_suite_architectures(suite.suite_name, skipall=skip_arch_all, skipsrc=True, session=session) 348 349 out.write("Architectures: %s\n" % (" ".join(a.arch_string for a in architectures))) 350 351 components = [c.component_name for c in suite.components] 352 353 out.write("Components: %s\n" % (" ".join(components))) 354 355 # For exact compatibility with old g-r, write out Description here instead 356 # of with the rest of the DB fields above 357 if getattr(suite, 'description') is not None: 358 out.write("Description: %s\n" % suite.description) 359 360 for comp in components: 361 for dirpath, dirnames, filenames in os.walk(os.path.join(self.suite_path(), comp), topdown=True): 362 if not re_gensubrelease.match(dirpath): 363 continue 364 365 subfile = os.path.join(dirpath, "Release") 366 subrel = open(subfile + '.new', "w") 367 368 for key, dbfield in subattribs: 369 if getattr(suite, dbfield) is not None: 370 subrel.write("%s: %s\n" % (key, getattr(suite, dbfield))) 371 372 for key, dbfield in boolattrs: 373 if getattr(suite, dbfield, False): 374 subrel.write("%s: yes\n" % (key)) 375 376 subrel.write("Component: %s%s\n" % (suite_suffix, comp)) 377 378 # Urgh, but until we have all the suite/component/arch stuff in the DB, 379 # this'll have to do 380 arch = os.path.split(dirpath)[-1] 381 if arch.startswith('binary-'): 382 arch = arch[7:] 383 384 subrel.write("Architecture: %s\n" % (arch)) 385 subrel.close() 386 387 os.rename(subfile + '.new', subfile) 388 389 # Now that we have done the groundwork, we want to get off and add the files with 390 # their checksums to the main Release file 391 oldcwd = os.getcwd() 392 393 os.chdir(self.suite_path()) 394 395 hashes = [x for x in RELEASE_HASHES if x.db_name in suite.checksums] 396 397 fileinfo = {} 398 fileinfo_byhash = {} 399 400 uncompnotseen = {} 401 402 for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True): 403 # SuiteSuffix deprecation: 404 # components on security-master are updates/{main,contrib,non-free}, but 405 # we want dists/${suite}/main. Until we can rename the components, 406 # we cheat by having an updates -> . symlink. This should not be visited. 407 if cnf_suite_suffix: 408 path = os.path.join(dirpath, cnf_suite_suffix) 409 try: 410 target = os.readlink(path) 411 if target == ".": 412 dirnames.remove(cnf_suite_suffix) 413 except (OSError, ValueError): 414 pass 415 for entry in filenames: 416 if dirpath == '.' and entry in ["Release", "Release.gpg", "InRelease"]: 417 continue 418 419 filename = os.path.join(dirpath.lstrip('./'), entry) 420 421 if re_includeinrelease_byhash.match(entry): 422 fileinfo[filename] = fileinfo_byhash[filename] = {} 423 elif re_includeinrelease_plain.match(entry): 424 fileinfo[filename] = {} 425 # Skip things we don't want to include 426 else: 427 continue 428 429 with open(filename, 'rb') as fd: 430 contents = fd.read() 431 432 # If we find a file for which we have a compressed version and 433 # haven't yet seen the uncompressed one, store the possibility 434 # for future use 435 if entry.endswith(".gz") and filename[:-3] not in uncompnotseen: 436 uncompnotseen[filename[:-3]] = (gzip.GzipFile, filename) 437 elif entry.endswith(".bz2") and filename[:-4] not in uncompnotseen: 438 uncompnotseen[filename[:-4]] = (bz2.BZ2File, filename) 439 elif entry.endswith(".xz") and filename[:-3] not in uncompnotseen: 440 uncompnotseen[filename[:-3]] = (XzFile, filename) 441 elif entry.endswith(".zst") and filename[:-3] not in uncompnotseen: 442 uncompnotseen[filename[:-3]] = (ZstdFile, filename) 443 444 fileinfo[filename]['len'] = len(contents) 445 446 for hf in hashes: 447 fileinfo[filename][hf.release_field] = hf.func(contents) 448 449 for filename, comp in uncompnotseen.items(): 450 # If we've already seen the uncompressed file, we don't 451 # need to do anything again 452 if filename in fileinfo: 453 continue 454 455 fileinfo[filename] = {} 456 457 # File handler is comp[0], filename of compressed file is comp[1] 458 contents = comp[0](comp[1], 'r').read() 459 460 fileinfo[filename]['len'] = len(contents) 461 462 for hf in hashes: 463 fileinfo[filename][hf.release_field] = hf.func(contents) 464 465 for field in sorted(h.release_field for h in hashes): 466 out.write('%s:\n' % field) 467 for filename in sorted(fileinfo.keys()): 468 out.write(" %s %8d %s\n" % (fileinfo[filename][field], fileinfo[filename]['len'], filename)) 469 470 out.close() 471 os.rename(outfile + '.new', outfile) 472 473 self._update_hashfile_table(session, fileinfo_byhash, hashes) 474 self._make_byhash_links(fileinfo_byhash, hashes) 475 self._make_byhash_base_symlink(fileinfo_byhash, hashes) 476 477 sign_release_dir(suite, os.path.dirname(outfile)) 478 479 os.chdir(oldcwd) 480 481 return

Source Code for Module dak.generate_releases