dak.generate_index

1 #! /usr/bin/env python3 2 3 """generates partial package updates list""" 4 5 ########################################################### 6 7 # idea and basic implementation by Anthony, some changes by Andreas 8 # parts are stolen from 'dak generate-releases' 9 # 10 # Copyright (C) 2004, 2005, 2006 Anthony Towns <aj@azure.humbug.org.au> 11 # Copyright (C) 2004, 2005 Andreas Barth <aba@not.so.argh.org> 12 13 # This program is free software; you can redistribute it and/or modify 14 # it under the terms of the GNU General Public License as published by 15 # the Free Software Foundation; either version 2 of the License, or 16 # (at your option) any later version. 17 18 # This program is distributed in the hope that it will be useful, 19 # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 21 # GNU General Public License for more details. 22 23 # You should have received a copy of the GNU General Public License 24 # along with this program; if not, write to the Free Software 25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 26 27 28 # < elmo> bah, don't bother me with annoying facts 29 # < elmo> I was on a roll 30 31 32 ################################################################################ 33 34 import asyncio 35 import errno 36 import os 37 import re 38 import sys 39 import time 40 import traceback 41 42 import apt_pkg 43 44 from daklib import pdiff, utils 45 from daklib.dbconn import ( 46 Archive, 47 Component, 48 DBConn, 49 Suite, 50 get_suite, 51 get_suite_architectures, 52 ) 53 from daklib.pdiff import PDiffIndex 54 55 re_includeinpdiff = re.compile(r"(Translation-[a-zA-Z_]+\.(?:bz2|xz))") 56 57 ################################################################################ 58 59 Cnf = None 60 Logger = None 61 Options = None 62 63 ################################################################################ 64 65

66 -def usage(exit_code=0):

67 print( 68 """Usage: dak generate-index-diffs [OPTIONS] [suites] 69 Write out ed-style diffs to Packages/Source lists 70 71 -h, --help show this help and exit 72 -a <archive> generate diffs for suites in <archive> 73 -c give the canonical path of the file 74 -p name for the patch (defaults to current time) 75 -d name for the hardlink farm for status 76 -m how many diffs to generate 77 -n take no action 78 -v be verbose and list each file as we work on it 79 """ 80 ) 81 sys.exit(exit_code)

82 83

84 -def tryunlink(file):

85 try: 86 os.unlink(file) 87 except OSError: 88 print("warning: removing of %s denied" % (file))

89 90

91 -def smartstat(file):

92 for ext in ["", ".gz", ".bz2", ".xz", ".zst"]: 93 if os.path.isfile(file + ext): 94 return (ext, os.stat(file + ext)) 95 return (None, None)

96 97

98 -async def smartlink(f, t):

99 async def call_decompressor(cmd, inpath, outpath): 100 with open(inpath, "rb") as rfd, open(outpath, "wb") as wfd: 101 await pdiff.asyncio_check_call( 102 *cmd, 103 stdin=rfd, 104 stdout=wfd, 105 )

106 107 if os.path.isfile(f): 108 os.link(f, t) 109 elif os.path.isfile("%s.gz" % (f)): 110 await call_decompressor(["gzip", "-d"], "{}.gz".format(f), t) 111 elif os.path.isfile("%s.bz2" % (f)): 112 await call_decompressor(["bzip2", "-d"], "{}.bz2".format(f), t) 113 elif os.path.isfile("%s.xz" % (f)): 114 await call_decompressor(["xz", "-d", "-T0"], "{}.xz".format(f), t) 115 elif os.path.isfile(f"{f}.zst"): 116 await call_decompressor(["zstd", "--decompress"], f"{f}.zst", t) 117 else: 118 print("missing: %s" % (f)) 119 raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), f) 120 121

122 -async def genchanges( 123 Options, outdir, oldfile, origfile, maxdiffs=56, merged_pdiffs=False 124 ):

125 if "NoAct" in Options: 126 print( 127 "Not acting on: od: %s, oldf: %s, origf: %s, md: %s" 128 % (outdir, oldfile, origfile, maxdiffs) 129 ) 130 return 131 132 patchname = Options["PatchName"] 133 134 # origfile = /path/to/Packages 135 # oldfile = ./Packages 136 # newfile = ./Packages.tmp 137 138 # (outdir, oldfile, origfile) = argv 139 140 (oldext, oldstat) = smartstat(oldfile) 141 (origext, origstat) = smartstat(origfile) 142 if not origstat: 143 print("%s: doesn't exist" % (origfile)) 144 return 145 # orig file with the (new) compression extension in case it changed 146 old_full_path = oldfile + origext 147 resolved_orig_path = os.path.realpath(origfile + origext) 148 149 if not oldstat: 150 print("%s: initial run" % origfile) 151 # The target file might have been copying over the symlink as an accident 152 # in a previous run. 153 if os.path.islink(old_full_path): 154 os.unlink(old_full_path) 155 os.link(resolved_orig_path, old_full_path) 156 return 157 158 if oldstat[1:3] == origstat[1:3]: 159 return 160 161 upd = PDiffIndex(outdir, int(maxdiffs), merged_pdiffs) 162 163 if "CanonicalPath" in Options: 164 upd.can_path = Options["CanonicalPath"] 165 166 # generate_and_add_patch_file needs an uncompressed file 167 # The `newfile` variable is our uncompressed copy of 'oldfile` thanks to 168 # smartlink 169 newfile = oldfile + ".new" 170 if os.path.exists(newfile): 171 os.unlink(newfile) 172 173 await smartlink(origfile, newfile) 174 175 try: 176 await upd.generate_and_add_patch_file(oldfile, newfile, patchname) 177 finally: 178 os.unlink(newfile) 179 180 upd.prune_patch_history() 181 182 for obsolete_patch in upd.find_obsolete_patches(): 183 tryunlink(obsolete_patch) 184 185 upd.update_index() 186 187 if oldfile + oldext != old_full_path and os.path.islink(old_full_path): 188 # The target file might have been copying over the symlink as an accident 189 # in a previous run. 190 os.unlink(old_full_path) 191 192 os.unlink(oldfile + oldext) 193 os.link(resolved_orig_path, old_full_path)

194 195

196 -def main():

197 global Cnf, Options, Logger 198 199 os.umask(0o002) 200 201 Cnf = utils.get_conf() 202 Arguments = [ 203 ("h", "help", "Generate-Index-Diffs::Options::Help"), 204 ("a", "archive", "Generate-Index-Diffs::Options::Archive", "hasArg"), 205 ("c", None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"), 206 ("p", "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"), 207 ("d", "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"), 208 ("m", "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"), 209 ("n", "no-act", "Generate-Index-Diffs::Options::NoAct"), 210 ("v", "verbose", "Generate-Index-Diffs::Options::Verbose"), 211 ] 212 suites = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) 213 Options = Cnf.subtree("Generate-Index-Diffs::Options") 214 if "Help" in Options: 215 usage() 216 217 maxdiffs = Options.get("MaxDiffs::Default", "56") 218 maxpackages = Options.get("MaxDiffs::Packages", maxdiffs) 219 maxcontents = Options.get("MaxDiffs::Contents", maxdiffs) 220 maxsources = Options.get("MaxDiffs::Sources", maxdiffs) 221 222 # can only be set via config at the moment 223 max_parallel = int(Options.get("MaxParallel", "8")) 224 225 if "PatchName" not in Options: 226 format = "%Y-%m-%d-%H%M.%S" 227 Options["PatchName"] = time.strftime(format) 228 229 session = DBConn().session() 230 pending_tasks = [] 231 232 if not suites: 233 query = session.query(Suite.suite_name) 234 if Options.get("Archive"): 235 archives = utils.split_args(Options["Archive"]) 236 query = query.join(Suite.archive).filter(Archive.archive_name.in_(archives)) 237 suites = [s.suite_name for s in query] 238 239 for suitename in suites: 240 print("Processing: " + suitename) 241 242 suiteobj = get_suite(suitename.lower(), session=session) 243 244 # Use the canonical version of the suite name 245 suite = suiteobj.suite_name 246 247 if suiteobj.untouchable: 248 print("Skipping: " + suite + " (untouchable)") 249 continue 250 251 skip_all = True 252 if ( 253 suiteobj.separate_contents_architecture_all 254 or suiteobj.separate_packages_architecture_all 255 ): 256 skip_all = False 257 258 architectures = get_suite_architectures( 259 suite, skipall=skip_all, session=session 260 ) 261 components = [c.component_name for c in session.query(Component.component_name)] 262 263 suite_suffix = utils.suite_suffix(suitename) 264 if components and suite_suffix: 265 longsuite = suite + "/" + suite_suffix 266 else: 267 longsuite = suite 268 269 merged_pdiffs = suiteobj.merged_pdiffs 270 271 tree = os.path.join(suiteobj.archive.path, "dists", longsuite) 272 273 # See if there are Translations which might need a new pdiff 274 cwd = os.getcwd() 275 for component in components: 276 workpath = os.path.join(tree, component, "i18n") 277 if os.path.isdir(workpath): 278 os.chdir(workpath) 279 for dirpath, dirnames, filenames in os.walk( 280 ".", followlinks=True, topdown=True 281 ): 282 for entry in filenames: 283 if not re_includeinpdiff.match(entry): 284 continue 285 (fname, fext) = os.path.splitext(entry) 286 processfile = os.path.join(workpath, fname) 287 storename = "%s/%s_%s_%s" % ( 288 Options["TempDir"], 289 suite, 290 component, 291 fname, 292 ) 293 coroutine = genchanges( 294 Options, 295 processfile + ".diff", 296 storename, 297 processfile, 298 maxdiffs, 299 merged_pdiffs, 300 ) 301 pending_tasks.append(coroutine) 302 os.chdir(cwd) 303 304 for archobj in architectures: 305 architecture = archobj.arch_string 306 307 if architecture == "source": 308 longarch = architecture 309 packages = "Sources" 310 maxsuite = maxsources 311 else: 312 longarch = "binary-%s" % architecture 313 packages = "Packages" 314 maxsuite = maxpackages 315 316 for component in components: 317 # Process Contents 318 file = "%s/%s/Contents-%s" % (tree, component, architecture) 319 320 storename = "%s/%s_%s_contents_%s" % ( 321 Options["TempDir"], 322 suite, 323 component, 324 architecture, 325 ) 326 coroutine = genchanges( 327 Options, file + ".diff", storename, file, maxcontents, merged_pdiffs 328 ) 329 pending_tasks.append(coroutine) 330 331 file = "%s/%s/%s/%s" % (tree, component, longarch, packages) 332 storename = "%s/%s_%s_%s" % ( 333 Options["TempDir"], 334 suite, 335 component, 336 architecture, 337 ) 338 coroutine = genchanges( 339 Options, file + ".diff", storename, file, maxsuite, merged_pdiffs 340 ) 341 pending_tasks.append(coroutine) 342 343 asyncio.run(process_pdiff_tasks(pending_tasks, max_parallel))

344 345

346 -async def process_pdiff_tasks(pending_coroutines, limit):

347 if limit is not None: 348 # If there is a limit, wrap the tasks with a semaphore to handle the limit 349 semaphore = asyncio.Semaphore(limit) 350 351 async def bounded_task(task): 352 async with semaphore: 353 return await task

354 355 pending_coroutines = [bounded_task(task) for task in pending_coroutines] 356 357 print( 358 f"Processing {len(pending_coroutines)} PDiff generation tasks (parallel limit {limit})" 359 ) 360 start = time.time() 361 pending_tasks = [asyncio.create_task(coroutine) for coroutine in pending_coroutines] 362 done, pending = await asyncio.wait(pending_tasks) 363 duration = round(time.time() - start, 2) 364 365 errors = False 366 367 for task in done: 368 try: 369 task.result() 370 except Exception: 371 traceback.print_exc() 372 errors = True 373 374 if errors: 375 print(f"Processing failed after {duration} seconds") 376 sys.exit(1) 377 378 print(f"Processing finished {duration} seconds") 379 380 381 ################################################################################ 382 383 384 if __name__ == "__main__": 385 main() 386

Source Code for Module dak.generate_index_diffs