Coverage for dak/generate_index_diffs.py: 83%
185 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-08-03 16:46 +0000
1"""generates partial package updates list"""
3###########################################################
5# idea and basic implementation by Anthony, some changes by Andreas
6# parts are stolen from 'dak generate-releases'
7#
8# Copyright (C) 2004, 2005, 2006 Anthony Towns <aj@azure.humbug.org.au>
9# Copyright (C) 2004, 2005 Andreas Barth <aba@not.so.argh.org>
11# This program is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation; either version 2 of the License, or
14# (at your option) any later version.
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
21# You should have received a copy of the GNU General Public License
22# along with this program; if not, write to the Free Software
23# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26# < elmo> bah, don't bother me with annoying facts
27# < elmo> I was on a roll
30################################################################################
32import asyncio
33import errno
34import os
35import re
36import sys
37import time
38import traceback
39from typing import NoReturn
41import apt_pkg
42from sqlalchemy import select
44from daklib import pdiff, utils
45from daklib.dbconn import (
46 Archive,
47 Component,
48 DBConn,
49 Suite,
50 get_suite,
51 get_suite_architectures,
52)
53from daklib.pdiff import PDiffIndex
55re_includeinpdiff = re.compile(r"(Translation-[a-zA-Z_]+\.(?:bz2|xz))")
57################################################################################
59Cnf: apt_pkg.Configuration
60Options: apt_pkg.Configuration
62################################################################################
65def usage(exit_code=0) -> NoReturn:
66 print(
67 """Usage: dak generate-index-diffs [OPTIONS] [suites]
68Write out ed-style diffs to Packages/Source lists
70 -h, --help show this help and exit
71 -a <archive> generate diffs for suites in <archive>
72 -c give the canonical path of the file
73 -p name for the patch (defaults to current time)
74 -d name for the hardlink farm for status
75 -m how many diffs to generate
76 -n take no action
77 -v be verbose and list each file as we work on it
78 """
79 )
80 sys.exit(exit_code)
83def tryunlink(file: str) -> None:
84 try:
85 os.unlink(file)
86 except OSError:
87 print("warning: removing of %s denied" % (file))
90def smartstat(file: str) -> tuple[str, os.stat_result] | tuple[None, None]:
91 for ext in ["", ".gz", ".bz2", ".xz", ".zst"]:
92 if os.path.isfile(file + ext):
93 return (ext, os.stat(file + ext))
94 return (None, None)
97async def smartlink(f: str, t: str) -> None:
98 async def call_decompressor(cmd, inpath, outpath):
99 with open(inpath, "rb") as rfd, open(outpath, "wb") as wfd:
100 await pdiff.asyncio_check_call(
101 *cmd,
102 stdin=rfd,
103 stdout=wfd,
104 )
106 if os.path.isfile(f): 106 ↛ 107line 106 didn't jump to line 107 because the condition on line 106 was never true
107 os.link(f, t)
108 elif os.path.isfile("%s.gz" % (f)):
109 await call_decompressor(["gzip", "-d"], "{}.gz".format(f), t)
110 elif os.path.isfile("%s.bz2" % (f)): 110 ↛ 111line 110 didn't jump to line 111 because the condition on line 110 was never true
111 await call_decompressor(["bzip2", "-d"], "{}.bz2".format(f), t)
112 elif os.path.isfile("%s.xz" % (f)): 112 ↛ 114line 112 didn't jump to line 114 because the condition on line 112 was always true
113 await call_decompressor(["xz", "-d", "-T0"], "{}.xz".format(f), t)
114 elif os.path.isfile(f"{f}.zst"):
115 await call_decompressor(["zstd", "--decompress"], f"{f}.zst", t)
116 else:
117 print("missing: %s" % (f))
118 raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), f)
121async def genchanges(
122 Options: apt_pkg.Configuration,
123 outdir: str,
124 oldfile: str,
125 origfile: str,
126 maxdiffs=56,
127 merged_pdiffs=False,
128) -> None:
129 if "NoAct" in Options: 129 ↛ 130line 129 didn't jump to line 130 because the condition on line 129 was never true
130 print(
131 "Not acting on: od: %s, oldf: %s, origf: %s, md: %s"
132 % (outdir, oldfile, origfile, maxdiffs)
133 )
134 return
136 patchname = Options["PatchName"]
138 # origfile = /path/to/Packages
139 # oldfile = ./Packages
140 # newfile = ./Packages.tmp
142 # (outdir, oldfile, origfile) = argv
144 (oldext, oldstat) = smartstat(oldfile)
145 (origext, origstat) = smartstat(origfile)
146 if not origstat: 146 ↛ 147line 146 didn't jump to line 147 because the condition on line 146 was never true
147 print("%s: doesn't exist" % (origfile))
148 return
149 # orig file with the (new) compression extension in case it changed
150 assert origext is not None
151 old_full_path = oldfile + origext
152 resolved_orig_path = os.path.realpath(origfile + origext)
154 if not oldstat:
155 print("%s: initial run" % origfile)
156 # The target file might have been copying over the symlink as an accident
157 # in a previous run.
158 if os.path.islink(old_full_path): 158 ↛ 159line 158 didn't jump to line 159 because the condition on line 158 was never true
159 os.unlink(old_full_path)
160 os.link(resolved_orig_path, old_full_path)
161 return
162 assert oldext is not None
164 if oldstat[1:3] == origstat[1:3]: 164 ↛ 165line 164 didn't jump to line 165 because the condition on line 164 was never true
165 return
167 upd = PDiffIndex(outdir, int(maxdiffs), merged_pdiffs)
169 if "CanonicalPath" in Options: 169 ↛ 170line 169 didn't jump to line 170 because the condition on line 169 was never true
170 upd.can_path = Options["CanonicalPath"]
172 # generate_and_add_patch_file needs an uncompressed file
173 # The `newfile` variable is our uncompressed copy of 'oldfile` thanks to
174 # smartlink
175 newfile = oldfile + ".new"
176 if os.path.exists(newfile): 176 ↛ 177line 176 didn't jump to line 177 because the condition on line 176 was never true
177 os.unlink(newfile)
179 await smartlink(origfile, newfile)
181 try:
182 await upd.generate_and_add_patch_file(oldfile, newfile, patchname)
183 finally:
184 os.unlink(newfile)
186 upd.prune_patch_history()
188 for obsolete_patch in upd.find_obsolete_patches():
189 tryunlink(obsolete_patch)
191 upd.update_index()
193 if oldfile + oldext != old_full_path and os.path.islink(old_full_path): 193 ↛ 196line 193 didn't jump to line 196 because the condition on line 193 was never true
194 # The target file might have been copying over the symlink as an accident
195 # in a previous run.
196 os.unlink(old_full_path)
198 os.unlink(oldfile + oldext)
199 os.link(resolved_orig_path, old_full_path)
202def main() -> None:
203 global Cnf, Options
205 os.umask(0o002)
207 Cnf = utils.get_conf()
208 Arguments = [
209 ("h", "help", "Generate-Index-Diffs::Options::Help"),
210 ("a", "archive", "Generate-Index-Diffs::Options::Archive", "hasArg"),
211 ("c", None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"),
212 ("p", "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"),
213 ("d", "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"),
214 ("m", "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"),
215 ("n", "no-act", "Generate-Index-Diffs::Options::NoAct"),
216 ("v", "verbose", "Generate-Index-Diffs::Options::Verbose"),
217 ]
218 suites = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) # type: ignore[attr-defined]
219 Options = Cnf.subtree("Generate-Index-Diffs::Options") # type: ignore[attr-defined]
220 if "Help" in Options:
221 usage()
223 maxdiffs = Options.get("MaxDiffs::Default", "56")
224 maxpackages = Options.get("MaxDiffs::Packages", maxdiffs)
225 maxcontents = Options.get("MaxDiffs::Contents", maxdiffs)
226 maxsources = Options.get("MaxDiffs::Sources", maxdiffs)
228 # can only be set via config at the moment
229 max_parallel = int(Options.get("MaxParallel", "8"))
231 if "PatchName" not in Options: 231 ↛ 235line 231 didn't jump to line 235 because the condition on line 231 was always true
232 format = "%Y-%m-%d-%H%M.%S"
233 Options["PatchName"] = time.strftime(format) # type: ignore[index]
235 session = DBConn().session()
236 pending_tasks = []
238 if not suites: 238 ↛ 245line 238 didn't jump to line 245 because the condition on line 238 was always true
239 query = select(Suite.suite_name)
240 if Options.get("Archive"): 240 ↛ 243line 240 didn't jump to line 243 because the condition on line 240 was always true
241 archives = utils.split_args(Options["Archive"])
242 query = query.join(Suite.archive).where(Archive.archive_name.in_(archives))
243 suites = list(session.scalars(query))
245 for suitename in suites:
246 print("Processing: " + suitename)
248 suiteobj = get_suite(suitename.lower(), session=session)
249 assert suiteobj is not None
251 # Use the canonical version of the suite name
252 suite = suiteobj.suite_name
254 if suiteobj.untouchable: 254 ↛ 255line 254 didn't jump to line 255 because the condition on line 254 was never true
255 print("Skipping: " + suite + " (untouchable)")
256 continue
258 skip_all = True
259 if (
260 suiteobj.separate_contents_architecture_all
261 or suiteobj.separate_packages_architecture_all
262 ):
263 skip_all = False
265 architectures = get_suite_architectures(
266 suite, skipall=skip_all, session=session
267 )
268 components = list(session.scalars(select(Component.component_name)))
270 suite_suffix = utils.suite_suffix(suitename)
271 if components and suite_suffix: 271 ↛ 272line 271 didn't jump to line 272 because the condition on line 271 was never true
272 longsuite = suite + "/" + suite_suffix
273 else:
274 longsuite = suite
276 merged_pdiffs = suiteobj.merged_pdiffs
278 tree = os.path.join(suiteobj.archive.path, "dists", longsuite)
280 # See if there are Translations which might need a new pdiff
281 cwd = os.getcwd()
282 for component in components:
283 workpath = os.path.join(tree, component, "i18n")
284 if os.path.isdir(workpath): 284 ↛ 282line 284 didn't jump to line 282 because the condition on line 284 was always true
285 os.chdir(workpath)
286 for dirpath, dirnames, filenames in os.walk(
287 ".", followlinks=True, topdown=True
288 ):
289 for entry in filenames:
290 if not re_includeinpdiff.match(entry):
291 continue
292 (fname, fext) = os.path.splitext(entry)
293 processfile = os.path.join(workpath, fname)
294 storename = "%s/%s_%s_%s" % (
295 Options["TempDir"],
296 suite,
297 component,
298 fname,
299 )
300 coroutine = genchanges(
301 Options,
302 processfile + ".diff",
303 storename,
304 processfile,
305 maxdiffs,
306 merged_pdiffs,
307 )
308 pending_tasks.append(coroutine)
309 os.chdir(cwd)
311 for archobj in architectures:
312 architecture = archobj.arch_string
314 if architecture == "source":
315 longarch = architecture
316 packages = "Sources"
317 maxsuite = maxsources
318 else:
319 longarch = "binary-%s" % architecture
320 packages = "Packages"
321 maxsuite = maxpackages
323 for component in components:
324 # Process Contents
325 file = "%s/%s/Contents-%s" % (tree, component, architecture)
327 storename = "%s/%s_%s_contents_%s" % (
328 Options["TempDir"],
329 suite,
330 component,
331 architecture,
332 )
333 coroutine = genchanges(
334 Options, file + ".diff", storename, file, maxcontents, merged_pdiffs
335 )
336 pending_tasks.append(coroutine)
338 file = "%s/%s/%s/%s" % (tree, component, longarch, packages)
339 storename = "%s/%s_%s_%s" % (
340 Options["TempDir"],
341 suite,
342 component,
343 architecture,
344 )
345 coroutine = genchanges(
346 Options, file + ".diff", storename, file, maxsuite, merged_pdiffs
347 )
348 pending_tasks.append(coroutine)
350 asyncio.run(process_pdiff_tasks(pending_tasks, max_parallel))
353async def process_pdiff_tasks(pending_coroutines, limit):
354 if limit is not None: 354 ↛ 364line 354 didn't jump to line 364 because the condition on line 354 was always true
355 # If there is a limit, wrap the tasks with a semaphore to handle the limit
356 semaphore = asyncio.Semaphore(limit)
358 async def bounded_task(task):
359 async with semaphore:
360 return await task
362 pending_coroutines = [bounded_task(task) for task in pending_coroutines]
364 print(
365 f"Processing {len(pending_coroutines)} PDiff generation tasks (parallel limit {limit})"
366 )
367 start = time.time()
368 pending_tasks = [asyncio.create_task(coroutine) for coroutine in pending_coroutines]
369 done, pending = await asyncio.wait(pending_tasks)
370 duration = round(time.time() - start, 2)
372 errors = False
374 for task in done:
375 try:
376 task.result()
377 except Exception:
378 traceback.print_exc()
379 errors = True
381 if errors: 381 ↛ 382line 381 didn't jump to line 382 because the condition on line 381 was never true
382 print(f"Processing failed after {duration} seconds")
383 sys.exit(1)
385 print(f"Processing finished {duration} seconds")