1#! /usr/bin/env python3
3"""generates partial package updates list"""
5###########################################################
7# idea and basic implementation by Anthony, some changes by Andreas
8# parts are stolen from 'dak generate-releases'
9#
10# Copyright (C) 2004, 2005, 2006 Anthony Towns <aj@azure.humbug.org.au>
11# Copyright (C) 2004, 2005 Andreas Barth <aba@not.so.argh.org>
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
28# < elmo> bah, don't bother me with annoying facts
29# < elmo> I was on a roll
32################################################################################
34import asyncio
35import errno
36import os
37import re
38import sys
39import time
40import traceback
42import apt_pkg
44from daklib import pdiff, utils
45from daklib.dbconn import (
46 Archive,
47 Component,
48 DBConn,
49 Suite,
50 get_suite,
51 get_suite_architectures,
52)
53from daklib.pdiff import PDiffIndex
55re_includeinpdiff = re.compile(r"(Translation-[a-zA-Z_]+\.(?:bz2|xz))")
57################################################################################
59Cnf = None
60Logger = None
61Options = None
63################################################################################
66def usage(exit_code=0):
67 print(
68 """Usage: dak generate-index-diffs [OPTIONS] [suites]
69Write out ed-style diffs to Packages/Source lists
71 -h, --help show this help and exit
72 -a <archive> generate diffs for suites in <archive>
73 -c give the canonical path of the file
74 -p name for the patch (defaults to current time)
75 -d name for the hardlink farm for status
76 -m how many diffs to generate
77 -n take no action
78 -v be verbose and list each file as we work on it
79 """
80 )
81 sys.exit(exit_code)
84def tryunlink(file):
85 try:
86 os.unlink(file)
87 except OSError:
88 print("warning: removing of %s denied" % (file))
91def smartstat(file):
92 for ext in ["", ".gz", ".bz2", ".xz", ".zst"]:
93 if os.path.isfile(file + ext):
94 return (ext, os.stat(file + ext))
95 return (None, None)
98async def smartlink(f, t):
99 async def call_decompressor(cmd, inpath, outpath):
100 with open(inpath, "rb") as rfd, open(outpath, "wb") as wfd:
101 await pdiff.asyncio_check_call(
102 *cmd,
103 stdin=rfd,
104 stdout=wfd,
105 )
107 if os.path.isfile(f): 107 ↛ 108line 107 didn't jump to line 108, because the condition on line 107 was never true
108 os.link(f, t)
109 elif os.path.isfile("%s.gz" % (f)):
110 await call_decompressor(["gzip", "-d"], "{}.gz".format(f), t)
111 elif os.path.isfile("%s.bz2" % (f)): 111 ↛ 112line 111 didn't jump to line 112, because the condition on line 111 was never true
112 await call_decompressor(["bzip2", "-d"], "{}.bz2".format(f), t)
113 elif os.path.isfile("%s.xz" % (f)): 113 ↛ 115line 113 didn't jump to line 115, because the condition on line 113 was never false
114 await call_decompressor(["xz", "-d", "-T0"], "{}.xz".format(f), t)
115 elif os.path.isfile(f"{f}.zst"):
116 await call_decompressor(["zstd", "--decompress"], f"{f}.zst", t)
117 else:
118 print("missing: %s" % (f))
119 raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), f)
122async def genchanges(
123 Options, outdir, oldfile, origfile, maxdiffs=56, merged_pdiffs=False
124):
125 if "NoAct" in Options: 125 ↛ 126line 125 didn't jump to line 126, because the condition on line 125 was never true
126 print(
127 "Not acting on: od: %s, oldf: %s, origf: %s, md: %s"
128 % (outdir, oldfile, origfile, maxdiffs)
129 )
130 return
132 patchname = Options["PatchName"]
134 # origfile = /path/to/Packages
135 # oldfile = ./Packages
136 # newfile = ./Packages.tmp
138 # (outdir, oldfile, origfile) = argv
140 (oldext, oldstat) = smartstat(oldfile)
141 (origext, origstat) = smartstat(origfile)
142 if not origstat: 142 ↛ 143line 142 didn't jump to line 143, because the condition on line 142 was never true
143 print("%s: doesn't exist" % (origfile))
144 return
145 # orig file with the (new) compression extension in case it changed
146 old_full_path = oldfile + origext
147 resolved_orig_path = os.path.realpath(origfile + origext)
149 if not oldstat:
150 print("%s: initial run" % origfile)
151 # The target file might have been copying over the symlink as an accident
152 # in a previous run.
153 if os.path.islink(old_full_path): 153 ↛ 154line 153 didn't jump to line 154, because the condition on line 153 was never true
154 os.unlink(old_full_path)
155 os.link(resolved_orig_path, old_full_path)
156 return
158 if oldstat[1:3] == origstat[1:3]: 158 ↛ 159line 158 didn't jump to line 159, because the condition on line 158 was never true
159 return
161 upd = PDiffIndex(outdir, int(maxdiffs), merged_pdiffs)
163 if "CanonicalPath" in Options: 163 ↛ 164line 163 didn't jump to line 164, because the condition on line 163 was never true
164 upd.can_path = Options["CanonicalPath"]
166 # generate_and_add_patch_file needs an uncompressed file
167 # The `newfile` variable is our uncompressed copy of 'oldfile` thanks to
168 # smartlink
169 newfile = oldfile + ".new"
170 if os.path.exists(newfile): 170 ↛ 171line 170 didn't jump to line 171, because the condition on line 170 was never true
171 os.unlink(newfile)
173 await smartlink(origfile, newfile)
175 try:
176 await upd.generate_and_add_patch_file(oldfile, newfile, patchname)
177 finally:
178 os.unlink(newfile)
180 upd.prune_patch_history()
182 for obsolete_patch in upd.find_obsolete_patches(): 182 ↛ 183line 182 didn't jump to line 183, because the loop on line 182 never started
183 tryunlink(obsolete_patch)
185 upd.update_index()
187 if oldfile + oldext != old_full_path and os.path.islink(old_full_path): 187 ↛ 190line 187 didn't jump to line 190, because the condition on line 187 was never true
188 # The target file might have been copying over the symlink as an accident
189 # in a previous run.
190 os.unlink(old_full_path)
192 os.unlink(oldfile + oldext)
193 os.link(resolved_orig_path, old_full_path)
196def main():
197 global Cnf, Options, Logger
199 os.umask(0o002)
201 Cnf = utils.get_conf()
202 Arguments = [
203 ("h", "help", "Generate-Index-Diffs::Options::Help"),
204 ("a", "archive", "Generate-Index-Diffs::Options::Archive", "hasArg"),
205 ("c", None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"),
206 ("p", "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"),
207 ("d", "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"),
208 ("m", "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"),
209 ("n", "no-act", "Generate-Index-Diffs::Options::NoAct"),
210 ("v", "verbose", "Generate-Index-Diffs::Options::Verbose"),
211 ]
212 suites = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
213 Options = Cnf.subtree("Generate-Index-Diffs::Options")
214 if "Help" in Options:
215 usage()
217 maxdiffs = Options.get("MaxDiffs::Default", "56")
218 maxpackages = Options.get("MaxDiffs::Packages", maxdiffs)
219 maxcontents = Options.get("MaxDiffs::Contents", maxdiffs)
220 maxsources = Options.get("MaxDiffs::Sources", maxdiffs)
222 # can only be set via config at the moment
223 max_parallel = int(Options.get("MaxParallel", "8"))
225 if "PatchName" not in Options: 225 ↛ 229line 225 didn't jump to line 229, because the condition on line 225 was never false
226 format = "%Y-%m-%d-%H%M.%S"
227 Options["PatchName"] = time.strftime(format)
229 session = DBConn().session()
230 pending_tasks = []
232 if not suites: 232 ↛ 239line 232 didn't jump to line 239, because the condition on line 232 was never false
233 query = session.query(Suite.suite_name)
234 if Options.get("Archive"): 234 ↛ 237line 234 didn't jump to line 237, because the condition on line 234 was never false
235 archives = utils.split_args(Options["Archive"])
236 query = query.join(Suite.archive).filter(Archive.archive_name.in_(archives))
237 suites = [s.suite_name for s in query]
239 for suitename in suites:
240 print("Processing: " + suitename)
242 suiteobj = get_suite(suitename.lower(), session=session)
244 # Use the canonical version of the suite name
245 suite = suiteobj.suite_name
247 if suiteobj.untouchable: 247 ↛ 248line 247 didn't jump to line 248, because the condition on line 247 was never true
248 print("Skipping: " + suite + " (untouchable)")
249 continue
251 skip_all = True
252 if (
253 suiteobj.separate_contents_architecture_all
254 or suiteobj.separate_packages_architecture_all
255 ):
256 skip_all = False
258 architectures = get_suite_architectures(
259 suite, skipall=skip_all, session=session
260 )
261 components = [c.component_name for c in session.query(Component.component_name)]
263 suite_suffix = utils.suite_suffix(suitename)
264 if components and suite_suffix: 264 ↛ 265line 264 didn't jump to line 265, because the condition on line 264 was never true
265 longsuite = suite + "/" + suite_suffix
266 else:
267 longsuite = suite
269 merged_pdiffs = suiteobj.merged_pdiffs
271 tree = os.path.join(suiteobj.archive.path, "dists", longsuite)
273 # See if there are Translations which might need a new pdiff
274 cwd = os.getcwd()
275 for component in components:
276 workpath = os.path.join(tree, component, "i18n")
277 if os.path.isdir(workpath): 277 ↛ 275line 277 didn't jump to line 275, because the condition on line 277 was never false
278 os.chdir(workpath)
279 for dirpath, dirnames, filenames in os.walk(
280 ".", followlinks=True, topdown=True
281 ):
282 for entry in filenames:
283 if not re_includeinpdiff.match(entry):
284 continue
285 (fname, fext) = os.path.splitext(entry)
286 processfile = os.path.join(workpath, fname)
287 storename = "%s/%s_%s_%s" % (
288 Options["TempDir"],
289 suite,
290 component,
291 fname,
292 )
293 coroutine = genchanges(
294 Options,
295 processfile + ".diff",
296 storename,
297 processfile,
298 maxdiffs,
299 merged_pdiffs,
300 )
301 pending_tasks.append(coroutine)
302 os.chdir(cwd)
304 for archobj in architectures:
305 architecture = archobj.arch_string
307 if architecture == "source":
308 longarch = architecture
309 packages = "Sources"
310 maxsuite = maxsources
311 else:
312 longarch = "binary-%s" % architecture
313 packages = "Packages"
314 maxsuite = maxpackages
316 for component in components:
317 # Process Contents
318 file = "%s/%s/Contents-%s" % (tree, component, architecture)
320 storename = "%s/%s_%s_contents_%s" % (
321 Options["TempDir"],
322 suite,
323 component,
324 architecture,
325 )
326 coroutine = genchanges(
327 Options, file + ".diff", storename, file, maxcontents, merged_pdiffs
328 )
329 pending_tasks.append(coroutine)
331 file = "%s/%s/%s/%s" % (tree, component, longarch, packages)
332 storename = "%s/%s_%s_%s" % (
333 Options["TempDir"],
334 suite,
335 component,
336 architecture,
337 )
338 coroutine = genchanges(
339 Options, file + ".diff", storename, file, maxsuite, merged_pdiffs
340 )
341 pending_tasks.append(coroutine)
343 asyncio.run(process_pdiff_tasks(pending_tasks, max_parallel))
346async def process_pdiff_tasks(pending_coroutines, limit):
347 if limit is not None: 347 ↛ 357line 347 didn't jump to line 357, because the condition on line 347 was never false
348 # If there is a limit, wrap the tasks with a semaphore to handle the limit
349 semaphore = asyncio.Semaphore(limit)
351 async def bounded_task(task):
352 async with semaphore:
353 return await task
355 pending_coroutines = [bounded_task(task) for task in pending_coroutines]
357 print(
358 f"Processing {len(pending_coroutines)} PDiff generation tasks (parallel limit {limit})"
359 )
360 start = time.time()
361 pending_tasks = [asyncio.create_task(coroutine) for coroutine in pending_coroutines]
362 done, pending = await asyncio.wait(pending_tasks)
363 duration = round(time.time() - start, 2)
365 errors = False
367 for task in done:
368 try:
369 task.result()
370 except Exception:
371 traceback.print_exc()
372 errors = True
374 if errors: 374 ↛ 375line 374 didn't jump to line 375, because the condition on line 374 was never true
375 print(f"Processing failed after {duration} seconds")
376 sys.exit(1)
378 print(f"Processing finished {duration} seconds")
381################################################################################
384if __name__ == "__main__":
385 main()