1#! /usr/bin/env python3
3""" generates partial package updates list"""
5###########################################################
7# idea and basic implementation by Anthony, some changes by Andreas
8# parts are stolen from 'dak generate-releases'
9#
10# Copyright (C) 2004, 2005, 2006 Anthony Towns <aj@azure.humbug.org.au>
11# Copyright (C) 2004, 2005 Andreas Barth <aba@not.so.argh.org>
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
28# < elmo> bah, don't bother me with annoying facts
29# < elmo> I was on a roll
32################################################################################
34import asyncio
35import errno
36import os
37import re
38import sys
39import time
40import traceback
42import apt_pkg
44from daklib import utils, pdiff
45from daklib.dbconn import Archive, Component, DBConn, Suite, get_suite, get_suite_architectures
46from daklib.pdiff import PDiffIndex
48re_includeinpdiff = re.compile(r"(Translation-[a-zA-Z_]+\.(?:bz2|xz))")
50################################################################################
52Cnf = None
53Logger = None
54Options = None
56################################################################################
59def usage(exit_code=0):
60 print("""Usage: dak generate-index-diffs [OPTIONS] [suites]
61Write out ed-style diffs to Packages/Source lists
63 -h, --help show this help and exit
64 -a <archive> generate diffs for suites in <archive>
65 -c give the canonical path of the file
66 -p name for the patch (defaults to current time)
67 -d name for the hardlink farm for status
68 -m how many diffs to generate
69 -n take no action
70 -v be verbose and list each file as we work on it
71 """)
72 sys.exit(exit_code)
75def tryunlink(file):
76 try:
77 os.unlink(file)
78 except OSError:
79 print("warning: removing of %s denied" % (file))
82def smartstat(file):
83 for ext in ["", ".gz", ".bz2", ".xz", ".zst"]:
84 if os.path.isfile(file + ext):
85 return (ext, os.stat(file + ext))
86 return (None, None)
89async def smartlink(f, t):
90 async def call_decompressor(cmd, inpath, outpath):
91 with open(inpath, "rb") as rfd, open(outpath, "wb") as wfd:
92 await pdiff.asyncio_check_call(
93 *cmd,
94 stdin=rfd,
95 stdout=wfd,
96 )
98 if os.path.isfile(f): 98 ↛ 99line 98 didn't jump to line 99, because the condition on line 98 was never true
99 os.link(f, t)
100 elif os.path.isfile("%s.gz" % (f)):
101 await call_decompressor(['gzip', '-d'], '{}.gz'.format(f), t)
102 elif os.path.isfile("%s.bz2" % (f)): 102 ↛ 103line 102 didn't jump to line 103, because the condition on line 102 was never true
103 await call_decompressor(['bzip2', '-d'], '{}.bz2'.format(f), t)
104 elif os.path.isfile("%s.xz" % (f)): 104 ↛ 106line 104 didn't jump to line 106, because the condition on line 104 was never false
105 await call_decompressor(['xz', '-d', '-T0'], '{}.xz'.format(f), t)
106 elif os.path.isfile(f"{f}.zst"):
107 await call_decompressor(['zstd', '--decompress'], f'{f}.zst', t)
108 else:
109 print("missing: %s" % (f))
110 raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), f)
113async def genchanges(Options, outdir, oldfile, origfile, maxdiffs=56, merged_pdiffs=False):
114 if "NoAct" in Options: 114 ↛ 115line 114 didn't jump to line 115, because the condition on line 114 was never true
115 print("Not acting on: od: %s, oldf: %s, origf: %s, md: %s" % (outdir, oldfile, origfile, maxdiffs))
116 return
118 patchname = Options["PatchName"]
120 # origfile = /path/to/Packages
121 # oldfile = ./Packages
122 # newfile = ./Packages.tmp
124 # (outdir, oldfile, origfile) = argv
126 (oldext, oldstat) = smartstat(oldfile)
127 (origext, origstat) = smartstat(origfile)
128 if not origstat: 128 ↛ 129line 128 didn't jump to line 129, because the condition on line 128 was never true
129 print("%s: doesn't exist" % (origfile))
130 return
131 # orig file with the (new) compression extension in case it changed
132 old_full_path = oldfile + origext
133 resolved_orig_path = os.path.realpath(origfile + origext)
135 if not oldstat:
136 print("%s: initial run" % origfile)
137 # The target file might have been copying over the symlink as an accident
138 # in a previous run.
139 if os.path.islink(old_full_path): 139 ↛ 140line 139 didn't jump to line 140, because the condition on line 139 was never true
140 os.unlink(old_full_path)
141 os.link(resolved_orig_path, old_full_path)
142 return
144 if oldstat[1:3] == origstat[1:3]: 144 ↛ 145line 144 didn't jump to line 145, because the condition on line 144 was never true
145 return
147 upd = PDiffIndex(outdir, int(maxdiffs), merged_pdiffs)
149 if "CanonicalPath" in Options: 149 ↛ 150line 149 didn't jump to line 150, because the condition on line 149 was never true
150 upd.can_path = Options["CanonicalPath"]
152 # generate_and_add_patch_file needs an uncompressed file
153 # The `newfile` variable is our uncompressed copy of 'oldfile` thanks to
154 # smartlink
155 newfile = oldfile + ".new"
156 if os.path.exists(newfile): 156 ↛ 157line 156 didn't jump to line 157, because the condition on line 156 was never true
157 os.unlink(newfile)
159 await smartlink(origfile, newfile)
161 try:
162 await upd.generate_and_add_patch_file(oldfile, newfile, patchname)
163 finally:
164 os.unlink(newfile)
166 upd.prune_patch_history()
168 for obsolete_patch in upd.find_obsolete_patches(): 168 ↛ 169line 168 didn't jump to line 169, because the loop on line 168 never started
169 tryunlink(obsolete_patch)
171 upd.update_index()
173 if oldfile + oldext != old_full_path and os.path.islink(old_full_path): 173 ↛ 176line 173 didn't jump to line 176, because the condition on line 173 was never true
174 # The target file might have been copying over the symlink as an accident
175 # in a previous run.
176 os.unlink(old_full_path)
178 os.unlink(oldfile + oldext)
179 os.link(resolved_orig_path, old_full_path)
182def main():
183 global Cnf, Options, Logger
185 os.umask(0o002)
187 Cnf = utils.get_conf()
188 Arguments = [('h', "help", "Generate-Index-Diffs::Options::Help"),
189 ('a', 'archive', 'Generate-Index-Diffs::Options::Archive', 'hasArg'),
190 ('c', None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"),
191 ('p', "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"),
192 ('d', "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"),
193 ('m', "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"),
194 ('n', "no-act", "Generate-Index-Diffs::Options::NoAct"),
195 ('v', "verbose", "Generate-Index-Diffs::Options::Verbose"),
196 ]
197 suites = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
198 Options = Cnf.subtree("Generate-Index-Diffs::Options")
199 if "Help" in Options:
200 usage()
202 maxdiffs = Options.get("MaxDiffs::Default", "56")
203 maxpackages = Options.get("MaxDiffs::Packages", maxdiffs)
204 maxcontents = Options.get("MaxDiffs::Contents", maxdiffs)
205 maxsources = Options.get("MaxDiffs::Sources", maxdiffs)
207 # can only be set via config at the moment
208 max_parallel = int(Options.get("MaxParallel", "8"))
210 if "PatchName" not in Options: 210 ↛ 214line 210 didn't jump to line 214, because the condition on line 210 was never false
211 format = "%Y-%m-%d-%H%M.%S"
212 Options["PatchName"] = time.strftime(format)
214 session = DBConn().session()
215 pending_tasks = []
217 if not suites: 217 ↛ 224line 217 didn't jump to line 224, because the condition on line 217 was never false
218 query = session.query(Suite.suite_name)
219 if Options.get('Archive'): 219 ↛ 222line 219 didn't jump to line 222, because the condition on line 219 was never false
220 archives = utils.split_args(Options['Archive'])
221 query = query.join(Suite.archive).filter(Archive.archive_name.in_(archives))
222 suites = [s.suite_name for s in query]
224 for suitename in suites:
225 print("Processing: " + suitename)
227 suiteobj = get_suite(suitename.lower(), session=session)
229 # Use the canonical version of the suite name
230 suite = suiteobj.suite_name
232 if suiteobj.untouchable: 232 ↛ 233line 232 didn't jump to line 233, because the condition on line 232 was never true
233 print("Skipping: " + suite + " (untouchable)")
234 continue
236 skip_all = True
237 if suiteobj.separate_contents_architecture_all or suiteobj.separate_packages_architecture_all:
238 skip_all = False
240 architectures = get_suite_architectures(suite, skipall=skip_all, session=session)
241 components = [c.component_name for c in session.query(Component.component_name)]
243 suite_suffix = utils.suite_suffix(suitename)
244 if components and suite_suffix: 244 ↛ 245line 244 didn't jump to line 245, because the condition on line 244 was never true
245 longsuite = suite + "/" + suite_suffix
246 else:
247 longsuite = suite
249 merged_pdiffs = suiteobj.merged_pdiffs
251 tree = os.path.join(suiteobj.archive.path, 'dists', longsuite)
253 # See if there are Translations which might need a new pdiff
254 cwd = os.getcwd()
255 for component in components:
256 workpath = os.path.join(tree, component, "i18n")
257 if os.path.isdir(workpath): 257 ↛ 255line 257 didn't jump to line 255, because the condition on line 257 was never false
258 os.chdir(workpath)
259 for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True):
260 for entry in filenames:
261 if not re_includeinpdiff.match(entry):
262 continue
263 (fname, fext) = os.path.splitext(entry)
264 processfile = os.path.join(workpath, fname)
265 storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, fname)
266 coroutine = genchanges(Options, processfile + ".diff", storename, processfile, maxdiffs, merged_pdiffs)
267 pending_tasks.append(coroutine)
268 os.chdir(cwd)
270 for archobj in architectures:
271 architecture = archobj.arch_string
273 if architecture == "source":
274 longarch = architecture
275 packages = "Sources"
276 maxsuite = maxsources
277 else:
278 longarch = "binary-%s" % architecture
279 packages = "Packages"
280 maxsuite = maxpackages
282 for component in components:
283 # Process Contents
284 file = "%s/%s/Contents-%s" % (tree, component, architecture)
286 storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture)
287 coroutine = genchanges(Options, file + ".diff", storename, file, maxcontents, merged_pdiffs)
288 pending_tasks.append(coroutine)
290 file = "%s/%s/%s/%s" % (tree, component, longarch, packages)
291 storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, architecture)
292 coroutine = genchanges(Options, file + ".diff", storename, file, maxsuite, merged_pdiffs)
293 pending_tasks.append(coroutine)
295 asyncio.run(process_pdiff_tasks(pending_tasks, max_parallel))
298async def process_pdiff_tasks(pending_coroutines, limit):
299 if limit is not None: 299 ↛ 309line 299 didn't jump to line 309, because the condition on line 299 was never false
300 # If there is a limit, wrap the tasks with a semaphore to handle the limit
301 semaphore = asyncio.Semaphore(limit)
303 async def bounded_task(task):
304 async with semaphore:
305 return await task
307 pending_coroutines = [bounded_task(task) for task in pending_coroutines]
309 print(f"Processing {len(pending_coroutines)} PDiff generation tasks (parallel limit {limit})")
310 start = time.time()
311 pending_tasks = [asyncio.create_task(coroutine) for coroutine in pending_coroutines]
312 done, pending = await asyncio.wait(pending_tasks)
313 duration = round(time.time() - start, 2)
315 errors = False
317 for task in done:
318 try:
319 task.result()
320 except Exception:
321 traceback.print_exc()
322 errors = True
324 if errors: 324 ↛ 325line 324 didn't jump to line 325, because the condition on line 324 was never true
325 print(f"Processing failed after {duration} seconds")
326 sys.exit(1)
328 print(f"Processing finished {duration} seconds")
330################################################################################
333if __name__ == '__main__':
334 main()