1#! /usr/bin/env python3
2#
3# Copyright (C) 2015, Ansgar Burchardt <ansgar@debian.org>
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License as published by
7# the Free Software Foundation; either version 2 of the License, or
8# (at your option) any later version.
9#
10# This program is distributed in the hope that it will be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13# GNU General Public License for more details.
14#
15# You should have received a copy of the GNU General Public License along
16# with this program; if not, write to the Free Software Foundation, Inc.,
17# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19import sys
20from collections import defaultdict
22import apt_pkg
24import daklib.archive
25import daklib.config
26import daklib.dbconn
27import daklib.import_repository
28import daklib.utils
30# This is still work-in-progress and far too incomplete.
31# ruff: noqa
34def usage(status=0):
35 print(
36 """
37dak import-repository
38 --keyring=/usr/share/keyrings/debian-archive-keyring.gpg
39 [--key=${fingerprint}]
40 [--architectures=a,b,c (default: architectures in origin suite)]
41 [--components=main,contrib (default: components in origin suite)]
42 [--target-suite=${suite} (default: origin suite name)]
43 [--add-overrides]
44 [--max-packages=${n} (import at maximum ${n} packages, default: no limit)]
45 http://httpredir.debian.org/debian unstable
47Things to think about:
48 - Import Built-Using sources
49 - all / only referenced
50 - Remove old packages:
51 - by-source: remove source X_v, if no X exists upstream
52 - by-version: remove source X_v, if no X_v exists upstream
53 (X denotes package name, v version, X_v package at a specific version)
54 - Import all or only newest?
55 - Expire binary packages?
56"""
57 )
58 sys.exit(status)
61def entry_is_newer(entry, packages):
62 version = entry["Version"]
63 for p in packages[entry["Package"]]:
64 if apt_pkg.version_compare(version, p.version) <= 0:
65 return False
66 return True
69def entry_in_packages(entry, packages):
70 return entry["Package"] in packages
73def get_packages_in_suite(suite):
74 sources = defaultdict(list)
75 for s in suite.sources:
76 sources[s.source].append(s)
78 packages = defaultdict(list)
79 for b in suite.binaries:
80 packages[b.package].append(b)
82 return sources, packages
85def import_sources(
86 base,
87 sources,
88 transaction,
89 target_suite,
90 component,
91 target_sources,
92 extra_sources,
93 extra_sources_comp,
94 max_packages=None,
95):
96 n = 0
97 for entry in sources:
98 if max_packages is not None and n > max_packages:
99 break
100 if entry.get("Extra-Source-Only", "no") == "yes":
101 # Remember package, we might need to import it later.
102 key = (entry["Package"], entry["Version"])
103 extra_sources[key] = entry
104 extra_sources_comp[key].add(c)
105 continue
106 if not entry_in_packages(entry, target_sources) or entry_is_newer(
107 entry, target_sources
108 ):
109 print("Importing {0}={1}".format(entry["Package"], entry["Version"]))
110 daklib.import_repository.import_source_to_suite(
111 base, entry, transaction, target_suite, component
112 )
113 n += 1
114 return n
117def import_built_using(
118 base,
119 source,
120 version,
121 transaction,
122 target_suite,
123 component,
124 extra_sources,
125 extra_sources_comp,
126):
127 if not daklib.import_repository.source_in_archive(
128 bu_source, bu_version, target_suite.archive
129 ):
130 print("Importing extra source {0}={1}".format(bu_source, bu_version))
131 key = (bu_source, bu_version)
132 extra_entry = extra_sources.get(key)
133 if extra_entry is None:
134 raise Exception(
135 "Extra source {0}={1} referenced by {2}={3} ({4}) not found in source suite.".format(
136 bu_source,
137 bu_version,
138 entry["Package"],
139 entry["Version"],
140 architecture,
141 )
142 )
143 # extra_components = extra_sources_comp[key]
144 if c in components:
145 extra_component = component
146 else:
147 # TODO: Take preferred components from those listed...
148 raise Exception("Not implemented.")
149 daklib.import_repository.import_source_to_suite(
150 base, extra_entry, transaction, target_suite, extra_component
151 )
154def import_packages(
155 base,
156 packages,
157 transaction,
158 target_suite,
159 component,
160 architecture,
161 target_binaries,
162 extra_sources,
163 extra_sources_comp,
164 max_packages=None,
165):
166 n = 0
167 for entry in packages:
168 if max_packages is not None and n > max_packages:
169 break
170 if not entry_in_packages(entry, target_binaries) or entry_is_newer(
171 entry, target_binaries
172 ):
173 print(
174 "Importing {0}={1} ({2})".format(
175 entry["Package"], entry["Version"], architecture
176 )
177 )
178 # Import Built-Using sources:
179 for bu_source, bu_version in daklib.utils.parse_built_using(entry):
180 import_built_using(
181 base,
182 bu_source,
183 bu_version,
184 transaction,
185 target_suite,
186 component,
187 extra_sources,
188 extra_sources_comp,
189 )
190 # Import binary:
191 daklib.import_repository.import_package_to_suite(
192 base, entry, transaction, target_suite, component
193 )
194 n += 1
195 return n
198def main(argv=None):
199 if argv is None: 199 ↛ 202line 199 didn't jump to line 202
200 argv = sys.argv
202 arguments = [
203 ("h", "help", "Import-Repository::Help"),
204 ("k", "keyring", "Import-Repository::Keyring", "HasArg"),
205 ("K", "key", "Import-Repository::Key", "HasArg"),
206 ("a", "architectures", "Import-Repository::Architectures", "HasArg"),
207 ("c", "components", "Import-Repository::Components", "HasArg"),
208 ("t", "target-suite", "Import-Repository::Target-Suite", "HasArg"),
209 ("A", "add-overrides", "Import-Repository::AddOverrides"),
210 ("n", "max-packages", "Import-Repository::MaxPackages", "HasArg"),
211 ]
213 cnf = daklib.config.Config()
214 argv = apt_pkg.parse_commandline(cnf.Cnf, arguments, argv)
215 options = cnf.subtree("Import-Repository")
217 if "Help" in options or len(argv) < 2: 217 ↛ 220line 217 didn't jump to line 220, because the condition on line 217 was never false
218 usage(0)
220 keyring = options.find("Keyring") or None
221 if keyring is None:
222 print("Error: No keyring specified")
223 print()
225 if "Key" in options:
226 raise Exception("Not implemented.")
228 if "AddOverrides" in options:
229 raise Exception("Not implemented.")
231 if "MaxPackages" in options:
232 max_packages = int(options["MaxPackages"])
233 else:
234 max_packages = None
236 base, suite = argv[0:2]
238 target_suite_name = options.find("Target-Suite") or suite
240 print(
241 "Importing packages from {0}/dists/{1} to {2}".format(
242 base, suite, target_suite_name
243 )
244 )
245 with daklib.archive.ArchiveTransaction() as transaction:
246 target_suite = daklib.dbconn.get_suite(target_suite_name, transaction.session)
247 if target_suite is None:
248 daklib.utils.fubar(
249 "Target suite '{0}' is unknown.".format(target_suite_name)
250 )
252 release = daklib.import_repository.obtain_release(base, suite, keyring)
253 target_sources, target_binaries = get_packages_in_suite(target_suite)
255 if "Architectures" in options:
256 architectures = options["Architectures"].split(",")
257 else:
258 architectures = ["all"] + release.architectures()
260 if "Components" in options:
261 components = options["Components"].split(",")
262 else:
263 components = release.components()
265 # TODO: Clean this up...
267 n = 0
269 # For Extra-Source-Only sources packages, keep a dict
270 # (name, version) -> entry and (name, version) -> set of components
271 # to allow importing needed packages at a later stage
272 extra_sources = dict()
273 extra_sources_comp = defaultdict(set)
275 for c in components:
276 component = daklib.dbconn.get_component(c, transaction.session)
277 print("Processing {0}/source...".format(c))
278 sources = release.sources(c)
279 imported = import_sources(
280 base,
281 sources,
282 transaction,
283 target_suite,
284 component,
285 target_sources,
286 extra_sources,
287 extra_sources_comp,
288 max_packages,
289 )
290 print(" imported {0} source packages".format(imported))
291 n += imported
292 if max_packages is not None:
293 max_packages -= n
295 for c in components:
296 component = daklib.dbconn.get_component(c, transaction.session)
297 for architecture in architectures:
298 print("Processing {0}/{1}...".format(c, architecture))
299 packages = release.packages(c, architecture)
300 imported = import_packages(
301 base,
302 packages,
303 transaction,
304 target_suite,
305 component,
306 architecture,
307 target_binaries,
308 extra_sources,
309 extra_sources_comp,
310 max_packages,
311 )
312 print(" imported {0} binary packages".format(imported))
313 n += imported
314 if max_packages is not None:
315 max_packages -= n
317 transaction.rollback()
320if __name__ == "__main__":
321 main()