1#! /usr/bin/env python3
2#
3# Copyright (C) 2015, Ansgar Burchardt <ansgar@debian.org>
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License as published by
7# the Free Software Foundation; either version 2 of the License, or
8# (at your option) any later version.
9#
10# This program is distributed in the hope that it will be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13# GNU General Public License for more details.
14#
15# You should have received a copy of the GNU General Public License along
16# with this program; if not, write to the Free Software Foundation, Inc.,
17# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19import daklib.archive
20import daklib.config
21import daklib.dbconn
22import daklib.import_repository
23import daklib.utils
25import apt_pkg
26import sys
28from collections import defaultdict
31def usage(status=0):
32 print("""
33dak import-repository
34 --keyring=/usr/share/keyrings/debian-archive-keyring.gpg
35 [--key=${fingerprint}]
36 [--architectures=a,b,c (default: architectures in origin suite)]
37 [--components=main,contrib (default: components in origin suite)]
38 [--target-suite=${suite} (default: origin suite name)]
39 [--add-overrides]
40 [--max-packages=${n} (import at maximum ${n} packages, default: no limit)]
41 http://httpredir.debian.org/debian unstable
43Things to think about:
44 - Import Built-Using sources
45 - all / only referenced
46 - Remove old packages:
47 - by-source: remove source X_v, if no X exists upstream
48 - by-version: remove source X_v, if no X_v exists upstream
49 (X denotes package name, v version, X_v package at a specific version)
50 - Import all or only newest?
51 - Expire binary packages?
52""")
53 sys.exit(status)
56def entry_is_newer(entry, packages):
57 version = entry['Version']
58 for p in packages[entry['Package']]:
59 if apt_pkg.version_compare(version, p.version) <= 0:
60 return False
61 return True
64def entry_in_packages(entry, packages):
65 return entry['Package'] in packages
68def get_packages_in_suite(suite):
69 sources = defaultdict(list)
70 for s in suite.sources:
71 sources[s.source].append(s)
73 packages = defaultdict(list)
74 for b in suite.binaries:
75 packages[b.package].append(b)
77 return sources, packages
80def import_sources(base, sources, transaction, target_suite, component, target_sources, extra_sources, extra_sources_comp, max_packages=None):
81 n = 0
82 for entry in sources:
83 if max_packages is not None and n > max_packages:
84 break
85 if entry.get('Extra-Source-Only', 'no') == 'yes':
86 # Remember package, we might need to import it later.
87 key = (entry['Package'], entry['Version'])
88 extra_sources[key] = entry
89 extra_sources_comp[key].add(c)
90 continue
91 if not entry_in_packages(entry, target_sources) or entry_is_newer(entry, target_sources):
92 print("Importing {0}={1}".format(entry['Package'], entry['Version']))
93 daklib.import_repository.import_source_to_suite(base, entry, transaction, target_suite, component)
94 n += 1
95 return n
98def import_built_using(base, source, version, transaction, target_suite, component, extra_sources, extra_sources_comp):
99 if not daklib.import_repository.source_in_archive(bu_source, bu_version, target_suite.archive):
100 print("Importing extra source {0}={1}".format(bu_source, bu_version))
101 key = (bu_source, bu_version)
102 extra_entry = extra_sources.get(key)
103 if extra_entry is None:
104 raise Exception("Extra source {0}={1} referenced by {2}={3} ({4}) not found in source suite.".format(bu_source, bu_version, entry['Package'], entry['Version'], architecture))
105 extra_components = extra_sources_comp[key]
106 if c in components:
107 extra_component = component
108 else:
109 # TODO: Take preferred components from those listed...
110 raise Exception("Not implemented.")
111 daklib.import_repository.import_source_to_suite(base, extra_entry, transaction, target_suite, extra_component)
114def import_packages(base, packages, transaction, target_suite, component, architecture, target_binaries, extra_sources, extra_sources_comp, max_packages=None):
115 n = 0
116 for entry in packages:
117 if max_packages is not None and n > max_packages:
118 break
119 if not entry_in_packages(entry, target_binaries) or entry_is_newer(entry, target_binaries):
120 print("Importing {0}={1} ({2})".format(entry['Package'], entry['Version'], architecture))
121 # Import Built-Using sources:
122 for bu_source, bu_version in daklib.utils.parse_built_using(entry):
123 import_built_using(base, bu_source, bu_version, transaction, target_suite, component, extra_sources, extra_sources_comp)
124 # Import binary:
125 daklib.import_repository.import_package_to_suite(base, entry, transaction, target_suite, component)
126 n += 1
127 return n
130def main(argv=None):
131 if argv is None: 131 ↛ 134line 131 didn't jump to line 134
132 argv = sys.argv
134 arguments = [
135 ('h', 'help', 'Import-Repository::Help'),
136 ('k', 'keyring', 'Import-Repository::Keyring', 'HasArg'),
137 ('K', 'key', 'Import-Repository::Key', 'HasArg'),
138 ('a', 'architectures', 'Import-Repository::Architectures', 'HasArg'),
139 ('c', 'components', 'Import-Repository::Components', 'HasArg'),
140 ('t', 'target-suite', 'Import-Repository::Target-Suite', 'HasArg'),
141 ('A', 'add-overrides', 'Import-Repository::AddOverrides'),
142 ('n', 'max-packages', 'Import-Repository::MaxPackages', 'HasArg'),
143 ]
145 cnf = daklib.config.Config()
146 argv = apt_pkg.parse_commandline(cnf.Cnf, arguments, argv)
147 options = cnf.subtree('Import-Repository')
149 if 'Help' in options or len(argv) < 2: 149 ↛ 152line 149 didn't jump to line 152, because the condition on line 149 was never false
150 usage(0)
152 keyring = options.find('Keyring') or None
153 if keyring is None:
154 print("Error: No keyring specified")
155 print()
157 if 'Key' in options:
158 raise Exception('Not implemented.')
160 if 'AddOverrides' in options:
161 raise Exception('Not implemented.')
163 if 'MaxPackages' in options:
164 max_packages = int(options['MaxPackages'])
165 else:
166 max_packages = None
168 base, suite = argv[0:2]
170 target_suite_name = options.find('Target-Suite') or suite
172 print("Importing packages from {0}/dists/{1} to {2}".format(base, suite, target_suite_name))
173 with daklib.archive.ArchiveTransaction() as transaction:
174 target_suite = daklib.dbconn.get_suite(target_suite_name, transaction.session)
175 if target_suite is None:
176 daklib.utils.fubar("Target suite '{0}' is unknown.".format(target_suite_name))
178 release = daklib.import_repository.obtain_release(base, suite, keyring)
179 target_sources, target_binaries = get_packages_in_suite(target_suite)
181 if 'Architectures' in options:
182 architectures = options['Architectures'].split(',')
183 else:
184 architectures = ['all'] + release.architectures()
186 if 'Components' in options:
187 components = options['Components'].split(',')
188 else:
189 components = release.components()
191 # TODO: Clean this up...
193 n = 0
195 # For Extra-Source-Only sources packages, keep a dict
196 # (name, version) -> entry and (name, version) -> set of components
197 # to allow importing needed packages at a later stage
198 extra_sources = dict()
199 extra_sources_comp = defaultdict(set)
201 for c in components:
202 component = daklib.dbconn.get_component(c, transaction.session)
203 print("Processing {0}/source...".format(c))
204 sources = release.sources(c)
205 imported = import_sources(base, sources, transaction, target_suite, component, target_sources, extra_sources, extra_sources_comp, max_packages)
206 print(" imported {0} source packages".format(imported))
207 n += imported
208 if max_packages is not None:
209 max_packages -= n
211 for c in components:
212 component = daklib.dbconn.get_component(c, transaction.session)
213 for architecture in architectures:
214 print("Processing {0}/{1}...".format(c, architecture))
215 packages = release.packages(c, architecture)
216 imported = import_packages(base, packages, transaction, target_suite, component, architecture, target_binaries, extra_sources, extra_sources_comp, max_packages)
217 print(" imported {0} binary packages".format(imported))
218 n += imported
219 if max_packages is not None:
220 max_packages -= n
222 transaction.rollback()
225if __name__ == '__main__':
226 main()