dak.import_repository

1 #! /usr/bin/env python3 2 # 3 # Copyright (C) 2015, Ansgar Burchardt <ansgar@debian.org> 4 # 5 # This program is free software; you can redistribute it and/or modify 6 # it under the terms of the GNU General Public License as published by 7 # the Free Software Foundation; either version 2 of the License, or 8 # (at your option) any later version. 9 # 10 # This program is distributed in the hope that it will be useful, 11 # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 # GNU General Public License for more details. 14 # 15 # You should have received a copy of the GNU General Public License along 16 # with this program; if not, write to the Free Software Foundation, Inc., 17 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 19 import sys 20 from collections import defaultdict 21 22 import apt_pkg 23 24 import daklib.archive 25 import daklib.config 26 import daklib.dbconn 27 import daklib.import_repository 28 import daklib.utils 29 30 # This is still work-in-progress and far too incomplete. 31 # ruff: noqa 32 33

34 -def usage(status=0):

35 print( 36 """ 37 dak import-repository 38 --keyring=/usr/share/keyrings/debian-archive-keyring.gpg 39 [--key=${fingerprint}] 40 [--architectures=a,b,c (default: architectures in origin suite)] 41 [--components=main,contrib (default: components in origin suite)] 42 [--target-suite=${suite} (default: origin suite name)] 43 [--add-overrides] 44 [--max-packages=${n} (import at maximum ${n} packages, default: no limit)] 45 http://httpredir.debian.org/debian unstable 46 47 Things to think about: 48 - Import Built-Using sources 49 - all / only referenced 50 - Remove old packages: 51 - by-source: remove source X_v, if no X exists upstream 52 - by-version: remove source X_v, if no X_v exists upstream 53 (X denotes package name, v version, X_v package at a specific version) 54 - Import all or only newest? 55 - Expire binary packages? 56 """ 57 ) 58 sys.exit(status)

59 60

61 -def entry_is_newer(entry, packages):

62 version = entry["Version"] 63 for p in packages[entry["Package"]]: 64 if apt_pkg.version_compare(version, p.version) <= 0: 65 return False 66 return True

67 68

69 -def entry_in_packages(entry, packages):

70 return entry["Package"] in packages

71 72

73 -def get_packages_in_suite(suite):

74 sources = defaultdict(list) 75 for s in suite.sources: 76 sources[s.source].append(s) 77 78 packages = defaultdict(list) 79 for b in suite.binaries: 80 packages[b.package].append(b) 81 82 return sources, packages

83 84

85 -def import_sources( 86 base, 87 sources, 88 transaction, 89 target_suite, 90 component, 91 target_sources, 92 extra_sources, 93 extra_sources_comp, 94 max_packages=None, 95 ):

96 n = 0 97 for entry in sources: 98 if max_packages is not None and n > max_packages: 99 break 100 if entry.get("Extra-Source-Only", "no") == "yes": 101 # Remember package, we might need to import it later. 102 key = (entry["Package"], entry["Version"]) 103 extra_sources[key] = entry 104 extra_sources_comp[key].add(c) 105 continue 106 if not entry_in_packages(entry, target_sources) or entry_is_newer( 107 entry, target_sources 108 ): 109 print("Importing {0}={1}".format(entry["Package"], entry["Version"])) 110 daklib.import_repository.import_source_to_suite( 111 base, entry, transaction, target_suite, component 112 ) 113 n += 1 114 return n

115 116

117 -def import_built_using( 118 base, 119 source, 120 version, 121 transaction, 122 target_suite, 123 component, 124 extra_sources, 125 extra_sources_comp, 126 ):

127 if not daklib.import_repository.source_in_archive( 128 bu_source, bu_version, target_suite.archive 129 ): 130 print("Importing extra source {0}={1}".format(bu_source, bu_version)) 131 key = (bu_source, bu_version) 132 extra_entry = extra_sources.get(key) 133 if extra_entry is None: 134 raise Exception( 135 "Extra source {0}={1} referenced by {2}={3} ({4}) not found in source suite.".format( 136 bu_source, 137 bu_version, 138 entry["Package"], 139 entry["Version"], 140 architecture, 141 ) 142 ) 143 # extra_components = extra_sources_comp[key] 144 if c in components: 145 extra_component = component 146 else: 147 # TODO: Take preferred components from those listed... 148 raise Exception("Not implemented.") 149 daklib.import_repository.import_source_to_suite( 150 base, extra_entry, transaction, target_suite, extra_component 151 )

152 153

154 -def import_packages( 155 base, 156 packages, 157 transaction, 158 target_suite, 159 component, 160 architecture, 161 target_binaries, 162 extra_sources, 163 extra_sources_comp, 164 max_packages=None, 165 ):

166 n = 0 167 for entry in packages: 168 if max_packages is not None and n > max_packages: 169 break 170 if not entry_in_packages(entry, target_binaries) or entry_is_newer( 171 entry, target_binaries 172 ): 173 print( 174 "Importing {0}={1} ({2})".format( 175 entry["Package"], entry["Version"], architecture 176 ) 177 ) 178 # Import Built-Using sources: 179 for bu_source, bu_version in daklib.utils.parse_built_using(entry): 180 import_built_using( 181 base, 182 bu_source, 183 bu_version, 184 transaction, 185 target_suite, 186 component, 187 extra_sources, 188 extra_sources_comp, 189 ) 190 # Import binary: 191 daklib.import_repository.import_package_to_suite( 192 base, entry, transaction, target_suite, component 193 ) 194 n += 1 195 return n

196 197

198 -def main(argv=None):

199 if argv is None: 200 argv = sys.argv 201 202 arguments = [ 203 ("h", "help", "Import-Repository::Help"), 204 ("k", "keyring", "Import-Repository::Keyring", "HasArg"), 205 ("K", "key", "Import-Repository::Key", "HasArg"), 206 ("a", "architectures", "Import-Repository::Architectures", "HasArg"), 207 ("c", "components", "Import-Repository::Components", "HasArg"), 208 ("t", "target-suite", "Import-Repository::Target-Suite", "HasArg"), 209 ("A", "add-overrides", "Import-Repository::AddOverrides"), 210 ("n", "max-packages", "Import-Repository::MaxPackages", "HasArg"), 211 ] 212 213 cnf = daklib.config.Config() 214 argv = apt_pkg.parse_commandline(cnf.Cnf, arguments, argv) 215 options = cnf.subtree("Import-Repository") 216 217 if "Help" in options or len(argv) < 2: 218 usage(0) 219 220 keyring = options.find("Keyring") or None 221 if keyring is None: 222 print("Error: No keyring specified") 223 print() 224 225 if "Key" in options: 226 raise Exception("Not implemented.") 227 228 if "AddOverrides" in options: 229 raise Exception("Not implemented.") 230 231 if "MaxPackages" in options: 232 max_packages = int(options["MaxPackages"]) 233 else: 234 max_packages = None 235 236 base, suite = argv[0:2] 237 238 target_suite_name = options.find("Target-Suite") or suite 239 240 print( 241 "Importing packages from {0}/dists/{1} to {2}".format( 242 base, suite, target_suite_name 243 ) 244 ) 245 with daklib.archive.ArchiveTransaction() as transaction: 246 target_suite = daklib.dbconn.get_suite(target_suite_name, transaction.session) 247 if target_suite is None: 248 daklib.utils.fubar( 249 "Target suite '{0}' is unknown.".format(target_suite_name) 250 ) 251 252 release = daklib.import_repository.obtain_release(base, suite, keyring) 253 target_sources, target_binaries = get_packages_in_suite(target_suite) 254 255 if "Architectures" in options: 256 architectures = options["Architectures"].split(",") 257 else: 258 architectures = ["all"] + release.architectures() 259 260 if "Components" in options: 261 components = options["Components"].split(",") 262 else: 263 components = release.components() 264 265 # TODO: Clean this up... 266 267 n = 0 268 269 # For Extra-Source-Only sources packages, keep a dict 270 # (name, version) -> entry and (name, version) -> set of components 271 # to allow importing needed packages at a later stage 272 extra_sources = dict() 273 extra_sources_comp = defaultdict(set) 274 275 for c in components: 276 component = daklib.dbconn.get_component(c, transaction.session) 277 print("Processing {0}/source...".format(c)) 278 sources = release.sources(c) 279 imported = import_sources( 280 base, 281 sources, 282 transaction, 283 target_suite, 284 component, 285 target_sources, 286 extra_sources, 287 extra_sources_comp, 288 max_packages, 289 ) 290 print(" imported {0} source packages".format(imported)) 291 n += imported 292 if max_packages is not None: 293 max_packages -= n 294 295 for c in components: 296 component = daklib.dbconn.get_component(c, transaction.session) 297 for architecture in architectures: 298 print("Processing {0}/{1}...".format(c, architecture)) 299 packages = release.packages(c, architecture) 300 imported = import_packages( 301 base, 302 packages, 303 transaction, 304 target_suite, 305 component, 306 architecture, 307 target_binaries, 308 extra_sources, 309 extra_sources_comp, 310 max_packages, 311 ) 312 print(" imported {0} binary packages".format(imported)) 313 n += imported 314 if max_packages is not None: 315 max_packages -= n 316 317 transaction.rollback()

318 319 320 if __name__ == "__main__": 321 main() 322

Source Code for Module dak.import_repository