Coverage for dak/import_repository.py: 15%

1#! /usr/bin/env python3 

2# 

3# Copyright (C) 2015, Ansgar Burchardt <ansgar@debian.org> 

4# 

5# This program is free software; you can redistribute it and/or modify 

6# it under the terms of the GNU General Public License as published by 

7# the Free Software Foundation; either version 2 of the License, or 

8# (at your option) any later version. 

9# 

10# This program is distributed in the hope that it will be useful, 

11# but WITHOUT ANY WARRANTY; without even the implied warranty of 

12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

13# GNU General Public License for more details. 

14# 

15# You should have received a copy of the GNU General Public License along 

16# with this program; if not, write to the Free Software Foundation, Inc., 

17# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 

18 

19# This is still work-in-progress and far too incomplete. 

20# ruff: noqa 

21# type: ignore 

22 

23import sys 

24from collections import defaultdict 

25 

26import apt_pkg 

27 

28import daklib.archive 

29import daklib.config 

30import daklib.dbconn 

31import daklib.import_repository 

32import daklib.utils 

33 

34 

35def usage(status=0): 

36 print( 

37 """ 

38dak import-repository 

39 --keyring=/usr/share/keyrings/debian-archive-keyring.gpg 

40 [--key=${fingerprint}] 

41 [--architectures=a,b,c (default: architectures in origin suite)] 

42 [--components=main,contrib (default: components in origin suite)] 

43 [--target-suite=${suite} (default: origin suite name)] 

44 [--add-overrides] 

45 [--max-packages=${n} (import at maximum ${n} packages, default: no limit)] 

46 http://httpredir.debian.org/debian unstable 

47 

48Things to think about: 

49 - Import Built-Using sources 

50 - all / only referenced 

51 - Remove old packages: 

52 - by-source: remove source X_v, if no X exists upstream 

53 - by-version: remove source X_v, if no X_v exists upstream 

54 (X denotes package name, v version, X_v package at a specific version) 

55 - Import all or only newest? 

56 - Expire binary packages? 

57""" 

58 ) 

59 sys.exit(status) 

60 

61 

62def entry_is_newer(entry, packages): 

63 version = entry["Version"] 

64 for p in packages[entry["Package"]]: 

65 if apt_pkg.version_compare(version, p.version) <= 0: 

66 return False 

67 return True 

68 

69 

70def entry_in_packages(entry, packages): 

71 return entry["Package"] in packages 

72 

73 

74def get_packages_in_suite(suite): 

75 sources = defaultdict(list) 

76 for s in suite.sources: 

77 sources[s.source].append(s) 

78 

79 packages = defaultdict(list) 

80 for b in suite.binaries: 

81 packages[b.package].append(b) 

82 

83 return sources, packages 

84 

85 

86def import_sources( 

87 base, 

88 sources, 

89 transaction, 

90 target_suite, 

91 component, 

92 target_sources, 

93 extra_sources, 

94 extra_sources_comp, 

95 max_packages=None, 

96): 

97 n = 0 

98 for entry in sources: 

99 if max_packages is not None and n > max_packages: 

100 break 

101 if entry.get("Extra-Source-Only", "no") == "yes": 

102 # Remember package, we might need to import it later. 

103 key = (entry["Package"], entry["Version"]) 

104 extra_sources[key] = entry 

105 extra_sources_comp[key].add(c) 

106 continue 

107 if not entry_in_packages(entry, target_sources) or entry_is_newer( 

108 entry, target_sources 

109 ): 

110 print("Importing {0}={1}".format(entry["Package"], entry["Version"])) 

111 daklib.import_repository.import_source_to_suite( 

112 base, entry, transaction, target_suite, component 

113 ) 

114 n += 1 

115 return n 

116 

117 

118def import_built_using( 

119 base, 

120 source, 

121 version, 

122 transaction, 

123 target_suite, 

124 component, 

125 extra_sources, 

126 extra_sources_comp, 

127): 

128 if not daklib.import_repository.source_in_archive( 

129 bu_source, bu_version, target_suite.archive 

130 ): 

131 print("Importing extra source {0}={1}".format(bu_source, bu_version)) 

132 key = (bu_source, bu_version) 

133 extra_entry = extra_sources.get(key) 

134 if extra_entry is None: 

135 raise Exception( 

136 "Extra source {0}={1} referenced by {2}={3} ({4}) not found in source suite.".format( 

137 bu_source, 

138 bu_version, 

139 entry["Package"], 

140 entry["Version"], 

141 architecture, 

142 ) 

143 ) 

144 # extra_components = extra_sources_comp[key] 

145 if c in components: 

146 extra_component = component 

147 else: 

148 # TODO: Take preferred components from those listed... 

149 raise Exception("Not implemented.") 

150 daklib.import_repository.import_source_to_suite( 

151 base, extra_entry, transaction, target_suite, extra_component 

152 ) 

153 

154 

155def import_packages( 

156 base, 

157 packages, 

158 transaction, 

159 target_suite, 

160 component, 

161 architecture, 

162 target_binaries, 

163 extra_sources, 

164 extra_sources_comp, 

165 max_packages=None, 

166): 

167 n = 0 

168 for entry in packages: 

169 if max_packages is not None and n > max_packages: 

170 break 

171 if not entry_in_packages(entry, target_binaries) or entry_is_newer( 

172 entry, target_binaries 

173 ): 

174 print( 

175 "Importing {0}={1} ({2})".format( 

176 entry["Package"], entry["Version"], architecture 

177 ) 

178 ) 

179 # Import Built-Using sources: 

180 for bu_source, bu_version in daklib.utils.parse_built_using(entry): 

181 import_built_using( 

182 base, 

183 bu_source, 

184 bu_version, 

185 transaction, 

186 target_suite, 

187 component, 

188 extra_sources, 

189 extra_sources_comp, 

190 ) 

191 # Import binary: 

192 daklib.import_repository.import_package_to_suite( 

193 base, entry, transaction, target_suite, component 

194 ) 

195 n += 1 

196 return n 

197 

198 

199def main(argv=None): 

200 if argv is None: 200 ↛ 203line 200 didn't jump to line 203

201 argv = sys.argv 

202 

203 arguments = [ 

204 ("h", "help", "Import-Repository::Help"), 

205 ("k", "keyring", "Import-Repository::Keyring", "HasArg"), 

206 ("K", "key", "Import-Repository::Key", "HasArg"), 

207 ("a", "architectures", "Import-Repository::Architectures", "HasArg"), 

208 ("c", "components", "Import-Repository::Components", "HasArg"), 

209 ("t", "target-suite", "Import-Repository::Target-Suite", "HasArg"), 

210 ("A", "add-overrides", "Import-Repository::AddOverrides"), 

211 ("n", "max-packages", "Import-Repository::MaxPackages", "HasArg"), 

212 ] 

213 

214 cnf = daklib.config.Config() 

215 argv = apt_pkg.parse_commandline(cnf.Cnf, arguments, argv) # type: ignore[attr-defined] 

216 options = cnf.subtree("Import-Repository") 

217 

218 if "Help" in options or len(argv) < 2: 218 ↛ 221line 218 didn't jump to line 221 because the condition on line 218 was always true

219 usage(0) 

220 

221 keyring = options.find("Keyring") or None 

222 if keyring is None: 

223 print("Error: No keyring specified") 

224 print() 

225 

226 if "Key" in options: 

227 raise Exception("Not implemented.") 

228 

229 if "AddOverrides" in options: 

230 raise Exception("Not implemented.") 

231 

232 if "MaxPackages" in options: 

233 max_packages = int(options["MaxPackages"]) 

234 else: 

235 max_packages = None 

236 

237 base, suite = argv[0:2] 

238 

239 target_suite_name = options.find("Target-Suite") or suite 

240 

241 print( 

242 "Importing packages from {0}/dists/{1} to {2}".format( 

243 base, suite, target_suite_name 

244 ) 

245 ) 

246 with daklib.archive.ArchiveTransaction() as transaction: 

247 target_suite = daklib.dbconn.get_suite(target_suite_name, transaction.session) 

248 if target_suite is None: 

249 daklib.utils.fubar( 

250 "Target suite '{0}' is unknown.".format(target_suite_name) 

251 ) 

252 

253 release = daklib.import_repository.obtain_release(base, suite, keyring) 

254 target_sources, target_binaries = get_packages_in_suite(target_suite) 

255 

256 if "Architectures" in options: 

257 architectures = options["Architectures"].split(",") 

258 else: 

259 architectures = ["all"] + release.architectures() 

260 

261 if "Components" in options: 

262 components = options["Components"].split(",") 

263 else: 

264 components = release.components() 

265 

266 # TODO: Clean this up... 

267 

268 n = 0 

269 

270 # For Extra-Source-Only sources packages, keep a dict 

271 # (name, version) -> entry and (name, version) -> set of components 

272 # to allow importing needed packages at a later stage 

273 extra_sources = dict() 

274 extra_sources_comp = defaultdict(set) 

275 

276 for c in components: 

277 component = daklib.dbconn.get_component(c, transaction.session) 

278 print("Processing {0}/source...".format(c)) 

279 sources = release.sources(c) 

280 imported = import_sources( 

281 base, 

282 sources, 

283 transaction, 

284 target_suite, 

285 component, 

286 target_sources, 

287 extra_sources, 

288 extra_sources_comp, 

289 max_packages, 

290 ) 

291 print(" imported {0} source packages".format(imported)) 

292 n += imported 

293 if max_packages is not None: 

294 max_packages -= n 

295 

296 for c in components: 

297 component = daklib.dbconn.get_component(c, transaction.session) 

298 for architecture in architectures: 

299 print("Processing {0}/{1}...".format(c, architecture)) 

300 packages = release.packages(c, architecture) 

301 imported = import_packages( 

302 base, 

303 packages, 

304 transaction, 

305 target_suite, 

306 component, 

307 architecture, 

308 target_binaries, 

309 extra_sources, 

310 extra_sources_comp, 

311 max_packages, 

312 ) 

313 print(" imported {0} binary packages".format(imported)) 

314 n += imported 

315 if max_packages is not None: 

316 max_packages -= n 

317 

318 transaction.rollback() 

319 

320 

321if __name__ == "__main__": 

322 main()