Coverage for dak/import_repository.py: 15%

1# Copyright (C) 2015, Ansgar Burchardt <ansgar@debian.org> 

2# 

3# This program is free software; you can redistribute it and/or modify 

4# it under the terms of the GNU General Public License as published by 

5# the Free Software Foundation; either version 2 of the License, or 

6# (at your option) any later version. 

7# 

8# This program is distributed in the hope that it will be useful, 

9# but WITHOUT ANY WARRANTY; without even the implied warranty of 

10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

11# GNU General Public License for more details. 

12# 

13# You should have received a copy of the GNU General Public License along 

14# with this program; if not, write to the Free Software Foundation, Inc., 

15# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 

16 

17# This is still work-in-progress and far too incomplete. 

18# ruff: noqa 

19# type: ignore 

20 

21import sys 

22from collections import defaultdict 

23 

24import apt_pkg 

25 

26import daklib.archive 

27import daklib.config 

28import daklib.dbconn 

29import daklib.import_repository 

30import daklib.utils 

31 

32 

33def usage(status=0): 

34 print( 

35 """ 

36dak import-repository 

37 --keyring=/usr/share/keyrings/debian-archive-keyring.gpg 

38 [--key=${fingerprint}] 

39 [--architectures=a,b,c (default: architectures in origin suite)] 

40 [--components=main,contrib (default: components in origin suite)] 

41 [--target-suite=${suite} (default: origin suite name)] 

42 [--add-overrides] 

43 [--max-packages=${n} (import at maximum ${n} packages, default: no limit)] 

44 http://httpredir.debian.org/debian unstable 

45 

46Things to think about: 

47 - Import Built-Using sources 

48 - all / only referenced 

49 - Remove old packages: 

50 - by-source: remove source X_v, if no X exists upstream 

51 - by-version: remove source X_v, if no X_v exists upstream 

52 (X denotes package name, v version, X_v package at a specific version) 

53 - Import all or only newest? 

54 - Expire binary packages? 

55""" 

56 ) 

57 sys.exit(status) 

58 

59 

60def entry_is_newer(entry, packages): 

61 version = entry["Version"] 

62 for p in packages[entry["Package"]]: 

63 if apt_pkg.version_compare(version, p.version) <= 0: 

64 return False 

65 return True 

66 

67 

68def entry_in_packages(entry, packages): 

69 return entry["Package"] in packages 

70 

71 

72def get_packages_in_suite(suite): 

73 sources = defaultdict(list) 

74 for s in suite.sources: 

75 sources[s.source].append(s) 

76 

77 packages = defaultdict(list) 

78 for b in suite.binaries: 

79 packages[b.package].append(b) 

80 

81 return sources, packages 

82 

83 

84def import_sources( 

85 base, 

86 sources, 

87 transaction, 

88 target_suite, 

89 component, 

90 target_sources, 

91 extra_sources, 

92 extra_sources_comp, 

93 max_packages=None, 

94): 

95 n = 0 

96 for entry in sources: 

97 if max_packages is not None and n > max_packages: 

98 break 

99 if entry.get("Extra-Source-Only", "no") == "yes": 

100 # Remember package, we might need to import it later. 

101 key = (entry["Package"], entry["Version"]) 

102 extra_sources[key] = entry 

103 extra_sources_comp[key].add(c) 

104 continue 

105 if not entry_in_packages(entry, target_sources) or entry_is_newer( 

106 entry, target_sources 

107 ): 

108 print("Importing {0}={1}".format(entry["Package"], entry["Version"])) 

109 daklib.import_repository.import_source_to_suite( 

110 base, entry, transaction, target_suite, component 

111 ) 

112 n += 1 

113 return n 

114 

115 

116def import_built_using( 

117 base, 

118 source, 

119 version, 

120 transaction, 

121 target_suite, 

122 component, 

123 extra_sources, 

124 extra_sources_comp, 

125): 

126 if not daklib.import_repository.source_in_archive( 

127 bu_source, bu_version, target_suite.archive 

128 ): 

129 print("Importing extra source {0}={1}".format(bu_source, bu_version)) 

130 key = (bu_source, bu_version) 

131 extra_entry = extra_sources.get(key) 

132 if extra_entry is None: 

133 raise Exception( 

134 "Extra source {0}={1} referenced by {2}={3} ({4}) not found in source suite.".format( 

135 bu_source, 

136 bu_version, 

137 entry["Package"], 

138 entry["Version"], 

139 architecture, 

140 ) 

141 ) 

142 # extra_components = extra_sources_comp[key] 

143 if c in components: 

144 extra_component = component 

145 else: 

146 # TODO: Take preferred components from those listed... 

147 raise Exception("Not implemented.") 

148 daklib.import_repository.import_source_to_suite( 

149 base, extra_entry, transaction, target_suite, extra_component 

150 ) 

151 

152 

153def import_packages( 

154 base, 

155 packages, 

156 transaction, 

157 target_suite, 

158 component, 

159 architecture, 

160 target_binaries, 

161 extra_sources, 

162 extra_sources_comp, 

163 max_packages=None, 

164): 

165 n = 0 

166 for entry in packages: 

167 if max_packages is not None and n > max_packages: 

168 break 

169 if not entry_in_packages(entry, target_binaries) or entry_is_newer( 

170 entry, target_binaries 

171 ): 

172 print( 

173 "Importing {0}={1} ({2})".format( 

174 entry["Package"], entry["Version"], architecture 

175 ) 

176 ) 

177 # Import Built-Using sources: 

178 for bu_source, bu_version in daklib.utils.parse_built_using(entry): 

179 import_built_using( 

180 base, 

181 bu_source, 

182 bu_version, 

183 transaction, 

184 target_suite, 

185 component, 

186 extra_sources, 

187 extra_sources_comp, 

188 ) 

189 # Import binary: 

190 daklib.import_repository.import_package_to_suite( 

191 base, entry, transaction, target_suite, component 

192 ) 

193 n += 1 

194 return n 

195 

196 

197def main(argv=None): 

198 if argv is None: 198 ↛ 201line 198 didn't jump to line 201

199 argv = sys.argv 

200 

201 arguments = [ 

202 ("h", "help", "Import-Repository::Help"), 

203 ("k", "keyring", "Import-Repository::Keyring", "HasArg"), 

204 ("K", "key", "Import-Repository::Key", "HasArg"), 

205 ("a", "architectures", "Import-Repository::Architectures", "HasArg"), 

206 ("c", "components", "Import-Repository::Components", "HasArg"), 

207 ("t", "target-suite", "Import-Repository::Target-Suite", "HasArg"), 

208 ("A", "add-overrides", "Import-Repository::AddOverrides"), 

209 ("n", "max-packages", "Import-Repository::MaxPackages", "HasArg"), 

210 ] 

211 

212 cnf = daklib.config.Config() 

213 argv = apt_pkg.parse_commandline(cnf.Cnf, arguments, argv) # type: ignore[attr-defined] 

214 options = cnf.subtree("Import-Repository") 

215 

216 if "Help" in options or len(argv) < 2: 216 ↛ 219line 216 didn't jump to line 219 because the condition on line 216 was always true

217 usage(0) 

218 

219 keyring = options.find("Keyring") or None 

220 if keyring is None: 

221 print("Error: No keyring specified") 

222 print() 

223 

224 if "Key" in options: 

225 raise Exception("Not implemented.") 

226 

227 if "AddOverrides" in options: 

228 raise Exception("Not implemented.") 

229 

230 if "MaxPackages" in options: 

231 max_packages = int(options["MaxPackages"]) 

232 else: 

233 max_packages = None 

234 

235 base, suite = argv[0:2] 

236 

237 target_suite_name = options.find("Target-Suite") or suite 

238 

239 print( 

240 "Importing packages from {0}/dists/{1} to {2}".format( 

241 base, suite, target_suite_name 

242 ) 

243 ) 

244 with daklib.archive.ArchiveTransaction() as transaction: 

245 target_suite = daklib.dbconn.get_suite(target_suite_name, transaction.session) 

246 if target_suite is None: 

247 daklib.utils.fubar( 

248 "Target suite '{0}' is unknown.".format(target_suite_name) 

249 ) 

250 

251 release = daklib.import_repository.obtain_release(base, suite, keyring) 

252 target_sources, target_binaries = get_packages_in_suite(target_suite) 

253 

254 if "Architectures" in options: 

255 architectures = options["Architectures"].split(",") 

256 else: 

257 architectures = ["all"] + release.architectures() 

258 

259 if "Components" in options: 

260 components = options["Components"].split(",") 

261 else: 

262 components = release.components() 

263 

264 # TODO: Clean this up... 

265 

266 n = 0 

267 

268 # For Extra-Source-Only sources packages, keep a dict 

269 # (name, version) -> entry and (name, version) -> set of components 

270 # to allow importing needed packages at a later stage 

271 extra_sources = dict() 

272 extra_sources_comp = defaultdict(set) 

273 

274 for c in components: 

275 component = daklib.dbconn.get_component(c, transaction.session) 

276 print("Processing {0}/source...".format(c)) 

277 sources = release.sources(c) 

278 imported = import_sources( 

279 base, 

280 sources, 

281 transaction, 

282 target_suite, 

283 component, 

284 target_sources, 

285 extra_sources, 

286 extra_sources_comp, 

287 max_packages, 

288 ) 

289 print(" imported {0} source packages".format(imported)) 

290 n += imported 

291 if max_packages is not None: 

292 max_packages -= n 

293 

294 for c in components: 

295 component = daklib.dbconn.get_component(c, transaction.session) 

296 for architecture in architectures: 

297 print("Processing {0}/{1}...".format(c, architecture)) 

298 packages = release.packages(c, architecture) 

299 imported = import_packages( 

300 base, 

301 packages, 

302 transaction, 

303 target_suite, 

304 component, 

305 architecture, 

306 target_binaries, 

307 extra_sources, 

308 extra_sources_comp, 

309 max_packages, 

310 ) 

311 print(" imported {0} binary packages".format(imported)) 

312 n += imported 

313 if max_packages is not None: 

314 max_packages -= n 

315 

316 transaction.rollback()