Coverage for dak/import.py: 58%
116 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-08-03 16:46 +0000
1# Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU General Public License as published by
5# the Free Software Foundation; either version 2 of the License, or
6# (at your option) any later version.
7#
8# This program is distributed in the hope that it will be useful,
9# but WITHOUT ANY WARRANTY; without even the implied warranty of
10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11# GNU General Public License for more details.
12#
13# You should have received a copy of the GNU General Public License along
14# with this program; if not, write to the Free Software Foundation, Inc.,
15# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17import os
18import sys
20import apt_pkg
21from sqlalchemy import select
23import daklib.archive
24import daklib.config
25import daklib.daklog
26import daklib.regexes
27import daklib.upload
28from daklib.dbconn import (
29 Fingerprint,
30 Override,
31 OverrideType,
32 Priority,
33 Section,
34 get_active_keyring_files,
35 get_component_by_name,
36 get_or_set_maintainer,
37 get_suite_by_name,
38)
41def usage():
42 print(
43 """Usage:
45dak import <suite> <component> <files...>
46dak import -D|--dump <file> <suite> <component>
47dak import -E|--export-dump <suite> <component>
49WARNING: This command does no sanity checks. Only use it on trusted packages.
51Options:
52 -h, --help: show this help message
53 -a, --add-overrides: add missing overrides automatically
54 -c, --changed-by: Changed-By for imported source packages
55 (default: maintainer)
56 -D, --dump <file>: Import all files listed in <file>. The format
57 is described below.
58 -E, --export-dump: Export list of files in the format required
59 by dak import --dump.
60 -s, --ignore-signature: ignore signature for imported source packages
62File format used by --dump:
64 <filename>:<md5>:<sha1>:<sha256>:[<fingerprint>]:[<changed-by>]
65"""
66 )
69def import_source(
70 log,
71 transaction,
72 suite,
73 component,
74 directory,
75 hashed_file,
76 fingerprint=None,
77 changed_by=None,
78 keyrings=None,
79 require_signature=True,
80 add_overrides=False,
81):
82 if keyrings is None: 82 ↛ 83line 82 didn't jump to line 83 because the condition on line 82 was never true
83 keyrings = []
84 filename = hashed_file.filename
85 session = transaction.session
87 source = daklib.upload.Source(directory, [hashed_file], keyrings, require_signature)
88 if source.valid_signature: 88 ↛ 94line 88 didn't jump to line 94 because the condition on line 88 was always true
89 fingerprint = session.scalars(
90 select(Fingerprint)
91 .filter_by(fingerprint=source.primary_fingerprint)
92 .limit(1)
93 ).first()
94 if changed_by is None: 94 ↛ 96line 94 didn't jump to line 96 because the condition on line 94 was always true
95 changed_by = source.dsc["Maintainer"]
96 db_changed_by = get_or_set_maintainer(changed_by, session)
98 transaction.install_source(
99 directory, source, suite, component, db_changed_by, fingerprint=fingerprint
100 )
101 log.log(["import-source", suite.suite_name, component.component_name, filename])
103 if ( 103 ↛ 117line 103 didn't jump to line 117
104 add_overrides
105 and not session.scalars(
106 select(Override)
107 .filter_by(
108 suite=suite.get_overridesuite(),
109 component=component,
110 package=source.dsc["Source"],
111 )
112 .join(OverrideType)
113 .where(OverrideType.overridetype == "dsc")
114 .limit(1)
115 ).first()
116 ):
117 overridetype = session.execute(
118 select(OverrideType).filter_by(overridetype="dsc")
119 ).scalar_one()
120 overridesuite = suite.get_overridesuite()
121 section_name = "misc"
122 if component.component_name != "main":
123 section_name = "{0}/{1}".format(component.component_name, section_name)
124 section = session.execute(
125 select(Section).filter_by(section=section_name)
126 ).scalar_one()
127 priority = session.execute(
128 select(Priority).filter_by(priority="optional")
129 ).scalar_one()
131 override = Override(
132 package=source.dsc["Source"],
133 suite=overridesuite,
134 component=component,
135 section=section,
136 priority=priority,
137 overridetype=overridetype,
138 )
139 session.add(override)
140 log.log(
141 [
142 "add-source-override",
143 suite.suite_name,
144 component.component_name,
145 source.dsc["Source"],
146 section.section,
147 priority.priority,
148 ]
149 )
152def import_binary(
153 log,
154 transaction,
155 suite,
156 component,
157 directory,
158 hashed_file,
159 fingerprint=None,
160 add_overrides=False,
161):
162 filename = hashed_file.filename
163 session = transaction.session
165 binary = daklib.upload.Binary(directory, hashed_file)
166 transaction.install_binary(
167 directory, binary, suite, component, fingerprint=fingerprint
168 )
169 log.log(["import-binary", suite.suite_name, component.component_name, filename])
171 if ( 171 ↛ 185line 171 didn't jump to line 185
172 add_overrides
173 and not session.scalars(
174 select(Override)
175 .filter_by(
176 suite=suite.get_overridesuite(),
177 component=component,
178 package=binary.control["Package"],
179 )
180 .join(OverrideType)
181 .where(OverrideType.overridetype == binary.type)
182 .limit(1)
183 ).first()
184 ):
185 overridetype = session.execute(
186 select(OverrideType).filter_by(overridetype=binary.type)
187 ).scalar_one()
188 overridesuite = suite.get_overridesuite()
189 section = session.execute(
190 select(Section).filter_by(section=binary.control["Section"])
191 ).scalar_one()
192 priority = session.execute(
193 select(Priority).filter_by(
194 priority=binary.control.get("Priority", "optional")
195 )
196 ).scalar_one()
198 override = Override(
199 package=binary.control["Package"],
200 suite=overridesuite,
201 component=component,
202 section=section,
203 priority=priority,
204 overridetype=overridetype,
205 )
206 session.add(override)
207 log.log(
208 [
209 "add-binary-override",
210 suite.suite_name,
211 component.component_name,
212 binary.control["Package"],
213 section.section,
214 priority.priority,
215 ]
216 )
219def import_file(
220 log,
221 transaction,
222 suite,
223 component,
224 directory,
225 hashed_file,
226 fingerprint=None,
227 changed_by=None,
228 keyrings=None,
229 require_signature=True,
230 add_overrides=False,
231):
232 filename = hashed_file.filename
233 if daklib.regexes.re_file_binary.match(filename):
234 import_binary(
235 log,
236 transaction,
237 suite,
238 component,
239 directory,
240 hashed_file,
241 fingerprint=fingerprint,
242 add_overrides=add_overrides,
243 )
244 elif daklib.regexes.re_file_dsc.match(filename): 244 ↛ 259line 244 didn't jump to line 259 because the condition on line 244 was always true
245 import_source(
246 log,
247 transaction,
248 suite,
249 component,
250 directory,
251 hashed_file,
252 fingerprint=fingerprint,
253 changed_by=changed_by,
254 keyrings=keyrings,
255 require_signature=require_signature,
256 add_overrides=add_overrides,
257 )
258 else:
259 raise Exception(
260 "File is neither source nor binary package: {0}".format(filename)
261 )
264def import_dump(
265 log,
266 transaction,
267 suite,
268 component,
269 fh,
270 keyrings=None,
271 require_signature=True,
272 add_overrides=False,
273):
274 session = transaction.session
275 for line in fh:
276 path, size, md5, sha1, sha256, fpr, changed_by = line.strip().split(":", 6)
277 directory, filename = os.path.split(os.path.abspath(path))
279 if not changed_by:
280 changed_by = None
281 fingerprint = None
282 if fpr:
283 fingerprint = session.scalars(
284 select(Fingerprint).filter_by(fingerprint=fpr).limit(1)
285 ).first()
286 if fingerprint is None:
287 print("W: {0}: unknown fingerprint {1}".format(filename, fpr))
289 hashed_file = daklib.upload.HashedFile(filename, int(size), md5, sha1, sha256)
290 hashed_file.check(directory)
292 import_file(
293 log,
294 transaction,
295 suite,
296 component,
297 directory,
298 hashed_file,
299 fingerprint=fingerprint,
300 changed_by=changed_by,
301 keyrings=keyrings,
302 require_signature=require_signature,
303 add_overrides=add_overrides,
304 )
306 transaction.commit()
309_export_query = r"""
310WITH
311tmp AS
312 (SELECT 1 AS order, s.file AS file_id, s.sig_fpr AS fingerprint_id, s.changedby AS changed_by, sa.suite AS suite_id
313 FROM source s
314 JOIN src_associations sa ON sa.source = s.id
315 UNION
316 SELECT 2 AS order, b.file AS file_id, b.sig_fpr AS fingerprint_id, NULL, ba.suite AS suite_id
317 FROM binaries b
318 JOIN bin_associations ba ON ba.bin = b.id
319 )
321SELECT
322 f.filename, f.size::TEXT, f.md5sum, f.sha1sum, f.sha256sum, COALESCE(fpr.fingerprint, ''), COALESCE(m.name, '')
323FROM files f
324JOIN tmp ON f.id = tmp.file_id
325JOIN suite ON suite.id = tmp.suite_id
326JOIN files_archive_map fam ON fam.file_id = f.id AND fam.archive_id = suite.archive_id
327LEFT JOIN fingerprint fpr ON fpr.id = tmp.fingerprint_id
328LEFT JOIN maintainer m ON m.id = tmp.changed_by
330WHERE
331 suite.id = :suite_id
332 AND fam.component_id = :component_id
334ORDER BY tmp.order, f.filename;
335"""
338def export_dump(transaction, suite, component):
339 session = transaction.session
340 query = session.execute(
341 _export_query,
342 {"suite_id": suite.suite_id, "component_id": component.component_id},
343 )
344 for row in query:
345 print(":".join(row))
348def main(argv=None):
349 if argv is None: 349 ↛ 352line 349 didn't jump to line 352
350 argv = sys.argv
352 arguments = [
353 ("h", "help", "Import::Options::Help"),
354 ("a", "add-overrides", "Import::Options::AddOverrides"),
355 ("c", "changed-by", "Import::Options::ChangedBy", "HasArg"),
356 ("D", "dump", "Import::Options::Dump", "HasArg"),
357 ("E", "export-dump", "Import::Options::Export"),
358 ("s", "ignore-signature", "Import::Options::IgnoreSignature"),
359 ]
361 cnf = daklib.config.Config()
362 cnf["Import::Options::Dummy"] = ""
363 argv = apt_pkg.parse_commandline(cnf.Cnf, arguments, argv) # type: ignore[attr-defined]
364 options = cnf.subtree("Import::Options")
366 if "Help" in options or len(argv) < 2:
367 usage()
368 sys.exit(0)
370 suite_name = argv[0]
371 component_name = argv[1]
373 add_overrides = options.find_b("AddOverrides")
374 require_signature = not options.find_b("IgnoreSignature")
375 changed_by = options.find("ChangedBy") or None
377 log = daklib.daklog.Logger("import")
379 with daklib.archive.ArchiveTransaction() as transaction:
380 session = transaction.session
381 suite = get_suite_by_name(suite_name, session)
382 component = get_component_by_name(component_name, session)
383 keyring_files = get_active_keyring_files(session)
385 dump = options.find("Dump") or None
386 if options.find_b("Export"): 386 ↛ 387line 386 didn't jump to line 387 because the condition on line 386 was never true
387 export_dump(transaction, suite, component)
388 transaction.rollback()
389 elif dump is not None: 389 ↛ 390line 389 didn't jump to line 390 because the condition on line 389 was never true
390 with open(dump, "r") as fh:
391 import_dump(
392 log,
393 transaction,
394 suite,
395 component,
396 fh,
397 keyring_files,
398 require_signature=require_signature,
399 add_overrides=add_overrides,
400 )
401 transaction.commit()
402 else:
403 files = argv[2:]
404 for f in files:
405 directory, filename = os.path.split(os.path.abspath(f))
406 hashed_file = daklib.upload.HashedFile.from_file(directory, filename)
407 import_file(
408 log,
409 transaction,
410 suite,
411 component,
412 directory,
413 hashed_file,
414 changed_by=changed_by,
415 keyrings=keyring_files,
416 require_signature=require_signature,
417 add_overrides=add_overrides,
418 )
419 transaction.commit()
421 log.close()