Coverage for daklib/archive.py: 75%
831 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-05-10 21:38 +0000
1# Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU General Public License as published by
5# the Free Software Foundation; either version 2 of the License, or
6# (at your option) any later version.
7#
8# This program is distributed in the hope that it will be useful,
9# but WITHOUT ANY WARRANTY; without even the implied warranty of
10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11# GNU General Public License for more details.
12#
13# You should have received a copy of the GNU General Public License along
14# with this program; if not, write to the Free Software Foundation, Inc.,
15# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17"""module to manipulate the archive
19This module provides classes to manipulate the archive.
20"""
22import os
23import shutil
24import subprocess
25import traceback
26from collections.abc import Callable, Collection, Iterable
27from typing import TYPE_CHECKING, Optional, Union
29import sqlalchemy.exc
30from sqlalchemy import sql
31from sqlalchemy.orm import object_session
32from sqlalchemy.orm.exc import NoResultFound
34import daklib.checks as checks
35import daklib.upload
36import daklib.utils
37from daklib import sandbox
38from daklib.config import Config
39from daklib.dbconn import (
40 Archive,
41 ArchiveFile,
42 Component,
43 DBBinary,
44 DBChange,
45 DBConn,
46 DBSource,
47 DSCFile,
48 Fingerprint,
49 Maintainer,
50 Override,
51 OverrideType,
52 PolicyQueue,
53 PolicyQueueByhandFile,
54 PolicyQueueUpload,
55 PoolFile,
56 Suite,
57 VersionCheck,
58 get_architecture,
59 get_mapped_component,
60 get_or_set_maintainer,
61 import_metadata_into_db,
62)
63from daklib.externalsignature import check_upload_for_external_signature_request
64from daklib.fstransactions import FilesystemTransaction
65from daklib.regexes import re_bin_only_nmu, re_changelog_versions
66from daklib.tag2upload import get_tag2upload_info_for_upload, parse_git_tag_info
68if TYPE_CHECKING:
69 import daklib.packagelist
72class ArchiveException(Exception):
73 pass
76class HashMismatchException(ArchiveException):
77 pass
80class ArchiveTransaction:
81 """manipulate the archive in a transaction"""
83 def __init__(self):
84 self.fs = FilesystemTransaction()
85 self.session = DBConn().session()
87 def get_file(
88 self,
89 hashed_file: daklib.upload.HashedFile,
90 source_name: str,
91 check_hashes: bool = True,
92 ) -> PoolFile:
93 """Look for file `hashed_file` in database
95 :param hashed_file: file to look for in the database
96 :param source_name: source package name
97 :param check_hashes: check size and hashes match
98 :return: database entry for the file
99 :raises KeyError: file was not found in the database
100 :raises HashMismatchException: hash mismatch
101 """
102 poolname = os.path.join(daklib.utils.poolify(source_name), hashed_file.filename)
103 try:
104 poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
105 if check_hashes and ( 105 ↛ 111line 105 didn't jump to line 111 because the condition on line 105 was never true
106 poolfile.filesize != hashed_file.size
107 or poolfile.md5sum != hashed_file.md5sum
108 or poolfile.sha1sum != hashed_file.sha1sum
109 or poolfile.sha256sum != hashed_file.sha256sum
110 ):
111 raise HashMismatchException(
112 "{0}: Does not match file already existing in the pool.".format(
113 hashed_file.filename
114 )
115 )
116 return poolfile
117 except NoResultFound:
118 raise KeyError("{0} not found in database.".format(poolname))
120 def _install_file(
121 self, directory, hashed_file, archive, component, source_name
122 ) -> PoolFile:
123 """Install a file
125 Will not give an error when the file is already present.
127 :return: database object for the new file
128 """
129 session = self.session
131 poolname = os.path.join(daklib.utils.poolify(source_name), hashed_file.filename)
132 try:
133 poolfile = self.get_file(hashed_file, source_name)
134 except KeyError:
135 poolfile = PoolFile(filename=poolname, filesize=hashed_file.size)
136 poolfile.md5sum = hashed_file.md5sum
137 poolfile.sha1sum = hashed_file.sha1sum
138 poolfile.sha256sum = hashed_file.sha256sum
139 session.add(poolfile)
140 session.flush()
142 try:
143 session.query(ArchiveFile).filter_by(
144 archive=archive, component=component, file=poolfile
145 ).one()
146 except NoResultFound:
147 archive_file = ArchiveFile(archive, component, poolfile)
148 session.add(archive_file)
149 session.flush()
151 path = os.path.join(
152 archive.path, "pool", component.component_name, poolname
153 )
154 hashed_file_path = os.path.join(directory, hashed_file.input_filename)
155 self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
157 return poolfile
159 def install_binary(
160 self,
161 directory: str,
162 binary: daklib.upload.Binary,
163 suite: Suite,
164 component: Component,
165 *,
166 allow_tainted: bool = False,
167 fingerprint: Optional[Fingerprint] = None,
168 authorized_by_fingerprint: Optional[Fingerprint] = None,
169 source_suites=None,
170 extra_source_archives: Optional[Iterable[Archive]] = None,
171 ) -> DBBinary:
172 """Install a binary package
174 :param directory: directory the binary package is located in
175 :param binary: binary package to install
176 :param suite: target suite
177 :param component: target component
178 :param allow_tainted: allow to copy additional files from tainted archives
179 :param fingerprint: optional fingerprint
180 :param source_suites: suites to copy the source from if they are not
181 in `suite` or :const:`True` to allow copying from any
182 suite.
183 Can be a SQLAlchemy subquery for :class:`Suite` or :const:`True`.
184 :param extra_source_archives: extra archives to copy Built-Using sources from
185 :return: database object for the new package
186 """
187 session = self.session
188 control = binary.control
189 maintainer = get_or_set_maintainer(control["Maintainer"], session)
190 architecture = get_architecture(control["Architecture"], session)
192 (source_name, source_version) = binary.source
193 source_query = session.query(DBSource).filter_by(
194 source=source_name, version=source_version
195 )
196 source = source_query.filter(DBSource.suites.contains(suite)).first()
197 if source is None:
198 if source_suites is not True:
199 source_query = source_query.join(DBSource.suites).filter(
200 Suite.suite_id == source_suites.c.id
201 )
202 source = source_query.first()
203 if source is None: 203 ↛ 204line 203 didn't jump to line 204 because the condition on line 203 was never true
204 raise ArchiveException(
205 "{0}: trying to install to {1}, but could not find source ({2} {3})".format(
206 binary.hashed_file.filename,
207 suite.suite_name,
208 source_name,
209 source_version,
210 )
211 )
212 self.copy_source(source, suite, source.poolfile.component)
214 db_file = self._install_file(
215 directory, binary.hashed_file, suite.archive, component, source_name
216 )
218 unique = dict(
219 package=control["Package"],
220 version=control["Version"],
221 architecture=architecture,
222 )
223 rest = dict(
224 source=source,
225 maintainer=maintainer,
226 poolfile=db_file,
227 binarytype=binary.type,
228 )
229 # Other attributes that are ignored for purposes of equality with
230 # an existing source
231 rest2 = dict(
232 fingerprint=fingerprint,
233 authorized_by_fingerprint=authorized_by_fingerprint,
234 )
236 try:
237 db_binary = session.query(DBBinary).filter_by(**unique).one()
238 for key, value in rest.items():
239 if getattr(db_binary, key) != value: 239 ↛ 240line 239 didn't jump to line 240 because the condition on line 239 was never true
240 raise ArchiveException(
241 "{0}: Does not match binary in database.".format(
242 binary.hashed_file.filename
243 )
244 )
245 except NoResultFound:
246 db_binary = DBBinary(**unique)
247 for key, value in rest.items():
248 setattr(db_binary, key, value)
249 for key, value in rest2.items():
250 setattr(db_binary, key, value)
251 session.add(db_binary)
252 session.flush()
253 import_metadata_into_db(db_binary, session)
255 self._add_built_using(
256 db_binary,
257 binary.hashed_file.filename,
258 control,
259 suite,
260 extra_archives=extra_source_archives,
261 )
263 if suite not in db_binary.suites:
264 db_binary.suites.append(suite)
266 session.flush()
268 return db_binary
270 def _ensure_extra_source_exists(
271 self,
272 filename: str,
273 source: DBSource,
274 archive: Archive,
275 extra_archives: Optional[Iterable[Archive]] = None,
276 ):
277 """ensure source exists in the given archive
279 This is intended to be used to check that Built-Using sources exist.
281 :param filename: filename to use in error messages
282 :param source: source to look for
283 :param archive: archive to look in
284 :param extra_archives: list of archives to copy the source package from
285 if it is not yet present in `archive`
286 """
287 session = self.session
288 db_file = (
289 session.query(ArchiveFile)
290 .filter_by(file=source.poolfile, archive=archive)
291 .first()
292 )
293 if db_file is not None: 293 ↛ 297line 293 didn't jump to line 297 because the condition on line 293 was always true
294 return True
296 # Try to copy file from one extra archive
297 if extra_archives is None:
298 extra_archives = []
299 db_file = (
300 session.query(ArchiveFile)
301 .filter_by(file=source.poolfile)
302 .filter(ArchiveFile.archive_id.in_([a.archive_id for a in extra_archives]))
303 .first()
304 )
305 if db_file is None:
306 raise ArchiveException(
307 "{0}: Built-Using refers to package {1} (= {2}) not in target archive {3}.".format(
308 filename, source.source, source.version, archive.archive_name
309 )
310 )
312 source_archive = db_file.archive
313 for dsc_file in source.srcfiles:
314 af = (
315 session.query(ArchiveFile)
316 .filter_by(
317 file=dsc_file.poolfile,
318 archive=source_archive,
319 component=db_file.component,
320 )
321 .one()
322 )
323 # We were given an explicit list of archives so it is okay to copy from tainted archives.
324 self._copy_file(af.file, archive, db_file.component, allow_tainted=True)
326 def _add_built_using(
327 self, db_binary, filename, control, suite, extra_archives=None
328 ) -> None:
329 """Add Built-Using sources to ``db_binary.extra_sources``"""
330 session = self.session
332 for bu_source_name, bu_source_version in daklib.utils.parse_built_using(
333 control
334 ):
335 bu_source = (
336 session.query(DBSource)
337 .filter_by(source=bu_source_name, version=bu_source_version)
338 .first()
339 )
340 if bu_source is None: 340 ↛ 341line 340 didn't jump to line 341 because the condition on line 340 was never true
341 raise ArchiveException(
342 "{0}: Built-Using refers to non-existing source package {1} (= {2})".format(
343 filename, bu_source_name, bu_source_version
344 )
345 )
347 self._ensure_extra_source_exists(
348 filename, bu_source, suite.archive, extra_archives=extra_archives
349 )
351 db_binary.extra_sources.append(bu_source)
353 def _add_dsc_files(
354 self,
355 directory: str,
356 archive: Archive,
357 component: Component,
358 source: DBSource,
359 files: Iterable[daklib.upload.HashedFile],
360 *,
361 allow_tainted: bool,
362 extra_file: bool = False,
363 ) -> None:
364 for hashed_file in files:
365 hashed_file_path = os.path.join(directory, hashed_file.input_filename)
366 if os.path.exists(hashed_file_path): 366 ↛ 372line 366 didn't jump to line 372 because the condition on line 366 was always true
367 db_file = self._install_file(
368 directory, hashed_file, archive, component, source.source
369 )
370 self.session.add(db_file)
371 else:
372 db_file = self.get_file(hashed_file, source.source)
373 self._copy_file(
374 db_file, archive, component, allow_tainted=allow_tainted
375 )
377 db_dsc_file = DSCFile()
378 db_dsc_file.source = source
379 db_dsc_file.poolfile = db_file
380 db_dsc_file.extra_file = extra_file
381 self.session.add(db_dsc_file)
383 def install_source_to_archive(
384 self,
385 directory: str,
386 source: daklib.upload.Source,
387 archive: Archive,
388 component: Component,
389 changed_by: Maintainer,
390 *,
391 allow_tainted=False,
392 fingerprint: Optional[Fingerprint] = None,
393 authorized_by_fingerprint: Optional[Fingerprint] = None,
394 extra_source_files: Iterable[daklib.upload.HashedFile] = [],
395 ) -> DBSource:
396 """Install source package to archive"""
397 session = self.session
398 control = source.dsc
399 maintainer = get_or_set_maintainer(control["Maintainer"], session)
400 source_name = control["Source"]
402 ### Add source package to database
404 # We need to install the .dsc first as the DBSource object refers to it.
405 db_file_dsc = self._install_file(
406 directory, source._dsc_file, archive, component, source_name
407 )
409 unique = dict(
410 source=source_name,
411 version=control["Version"],
412 )
413 rest = dict(
414 maintainer=maintainer,
415 poolfile=db_file_dsc,
416 dm_upload_allowed=(control.get("DM-Upload-Allowed", "no") == "yes"),
417 )
418 # Other attributes that are ignored for purposes of equality with
419 # an existing source
420 rest2 = dict(
421 changedby=changed_by,
422 fingerprint=fingerprint,
423 authorized_by_fingerprint=authorized_by_fingerprint,
424 )
426 created = False
427 try:
428 db_source = session.query(DBSource).filter_by(**unique).one()
429 for key, value in rest.items():
430 if getattr(db_source, key) != value: 430 ↛ 431line 430 didn't jump to line 431 because the condition on line 430 was never true
431 raise ArchiveException(
432 "{0}: Does not match source in database.".format(
433 source._dsc_file.filename
434 )
435 )
436 except NoResultFound:
437 created = True
438 db_source = DBSource(**unique)
439 for key, value in rest.items():
440 setattr(db_source, key, value)
441 for key, value in rest2.items():
442 setattr(db_source, key, value)
443 session.add(db_source)
444 session.flush()
446 # Add .dsc file. Other files will be added later.
447 db_dsc_file = DSCFile()
448 db_dsc_file.source = db_source
449 db_dsc_file.poolfile = db_file_dsc
450 session.add(db_dsc_file)
451 session.flush()
453 if not created:
454 for f in db_source.srcfiles:
455 self._copy_file(
456 f.poolfile, archive, component, allow_tainted=allow_tainted
457 )
458 return db_source
460 ### Now add remaining files and copy them to the archive.
461 self._add_dsc_files(
462 directory,
463 archive,
464 component,
465 db_source,
466 source.files.values(),
467 allow_tainted=allow_tainted,
468 )
469 self._add_dsc_files(
470 directory,
471 archive,
472 component,
473 db_source,
474 extra_source_files,
475 allow_tainted=allow_tainted,
476 extra_file=True,
477 )
479 session.flush()
481 # Importing is safe as we only arrive here when we did not find the source already installed earlier.
482 import_metadata_into_db(db_source, session)
484 # Uploaders are the maintainer and co-maintainers from the Uploaders field
485 db_source.uploaders.append(maintainer)
486 if "Uploaders" in control:
487 from daklib.textutils import split_uploaders
489 for u in split_uploaders(control["Uploaders"]):
490 db_source.uploaders.append(get_or_set_maintainer(u, session))
491 session.flush()
493 return db_source
495 def install_source(
496 self,
497 directory: str,
498 source: daklib.upload.Source,
499 suite: Suite,
500 component: Component,
501 changed_by: Maintainer,
502 *,
503 allow_tainted: bool = False,
504 fingerprint: Optional[Fingerprint] = None,
505 authorized_by_fingerprint: Optional[Fingerprint] = None,
506 extra_source_files: Iterable[daklib.upload.HashedFile] = [],
507 ) -> DBSource:
508 """Install a source package
510 :param directory: directory the source package is located in
511 :param source: source package to install
512 :param suite: target suite
513 :param component: target component
514 :param changed_by: person who prepared this version of the package
515 :param allow_tainted: allow to copy additional files from tainted archives
516 :param fingerprint: optional fingerprint
517 :return: database object for the new source
518 """
519 db_source = self.install_source_to_archive(
520 directory,
521 source,
522 suite.archive,
523 component,
524 changed_by,
525 allow_tainted=allow_tainted,
526 fingerprint=fingerprint,
527 authorized_by_fingerprint=authorized_by_fingerprint,
528 extra_source_files=extra_source_files,
529 )
531 if suite in db_source.suites:
532 return db_source
533 db_source.suites.append(suite)
534 self.session.flush()
536 return db_source
538 def _copy_file(
539 self,
540 db_file: PoolFile,
541 archive: Archive,
542 component: Component,
543 allow_tainted: bool = False,
544 ) -> None:
545 """Copy a file to the given archive and component
547 :param db_file: file to copy
548 :param archive: target archive
549 :param component: target component
550 :param allow_tainted: allow to copy from tainted archives (such as NEW)
551 """
552 session = self.session
554 if (
555 session.query(ArchiveFile)
556 .filter_by(archive=archive, component=component, file=db_file)
557 .first()
558 is None
559 ):
560 query = session.query(ArchiveFile).filter_by(file=db_file)
561 if not allow_tainted:
562 query = query.join(Archive).filter(
563 Archive.tainted == False # noqa:E712
564 )
566 source_af = query.first()
567 if source_af is None: 567 ↛ 568line 567 didn't jump to line 568 because the condition on line 567 was never true
568 raise ArchiveException(
569 "cp: Could not find {0} in any archive.".format(db_file.filename)
570 )
571 target_af = ArchiveFile(archive, component, db_file)
572 session.add(target_af)
573 session.flush()
574 self.fs.copy(source_af.path, target_af.path, link=False, mode=archive.mode)
576 def copy_binary(
577 self,
578 db_binary: DBBinary,
579 suite: Suite,
580 component: Component,
581 allow_tainted: bool = False,
582 extra_archives: Optional[Iterable[Archive]] = None,
583 ) -> None:
584 """Copy a binary package to the given suite and component
586 :param db_binary: binary to copy
587 :param suite: target suite
588 :param component: target component
589 :param allow_tainted: allow to copy from tainted archives (such as NEW)
590 :param extra_archives: extra archives to copy Built-Using sources from
591 """
592 session = self.session
593 archive = suite.archive
594 if archive.tainted:
595 allow_tainted = True
597 filename = db_binary.poolfile.filename
599 # make sure source is present in target archive
600 db_source = db_binary.source
601 if ( 601 ↛ 607line 601 didn't jump to line 607
602 session.query(ArchiveFile)
603 .filter_by(archive=archive, file=db_source.poolfile)
604 .first()
605 is None
606 ):
607 raise ArchiveException(
608 "{0}: cannot copy to {1}: source is not present in target archive".format(
609 filename, suite.suite_name
610 )
611 )
613 # make sure built-using packages are present in target archive
614 for db_source in db_binary.extra_sources:
615 self._ensure_extra_source_exists(
616 filename, db_source, archive, extra_archives=extra_archives
617 )
619 # copy binary
620 db_file = db_binary.poolfile
621 self._copy_file(db_file, suite.archive, component, allow_tainted=allow_tainted)
622 if suite not in db_binary.suites:
623 db_binary.suites.append(suite)
624 self.session.flush()
626 def copy_source(
627 self,
628 db_source: DBSource,
629 suite: Suite,
630 component: Component,
631 allow_tainted: bool = False,
632 ) -> None:
633 """Copy a source package to the given suite and component
635 :param db_source: source to copy
636 :param suite: target suite
637 :param component: target component
638 :param allow_tainted: allow to copy from tainted archives (such as NEW)
639 """
640 archive = suite.archive
641 if archive.tainted:
642 allow_tainted = True
643 for db_dsc_file in db_source.srcfiles:
644 self._copy_file(
645 db_dsc_file.poolfile, archive, component, allow_tainted=allow_tainted
646 )
647 if suite not in db_source.suites:
648 db_source.suites.append(suite)
649 self.session.flush()
651 def remove_file(
652 self, db_file: PoolFile, archive: Archive, component: Component
653 ) -> None:
654 """Remove a file from a given archive and component
656 :param db_file: file to remove
657 :param archive: archive to remove the file from
658 :param component: component to remove the file from
659 """
660 af: ArchiveFile = (
661 self.session.query(ArchiveFile)
662 .filter_by(file=db_file, archive=archive, component=component)
663 .scalar()
664 )
665 self.fs.unlink(af.path)
666 self.session.delete(af)
668 def remove_binary(self, binary: DBBinary, suite: Suite) -> None:
669 """Remove a binary from a given suite and component
671 :param binary: binary to remove
672 :param suite: suite to remove the package from
673 """
674 binary.suites.remove(suite)
675 self.session.flush()
677 def remove_source(self, source: DBSource, suite: Suite) -> None:
678 """Remove a source from a given suite and component
680 :param source: source to remove
681 :param suite: suite to remove the package from
683 :raises ArchiveException: source package is still referenced by other
684 binaries in the suite
685 """
686 session = self.session
688 query = (
689 session.query(DBBinary)
690 .filter_by(source=source)
691 .filter(DBBinary.suites.contains(suite))
692 )
693 if query.first() is not None: 693 ↛ 694line 693 didn't jump to line 694 because the condition on line 693 was never true
694 raise ArchiveException(
695 "src:{0} is still used by binaries in suite {1}".format(
696 source.source, suite.suite_name
697 )
698 )
700 source.suites.remove(suite)
701 session.flush()
703 def commit(self) -> None:
704 """commit changes"""
705 try:
706 self.session.commit()
707 self.fs.commit()
708 finally:
709 self.session.rollback()
710 self.fs.rollback()
712 def rollback(self) -> None:
713 """rollback changes"""
714 self.session.rollback()
715 self.fs.rollback()
717 def flush(self) -> None:
718 """flush underlying database session"""
719 self.session.flush()
721 def __enter__(self):
722 return self
724 def __exit__(self, type, value, traceback):
725 if type is None:
726 self.commit()
727 else:
728 self.rollback()
729 return None
732def source_component_from_package_list(
733 package_list: "daklib.packagelist.PackageList", suite: Suite
734) -> Optional[Component]:
735 """Get component for a source package
737 This function will look at the Package-List field to determine the
738 component the source package belongs to. This is the first component
739 the source package provides binaries for (first with respect to the
740 ordering of components).
742 It the source package has no Package-List field, None is returned.
744 :param package_list: package list of the source to get the override for
745 :param suite: suite to consider for binaries produced
746 :return: component for the given source or :const:`None`
747 """
748 if package_list.fallback: 748 ↛ 749line 748 didn't jump to line 749 because the condition on line 748 was never true
749 return None
750 session = object_session(suite)
751 assert session is not None
752 packages = package_list.packages_for_suite(suite)
753 components = set(p.component for p in packages)
754 query = (
755 session.query(Component)
756 .order_by(Component.ordering)
757 .filter(Component.component_name.in_(components))
758 )
759 return query.first()
762class ArchiveUpload:
763 """handle an upload
765 This class can be used in a with-statement::
767 with ArchiveUpload(...) as upload:
768 ...
770 Doing so will automatically run any required cleanup and also rollback the
771 transaction if it was not committed.
772 """
774 def __init__(
775 self, directory: str, changes: daklib.upload.Changes, keyrings: Collection[str]
776 ):
777 self.transaction: ArchiveTransaction = ArchiveTransaction()
778 """transaction used to handle the upload"""
780 self.session = self.transaction.session
781 """database session"""
783 self.original_directory: str = directory
784 self.original_changes = changes
786 self._changes: Optional[daklib.upload.Changes] = None
787 """upload to process"""
789 self._extra_source_files: list[daklib.upload.HashedFile] = []
790 """extra source files"""
792 self._directory: str | None = None
793 """directory with temporary copy of files. set by :meth:`prepare`"""
795 self.keyrings = keyrings
797 self.fingerprint: Fingerprint = (
798 self.session.query(Fingerprint)
799 .filter_by(fingerprint=changes.primary_fingerprint)
800 .one()
801 )
802 """fingerprint of the key used to sign the upload"""
804 self._authorized_by_fingerprint: Optional[Fingerprint] = None
805 """fingerprint of the key that authorized the upload"""
807 self.reject_reasons: list[str] = []
808 """reasons why the upload cannot by accepted"""
810 self.warnings: list[str] = []
811 """warnings
813 .. note::
815 Not used yet.
816 """
818 self.final_suites: Optional[list[Suite]] = None
820 self.new: bool = False
821 """upload is NEW. set by :meth:`check`"""
823 self._checked: bool = False
824 """checks passes. set by :meth:`check`"""
826 self._new_queue = (
827 self.session.query(PolicyQueue).filter_by(queue_name="new").one()
828 )
829 self._new = self._new_queue.suite
831 @property
832 def changes(self) -> daklib.upload.Changes:
833 assert self._changes is not None
834 return self._changes
836 @property
837 def directory(self) -> str:
838 assert self._directory is not None
839 return self._directory
841 @property
842 def authorized_by_fingerprint(self) -> Fingerprint:
843 """
844 fingerprint of the key that authorized the upload
845 """
847 return (
848 self._authorized_by_fingerprint
849 if self._authorized_by_fingerprint is not None
850 else self.fingerprint
851 )
853 @authorized_by_fingerprint.setter
854 def authorized_by_fingerprint(self, fingerprint: Fingerprint) -> None:
855 self._authorized_by_fingerprint = fingerprint
857 def warn(self, message: str) -> None:
858 """add a warning message
860 Adds a warning message that can later be seen in :attr:`warnings`
862 :param message: warning message
863 """
864 self.warnings.append(message)
866 def prepare(self) -> None:
867 """prepare upload for further processing
869 This copies the files involved to a temporary directory. If you use
870 this method directly, you have to remove the directory given by the
871 :attr:`directory` attribute later on your own.
873 Instead of using the method directly, you can also use a with-statement::
875 with ArchiveUpload(...) as upload:
876 ...
878 This will automatically handle any required cleanup.
879 """
880 assert self._directory is None
881 assert self.original_changes.valid_signature
883 cnf = Config()
884 session = self.transaction.session
886 group = cnf.get("Dinstall::UnprivGroup") or None
887 self._directory = daklib.utils.temp_dirname(
888 parent=cnf.get("Dir::TempPath"), mode=0o2750, group=group
889 )
890 with FilesystemTransaction() as fs:
891 src = os.path.join(self.original_directory, self.original_changes.filename)
892 dst = os.path.join(self._directory, self.original_changes.filename)
893 fs.copy(src, dst, mode=0o640)
895 self._changes = daklib.upload.Changes(
896 self._directory, self.original_changes.filename, self.keyrings
897 )
899 files = {}
900 try:
901 files = self.changes.files
902 except daklib.upload.InvalidChangesException:
903 # Do not raise an exception; upload will be rejected later
904 # due to the missing files
905 pass
907 for f in files.values():
908 src = os.path.join(self.original_directory, f.filename)
909 dst = os.path.join(self._directory, f.filename)
910 if not os.path.exists(src):
911 continue
912 fs.copy(src, dst, mode=0o640)
914 source = None
915 try:
916 source = self.changes.source
917 except Exception:
918 # Do not raise an exception here if the .dsc is invalid.
919 pass
921 if source is not None:
922 for f in source.files.values():
923 src = os.path.join(self.original_directory, f.filename)
924 dst = os.path.join(self._directory, f.filename)
925 if not os.path.exists(dst):
926 try:
927 db_file = self.transaction.get_file(
928 f, source.dsc["Source"], check_hashes=False
929 )
930 db_archive_file = (
931 session.query(ArchiveFile)
932 .filter_by(file=db_file)
933 .first()
934 )
935 assert db_archive_file is not None
936 fs.copy(db_archive_file.path, dst, mode=0o640)
937 except KeyError:
938 # Ignore if get_file could not find it. Upload will
939 # probably be rejected later.
940 pass
942 def unpacked_source(self) -> Optional[str]:
943 """Path to unpacked source
945 Get path to the unpacked source. This method does unpack the source
946 into a temporary directory under :attr:`directory` if it has not
947 been done so already.
949 :return: string giving the path to the unpacked source directory
950 or :const:`None` if no source was included in the upload.
951 """
952 source = self.changes.source
953 if source is None:
954 return None
955 dsc_path = os.path.join(self.directory, source._dsc_file.filename)
957 sourcedir = os.path.join(self.directory, "source")
958 if not os.path.exists(sourcedir):
959 sandbox.run(
960 ["dpkg-source", "--no-copy", "--no-check", "-x", dsc_path, sourcedir],
961 sandbox=sandbox.Sandbox(
962 extra_read_write_paths=[
963 self.directory,
964 os.environ.get("TMPDIR", "/tmp"),
965 ],
966 ),
967 stdout=subprocess.DEVNULL,
968 check=True,
969 )
970 daklib.utils.remove_unsafe_symlinks(sourcedir)
971 if not os.path.isdir(sourcedir):
972 raise Exception(
973 "{0} is not a directory after extracting source package".format(
974 sourcedir
975 )
976 )
977 return sourcedir
979 def _map_suite(self, suite_name: str) -> set[str]:
980 suite_names = set((suite_name,))
981 for rule in Config().value_list("SuiteMappings"):
982 fields = rule.split()
983 rtype = fields[0]
984 if rtype == "map" or rtype == "silent-map": 984 ↛ 985line 984 didn't jump to line 985 because the condition on line 984 was never true
985 (src, dst) = fields[1:3]
986 if src in suite_names:
987 suite_names.remove(src)
988 suite_names.add(dst)
989 if rtype != "silent-map":
990 self.warnings.append("Mapping {0} to {1}.".format(src, dst))
991 elif rtype == "copy" or rtype == "silent-copy": 991 ↛ 992line 991 didn't jump to line 992 because the condition on line 991 was never true
992 (src, dst) = fields[1:3]
993 if src in suite_names:
994 suite_names.add(dst)
995 if rtype != "silent-copy":
996 self.warnings.append("Copy {0} to {1}.".format(src, dst))
997 elif rtype == "ignore": 997 ↛ 998line 997 didn't jump to line 998 because the condition on line 997 was never true
998 ignored = fields[1]
999 if ignored in suite_names:
1000 suite_names.remove(ignored)
1001 self.warnings.append("Ignoring target suite {0}.".format(ignored))
1002 elif rtype == "reject": 1002 ↛ 1003line 1002 didn't jump to line 1003 because the condition on line 1002 was never true
1003 rejected = fields[1]
1004 if rejected in suite_names:
1005 raise checks.Reject(
1006 "Uploads to {0} are not accepted.".format(rejected)
1007 )
1008 ## XXX: propup-version and map-unreleased not yet implemented
1009 return suite_names
1011 def _mapped_suites(self) -> list[Suite]:
1012 """Get target suites after mappings
1014 :return: list giving the mapped target suites of this upload
1015 """
1016 session = self.session
1018 suite_names = set()
1019 for dist in self.changes.distributions:
1020 suite_names.update(self._map_suite(dist))
1022 suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
1023 return suites.all()
1025 def _check_new_binary_overrides(self, suite: Suite, overridesuite: Suite) -> bool:
1026 new = False
1027 source = self.changes.source
1029 # Check binaries listed in the source package's Package-List field:
1030 if source is not None and not source.package_list.fallback:
1031 packages = source.package_list.packages_for_suite(suite)
1032 for b in packages:
1033 override = self._binary_override(overridesuite, b)
1034 if override is None:
1035 self.warnings.append("binary:{0} is NEW.".format(b.name))
1036 new = True
1038 # Check all uploaded packages.
1039 # This is necessary to account for packages without a Package-List
1040 # field, really late binary-only uploads (where an unused override
1041 # was already removed), and for debug packages uploaded to a suite
1042 # without a debug suite (which are then considered as NEW).
1043 for b2 in self.changes.binaries:
1044 if (
1045 daklib.utils.is_in_debug_section(b2.control)
1046 and suite.debug_suite is not None
1047 ):
1048 continue
1049 override = self._binary_override(overridesuite, b2)
1050 if override is None:
1051 self.warnings.append("binary:{0} is NEW.".format(b2.name))
1052 new = True
1054 return new
1056 def _check_new(self, suite: Suite, overridesuite: Suite) -> bool:
1057 """Check if upload is NEW
1059 An upload is NEW if it has binary or source packages that do not have
1060 an override in `overridesuite` OR if it references files ONLY in a
1061 tainted archive (eg. when it references files in NEW).
1063 Debug packages (*-dbgsym in Section: debug) are not considered as NEW
1064 if `suite` has a separate debug suite.
1066 :return: :const:`True` if the upload is NEW, :const:`False` otherwise
1067 """
1068 session = self.session
1069 new = False
1071 # Check for missing overrides
1072 if self._check_new_binary_overrides(suite, overridesuite):
1073 new = True
1074 if self.changes.source is not None:
1075 override = self._source_override(overridesuite, self.changes.source)
1076 if override is None:
1077 self.warnings.append(
1078 "source:{0} is NEW.".format(self.changes.source.dsc["Source"])
1079 )
1080 new = True
1082 # Check if we reference a file only in a tainted archive
1083 files = list(self.changes.files.values())
1084 if self.changes.source is not None:
1085 files.extend(self.changes.source.files.values())
1086 for f in files:
1087 query = (
1088 session.query(ArchiveFile)
1089 .join(PoolFile)
1090 .filter(PoolFile.sha1sum == f.sha1sum)
1091 )
1092 query_untainted = query.join(Archive).filter(
1093 Archive.tainted == False # noqa:E712
1094 )
1096 in_archive = query.first() is not None
1097 in_untainted_archive = query_untainted.first() is not None
1099 if in_archive and not in_untainted_archive:
1100 self.warnings.append("{0} is only available in NEW.".format(f.filename))
1101 new = True
1103 return new
1105 def _final_suites(self) -> list[Suite]:
1106 session = self.session
1108 mapped_suites = self._mapped_suites()
1109 final_suites: list[Suite] = []
1111 for suite in mapped_suites:
1112 overridesuite = suite
1113 if suite.overridesuite is not None:
1114 overridesuite = (
1115 session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
1116 )
1117 if self._check_new(suite, overridesuite):
1118 self.new = True
1119 if suite not in final_suites: 1119 ↛ 1111line 1119 didn't jump to line 1111 because the condition on line 1119 was always true
1120 final_suites.append(suite)
1122 return final_suites
1124 def _binary_override(
1125 self,
1126 suite: Suite,
1127 binary: "Union[daklib.upload.Binary, daklib.packagelist.PackageListEntry]",
1128 ) -> Optional[Override]:
1129 """Get override entry for a binary
1131 :param suite: suite to get override for
1132 :param binary: binary to get override for
1133 :return: override for the given binary or :const:`None`
1134 """
1135 if suite.overridesuite is not None:
1136 suite = (
1137 self.session.query(Suite)
1138 .filter_by(suite_name=suite.overridesuite)
1139 .one()
1140 )
1142 if binary.component is None: 1142 ↛ 1143line 1142 didn't jump to line 1143 because the condition on line 1142 was never true
1143 return None
1144 mapped_component = get_mapped_component(binary.component)
1145 if mapped_component is None: 1145 ↛ 1146line 1145 didn't jump to line 1146 because the condition on line 1145 was never true
1146 return None
1148 query = (
1149 self.session.query(Override)
1150 .filter_by(suite=suite, package=binary.name)
1151 .join(Component)
1152 .filter(Component.component_name == mapped_component.component_name)
1153 .join(OverrideType)
1154 .filter(OverrideType.overridetype == binary.type)
1155 )
1157 return query.one_or_none()
1159 def _source_override(
1160 self, suite: Suite, source: daklib.upload.Source
1161 ) -> Optional[Override]:
1162 """Get override entry for a source
1164 :param suite: suite to get override for
1165 :param source: source to get override for
1166 :return: override for the given source or :const:`None`
1167 """
1168 if suite.overridesuite is not None: 1168 ↛ 1169line 1168 didn't jump to line 1169
1169 suite = (
1170 self.session.query(Suite)
1171 .filter_by(suite_name=suite.overridesuite)
1172 .one()
1173 )
1175 query = (
1176 self.session.query(Override)
1177 .filter_by(suite=suite, package=source.dsc["Source"])
1178 .join(OverrideType)
1179 .filter(OverrideType.overridetype == "dsc")
1180 )
1182 component = source_component_from_package_list(source.package_list, suite)
1183 if component is not None:
1184 query = query.filter(Override.component == component)
1186 return query.one_or_none()
1188 def _binary_component(
1189 self, suite: Suite, binary: daklib.upload.Binary, only_overrides: bool = True
1190 ) -> Optional[Component]:
1191 """get component for a binary
1193 By default this will only look at overrides to get the right component;
1194 if `only_overrides` is :const:`False` this method will also look at the
1195 Section field.
1197 :param only_overrides: only use overrides to get the right component
1198 """
1199 override = self._binary_override(suite, binary)
1200 if override is not None:
1201 return override.component
1202 if only_overrides: 1202 ↛ 1203line 1202 didn't jump to line 1203 because the condition on line 1202 was never true
1203 return None
1204 return get_mapped_component(binary.component, self.session)
1206 def _source_component(
1207 self, suite: Suite, source: daklib.upload.Source, only_overrides: bool = True
1208 ) -> Optional[Component]:
1209 """get component for a source
1211 By default this will only look at overrides to get the right component;
1212 if `only_overrides` is :const:`False` this method will also look at the
1213 Section field.
1215 :param only_overrides: only use overrides to get the right component
1216 """
1217 override = self._source_override(suite, source)
1218 if override is not None: 1218 ↛ 1220line 1218 didn't jump to line 1220 because the condition on line 1218 was always true
1219 return override.component
1220 if only_overrides:
1221 return None
1222 return get_mapped_component(source.component, self.session)
1224 def _run_checks(
1225 self,
1226 force: bool,
1227 simple_checks: Iterable[type[checks.Check]],
1228 per_suite_checks: Collection[type[checks.Check]],
1229 suites: Collection[Suite],
1230 ) -> bool:
1231 try:
1232 for check in simple_checks:
1233 check().check(self)
1235 if per_suite_checks and not suites: 1235 ↛ 1236line 1235 didn't jump to line 1236 because the condition on line 1235 was never true
1236 raise ValueError(
1237 "Per-suite checks should be called, but no suites given."
1238 )
1239 for check in per_suite_checks:
1240 for suite in suites:
1241 check().per_suite_check(self, suite)
1242 except checks.Reject as e: 1242 ↛ 1245line 1242 didn't jump to line 1245
1243 self.reject_reasons.append(str(e))
1244 return False
1245 except Exception as e:
1246 self.reject_reasons.append(
1247 "Processing raised an exception: {0}.\n{1}".format(
1248 e, traceback.format_exc()
1249 )
1250 )
1251 return False
1253 if len(self.reject_reasons) != 0: 1253 ↛ 1254line 1253 didn't jump to line 1254 because the condition on line 1253 was never true
1254 return False
1255 return True
1257 def _run_checks_very_early(self, force: bool) -> bool:
1258 """
1259 run very early checks
1261 These check validate signatures on .changes and hashes.
1262 """
1263 return self._run_checks(
1264 force=force,
1265 simple_checks=[
1266 checks.SignatureAndHashesCheck,
1267 checks.WeakSignatureCheck,
1268 checks.SignatureTimestampCheck,
1269 ],
1270 per_suite_checks=[],
1271 suites=[],
1272 )
1274 def _run_checks_early(self, force: bool) -> bool:
1275 """
1276 run early checks
1278 These are checks that run after checking signatures, but
1279 before deciding the target suite.
1281 This should cover archive-wide policies, sanity checks, ...
1282 """
1283 return self._run_checks(
1284 force=force,
1285 simple_checks=[
1286 checks.ChangesCheck,
1287 checks.ExternalHashesCheck,
1288 checks.SourceCheck,
1289 checks.BinaryCheck,
1290 checks.BinaryMembersCheck,
1291 checks.BinaryTimestampCheck,
1292 checks.SingleDistributionCheck,
1293 checks.ArchAllBinNMUCheck,
1294 ],
1295 per_suite_checks=[],
1296 suites=[],
1297 )
1299 def _run_checks_late(self, force: bool, suites: Collection[Suite]) -> bool:
1300 """
1301 run late checks
1303 These are checks that run after the target suites are known.
1305 This should cover permission checks, suite-specific polices
1306 (e.g., lintian), version constraints, ...
1307 """
1308 return self._run_checks(
1309 force=force,
1310 simple_checks=[
1311 checks.TransitionCheck,
1312 checks.ACLCheck,
1313 checks.NewOverrideCheck,
1314 checks.NoSourceOnlyCheck,
1315 checks.LintianCheck,
1316 ],
1317 per_suite_checks=[
1318 checks.SuiteCheck,
1319 checks.ACLCheck,
1320 checks.SourceFormatCheck,
1321 checks.SuiteArchitectureCheck,
1322 checks.VersionCheck,
1323 ],
1324 suites=suites,
1325 )
1327 def _handle_tag2upload(self) -> bool:
1328 """
1329 check if upload is via tag2upload
1331 if so, determine who authorized the upload to notify them of
1332 rejections and for ACL checks
1333 """
1335 if not (keyring := self.fingerprint.keyring) or not keyring.tag2upload:
1336 return True
1338 source = self.changes.source
1339 if not source: 1339 ↛ 1340line 1339 didn't jump to line 1340 because the condition on line 1339 was never true
1340 self.reject_reasons.append("tag2upload: upload missing source")
1341 return False
1343 try:
1344 tag2upload_file, info = get_tag2upload_info_for_upload(self)
1345 except Exception as e:
1346 self.reject_reasons.append(f"tag2upload: invalid metadata: {e}")
1347 return False
1348 self._extra_source_files.append(tag2upload_file)
1350 success = True
1352 if self.changes.binaries: 1352 ↛ 1353line 1352 didn't jump to line 1353 because the condition on line 1352 was never true
1353 success = False
1354 self.reject_reasons.append("tag2upload: upload includes binaries")
1355 if self.changes.byhand_files: 1355 ↛ 1356line 1355 didn't jump to line 1356 because the condition on line 1355 was never true
1356 success = False
1357 self.reject_reasons.append("tag2upload: upload included by-hand files")
1359 if not info.signed_file.valid: 1359 ↛ 1360line 1359 didn't jump to line 1360 because the condition on line 1359 was never true
1360 success = False
1361 self.reject_reasons.append("tag2upload: no valid signature on tag")
1362 else:
1363 # Only set with a valid signature, but also when we reject
1364 # the upload so the signer might get included in the
1365 # rejection mail.
1366 self.authorized_by_fingerprint = (
1367 self.session.query(Fingerprint)
1368 .filter_by(fingerprint=info.signed_file.primary_fingerprint)
1369 .one()
1370 )
1371 if info.signed_file.weak_signature: 1371 ↛ 1372line 1371 didn't jump to line 1372 because the condition on line 1371 was never true
1372 success = False
1373 self.reject_reasons.append(
1374 "tag2upload: tag was signed using a weak algorithm (such as SHA-1)"
1375 )
1376 try:
1377 checks.check_signature_timestamp("tag2upload", info.signed_file)
1378 except checks.Reject as e:
1379 success = False
1380 self.reject_reasons.append(str(e))
1382 if info.metadata.get("distro") != "debian": 1382 ↛ 1383line 1382 didn't jump to line 1383 because the condition on line 1382 was never true
1383 success = False
1384 self.reject_reasons.append("tag2upload: upload not targeted at Debian.")
1385 if info.metadata.get("source") != source.dsc["Source"]: 1385 ↛ 1386line 1385 didn't jump to line 1386 because the condition on line 1385 was never true
1386 success = False
1387 self.reject_reasons.append(
1388 "tag2upload: source from tag metadata does not match upload"
1389 )
1390 if info.metadata.get("version") != source.dsc["Version"]: 1390 ↛ 1391line 1390 didn't jump to line 1391 because the condition on line 1390 was never true
1391 success = False
1392 self.reject_reasons.append(
1393 "tag2upload: version from tag metadata does not match upload"
1394 )
1396 tag_info_field = source.dsc.get("Git-Tag-Info")
1397 if not tag_info_field: 1397 ↛ 1398line 1397 didn't jump to line 1398 because the condition on line 1397 was never true
1398 success = False
1399 self.reject_reasons.append("tag2upload: source misses Git-Tag-Info field")
1400 else:
1401 try:
1402 tag_info = parse_git_tag_info(tag_info_field)
1403 except ValueError:
1404 success = False
1405 self.reject_reasons.append("tag2upload: could not parse Git-Tag-Info")
1406 else:
1407 if tag_info.fp.upper() != info.signed_file.fingerprint: 1407 ↛ 1408line 1407 didn't jump to line 1408 because the condition on line 1407 was never true
1408 success = False
1409 self.reject_reasons.append(
1410 "tag2upload: signing key from Git and Git-Tag-Info differ"
1411 )
1413 return success
1415 def check(self, force: bool = False) -> bool:
1416 """run checks against the upload
1418 :param force: ignore failing forcable checks
1419 :return: :const:`True` if all checks passed, :const:`False` otherwise
1420 """
1421 # XXX: needs to be better structured.
1422 assert self.changes.valid_signature
1424 # Validate signatures and hashes before we do any real work:
1425 if not self._run_checks_very_early(force):
1426 return False
1428 if not self._handle_tag2upload(): 1428 ↛ 1429line 1428 didn't jump to line 1429 because the condition on line 1428 was never true
1429 return False
1431 if not self._run_checks_early(force): 1431 ↛ 1432line 1431 didn't jump to line 1432 because the condition on line 1431 was never true
1432 return False
1434 try:
1435 final_suites = self._final_suites()
1436 except Exception as e:
1437 self.reject_reasons.append(
1438 "Processing raised an exception: {0}.\n{1}".format(
1439 e, traceback.format_exc()
1440 )
1441 )
1442 return False
1443 if len(final_suites) == 0:
1444 self.reject_reasons.append(
1445 "No target suite found. Please check your target distribution and that you uploaded to the right archive."
1446 )
1447 return False
1449 self.final_suites = final_suites
1451 if not self._run_checks_late(force, final_suites):
1452 return False
1454 if len(self.reject_reasons) != 0: 1454 ↛ 1455line 1454 didn't jump to line 1455 because the condition on line 1454 was never true
1455 return False
1457 self._checked = True
1458 return True
1460 def _install_to_suite(
1461 self,
1462 target_suite: Suite,
1463 suite: Suite,
1464 source_component_func: Callable[[daklib.upload.Source], Component],
1465 binary_component_func: Callable[[daklib.upload.Binary], Component],
1466 source_suites=None,
1467 extra_source_archives: Optional[Iterable[Archive]] = None,
1468 policy_upload: bool = False,
1469 ) -> tuple[Optional[DBSource], list[DBBinary]]:
1470 """Install upload to the given suite
1472 :param target_suite: target suite (before redirection to policy queue or NEW)
1473 :param suite: suite to install the package into. This is the real suite,
1474 ie. after any redirection to NEW or a policy queue
1475 :param source_component_func: function to get the :class:`daklib.dbconn.Component`
1476 for a :class:`daklib.upload.Source` object
1477 :param binary_component_func: function to get the :class:`daklib.dbconn.Component`
1478 for a :class:`daklib.upload.Binary` object
1479 :param source_suites: see :meth:`daklib.archive.ArchiveTransaction.install_binary`
1480 :param extra_source_archives: see :meth:`daklib.archive.ArchiveTransaction.install_binary`
1481 :param policy_upload: Boolean indicating upload to policy queue (including NEW)
1482 :return: tuple with two elements. The first is a :class:`daklib.dbconn.DBSource`
1483 object for the install source or :const:`None` if no source was
1484 included. The second is a list of :class:`daklib.dbconn.DBBinary`
1485 objects for the installed binary packages.
1486 """
1487 # XXX: move this function to ArchiveTransaction?
1489 control = self.changes.changes
1490 changed_by = get_or_set_maintainer(
1491 control.get("Changed-By", control["Maintainer"]), self.session
1492 )
1494 if source_suites is None: 1494 ↛ 1495line 1494 didn't jump to line 1495
1495 source_suites = (
1496 self.session.query(Suite)
1497 .join(VersionCheck, VersionCheck.reference_id == Suite.suite_id)
1498 .filter(VersionCheck.check == "Enhances")
1499 .filter(VersionCheck.suite == suite)
1500 .subquery()
1501 )
1503 source = self.changes.source
1504 if source is not None:
1505 component = source_component_func(source)
1506 db_source = self.transaction.install_source(
1507 self.directory,
1508 source,
1509 suite,
1510 component,
1511 changed_by,
1512 fingerprint=self.fingerprint,
1513 authorized_by_fingerprint=self.authorized_by_fingerprint,
1514 extra_source_files=self._extra_source_files,
1515 )
1516 else:
1517 db_source = None
1519 db_binaries = []
1520 for binary in sorted(self.changes.binaries, key=lambda x: x.name):
1521 copy_to_suite = suite
1522 if (
1523 daklib.utils.is_in_debug_section(binary.control)
1524 and suite.debug_suite is not None
1525 ):
1526 copy_to_suite = suite.debug_suite
1528 component = binary_component_func(binary)
1529 db_binary = self.transaction.install_binary(
1530 self.directory,
1531 binary,
1532 copy_to_suite,
1533 component,
1534 fingerprint=self.fingerprint,
1535 authorized_by_fingerprint=self.authorized_by_fingerprint,
1536 source_suites=source_suites,
1537 extra_source_archives=extra_source_archives,
1538 )
1539 db_binaries.append(db_binary)
1541 if not policy_upload:
1542 check_upload_for_external_signature_request(
1543 self.session, target_suite, copy_to_suite, db_binary
1544 )
1546 if suite.copychanges: 1546 ↛ 1547line 1546 didn't jump to line 1547 because the condition on line 1546 was never true
1547 src = os.path.join(self.directory, self.changes.filename)
1548 dst = os.path.join(
1549 suite.archive.path, "dists", suite.suite_name, self.changes.filename
1550 )
1551 self.transaction.fs.copy(src, dst, mode=suite.archive.mode)
1553 suite.update_last_changed()
1555 return (db_source, db_binaries)
1557 def _install_changes(self) -> DBChange:
1558 assert self.changes.valid_signature
1559 control = self.changes.changes
1560 session = self.transaction.session
1562 changelog_id = None
1563 # Only add changelog for sourceful uploads and binNMUs
1564 if self.changes.sourceful or re_bin_only_nmu.search(control["Version"]):
1565 query = "INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id"
1566 changelog_id = session.execute(
1567 sql.text(query), {"changelog": control["Changes"]}
1568 ).scalar()
1569 assert changelog_id is not None
1571 db_changes = DBChange()
1572 db_changes.changesname = self.changes.filename
1573 db_changes.source = control["Source"]
1574 db_changes.binaries = control.get("Binary", None)
1575 db_changes.architecture = control["Architecture"]
1576 db_changes.version = control["Version"]
1577 db_changes.distribution = control["Distribution"]
1578 db_changes.urgency = control["Urgency"]
1579 db_changes.maintainer = control["Maintainer"]
1580 db_changes.changedby = control.get("Changed-By", control["Maintainer"])
1581 db_changes.date = control["Date"]
1582 db_changes.fingerprint = self.fingerprint.fingerprint
1583 db_changes.authorized_by_fingerprint = (
1584 self.authorized_by_fingerprint.fingerprint
1585 )
1586 db_changes.changelog_id = changelog_id
1587 db_changes.closes = self.changes.closed_bugs
1589 try:
1590 self.transaction.session.add(db_changes)
1591 self.transaction.session.flush()
1592 except sqlalchemy.exc.IntegrityError:
1593 raise ArchiveException(
1594 "{0} is already known.".format(self.changes.filename)
1595 )
1597 return db_changes
1599 def _install_policy(
1600 self, policy_queue, target_suite, db_changes, db_source, db_binaries
1601 ) -> PolicyQueueUpload:
1602 """install upload to policy queue"""
1603 u = PolicyQueueUpload()
1604 u.policy_queue = policy_queue
1605 u.target_suite = target_suite
1606 u.changes = db_changes
1607 u.source = db_source
1608 u.binaries = db_binaries
1609 self.transaction.session.add(u)
1610 self.transaction.session.flush()
1612 queue_files = [self.changes.filename]
1613 queue_files.extend(f.filename for f in self.changes.buildinfo_files)
1614 for fn in queue_files:
1615 src = os.path.join(self.changes.directory, fn)
1616 dst = os.path.join(policy_queue.path, fn)
1617 self.transaction.fs.copy(src, dst, mode=policy_queue.change_perms)
1619 return u
1621 def try_autobyhand(self) -> bool:
1622 """Try AUTOBYHAND
1624 Try to handle byhand packages automatically.
1625 """
1626 assert len(self.reject_reasons) == 0
1627 assert self.changes.valid_signature
1628 assert self.final_suites is not None
1629 assert self._checked
1631 byhand = self.changes.byhand_files
1632 if len(byhand) == 0: 1632 ↛ 1635line 1632 didn't jump to line 1635 because the condition on line 1632 was always true
1633 return True
1635 suites = list(self.final_suites)
1636 assert len(suites) == 1, "BYHAND uploads must be to a single suite"
1637 suite = suites[0]
1639 cnf = Config()
1640 control = self.changes.changes
1641 automatic_byhand_packages = cnf.subtree("AutomaticByHandPackages")
1643 remaining = []
1644 for f in byhand:
1645 if "_" in f.filename:
1646 parts = f.filename.split("_", 2)
1647 if len(parts) != 3:
1648 print(
1649 "W: unexpected byhand filename {0}. No automatic processing.".format(
1650 f.filename
1651 )
1652 )
1653 remaining.append(f)
1654 continue
1656 package, _, archext = parts
1657 arch, ext = archext.split(".", 1)
1658 else:
1659 parts = f.filename.split(".")
1660 if len(parts) < 2:
1661 print(
1662 "W: unexpected byhand filename {0}. No automatic processing.".format(
1663 f.filename
1664 )
1665 )
1666 remaining.append(f)
1667 continue
1669 package = parts[0]
1670 arch = "all"
1671 ext = parts[-1]
1673 try:
1674 rule = automatic_byhand_packages.subtree(package)
1675 except KeyError:
1676 remaining.append(f)
1677 continue
1679 if (
1680 rule["Source"] != self.changes.source_name
1681 or rule["Section"] != f.section
1682 or ("Extension" in rule and rule["Extension"] != ext)
1683 ):
1684 remaining.append(f)
1685 continue
1687 script = rule["Script"]
1688 retcode = subprocess.call(
1689 [
1690 script,
1691 os.path.join(self.directory, f.filename),
1692 control["Version"],
1693 arch,
1694 os.path.join(self.directory, self.changes.filename),
1695 suite.suite_name,
1696 ],
1697 shell=False,
1698 )
1699 if retcode != 0:
1700 print("W: error processing {0}.".format(f.filename))
1701 remaining.append(f)
1703 return len(remaining) == 0
1705 def _install_byhand(
1706 self,
1707 policy_queue_upload: PolicyQueueUpload,
1708 hashed_file: daklib.upload.HashedFile,
1709 ) -> PolicyQueueByhandFile:
1710 """install byhand file"""
1711 fs = self.transaction.fs
1712 session = self.transaction.session
1713 policy_queue = policy_queue_upload.policy_queue
1715 byhand_file = PolicyQueueByhandFile()
1716 byhand_file.upload = policy_queue_upload
1717 byhand_file.filename = hashed_file.filename
1718 session.add(byhand_file)
1719 session.flush()
1721 src = os.path.join(self.directory, hashed_file.filename)
1722 dst = os.path.join(policy_queue.path, hashed_file.filename)
1723 fs.copy(src, dst, mode=policy_queue.change_perms)
1725 return byhand_file
1727 def _do_bts_versiontracking(self) -> None:
1728 cnf = Config()
1729 fs = self.transaction.fs
1731 btsdir = cnf.get("Dir::BTSVersionTrack")
1732 if btsdir is None or btsdir == "": 1732 ↛ 1735line 1732 didn't jump to line 1735 because the condition on line 1732 was always true
1733 return
1735 base = os.path.join(btsdir, self.changes.filename[:-8])
1737 # version history
1738 sourcedir = self.unpacked_source()
1739 if sourcedir is not None:
1740 dch_path = os.path.join(sourcedir, "debian", "changelog")
1741 with open(dch_path, "r") as fh:
1742 versions = fs.create("{0}.versions".format(base), mode=0o644)
1743 for line in fh.readlines():
1744 if re_changelog_versions.match(line):
1745 versions.write(line)
1746 versions.close()
1748 # binary -> source mapping
1749 if self.changes.binaries:
1750 debinfo = fs.create("{0}.debinfo".format(base), mode=0o644)
1751 for binary in self.changes.binaries:
1752 control = binary.control
1753 source_package, source_version = binary.source
1754 line = " ".join(
1755 [
1756 control["Package"],
1757 control["Version"],
1758 control["Architecture"],
1759 source_package,
1760 source_version,
1761 ]
1762 )
1763 print(line, file=debinfo)
1764 debinfo.close()
1766 def _policy_queue(self, suite) -> Optional[PolicyQueue]:
1767 if suite.policy_queue is not None:
1768 return suite.policy_queue
1769 return None
1771 def install(self) -> None:
1772 """install upload
1774 Install upload to a suite or policy queue. This method does **not**
1775 handle uploads to NEW.
1777 You need to have called the :meth:`check` method before calling this method.
1778 """
1779 assert len(self.reject_reasons) == 0
1780 assert self.changes.valid_signature
1781 assert self.final_suites is not None
1782 assert self._checked
1783 assert not self.new
1785 db_changes = self._install_changes()
1787 for suite in self.final_suites:
1788 overridesuite = suite
1789 if suite.overridesuite is not None:
1790 overridesuite = (
1791 self.session.query(Suite)
1792 .filter_by(suite_name=suite.overridesuite)
1793 .one()
1794 )
1796 policy_queue = self._policy_queue(suite)
1797 policy_upload = False
1799 redirected_suite = suite
1800 if policy_queue is not None:
1801 redirected_suite = policy_queue.suite
1802 policy_upload = True
1804 # source can be in the suite we install to or any suite we enhance
1805 source_suite_ids = set([suite.suite_id, redirected_suite.suite_id])
1806 for (enhanced_suite_id,) in (
1807 self.session.query(VersionCheck.reference_id)
1808 .filter(VersionCheck.suite_id.in_(source_suite_ids))
1809 .filter(VersionCheck.check == "Enhances")
1810 ):
1811 source_suite_ids.add(enhanced_suite_id)
1813 source_suites = (
1814 self.session.query(Suite)
1815 .filter(Suite.suite_id.in_(source_suite_ids))
1816 .subquery()
1817 )
1819 def source_component_func(source: daklib.upload.Source) -> Component:
1820 component = self._source_component(
1821 overridesuite, source, only_overrides=False
1822 )
1823 assert component is not None
1824 return component
1826 def binary_component_func(binary: daklib.upload.Binary) -> Component:
1827 component = self._binary_component(
1828 overridesuite, binary, only_overrides=False
1829 )
1830 assert component is not None
1831 return component
1833 (db_source, db_binaries) = self._install_to_suite(
1834 suite,
1835 redirected_suite,
1836 source_component_func,
1837 binary_component_func,
1838 source_suites=source_suites,
1839 extra_source_archives=[suite.archive],
1840 policy_upload=policy_upload,
1841 )
1843 if policy_queue is not None:
1844 self._install_policy(
1845 policy_queue, suite, db_changes, db_source, db_binaries
1846 )
1848 # copy to build queues
1849 if policy_queue is None or policy_queue.send_to_build_queues: 1849 ↛ 1787line 1849 didn't jump to line 1787 because the condition on line 1849 was always true
1850 for build_queue in suite.copy_queues:
1851 self._install_to_suite(
1852 suite,
1853 build_queue.suite,
1854 source_component_func,
1855 binary_component_func,
1856 source_suites=source_suites,
1857 extra_source_archives=[suite.archive],
1858 )
1860 self._do_bts_versiontracking()
1862 def install_to_new(self) -> None:
1863 """install upload to NEW
1865 Install upload to NEW. This method does **not** handle regular uploads
1866 to suites or policy queues.
1868 You need to have called the :meth:`check` method before calling this method.
1869 """
1870 # Uploads to NEW are special as we don't have overrides.
1871 assert len(self.reject_reasons) == 0
1872 assert self.changes.valid_signature
1873 assert self.final_suites is not None
1875 binaries = self.changes.binaries
1876 byhand = self.changes.byhand_files
1878 # we need a suite to guess components
1879 suites = list(self.final_suites)
1880 assert len(suites) == 1, "NEW uploads must be to a single suite"
1881 suite = suites[0]
1883 # decide which NEW queue to use
1884 if suite.new_queue is None: 1884 ↛ 1891line 1884 didn't jump to line 1891 because the condition on line 1884 was always true
1885 new_queue = (
1886 self.transaction.session.query(PolicyQueue)
1887 .filter_by(queue_name="new")
1888 .one()
1889 )
1890 else:
1891 new_queue = suite.new_queue
1892 if len(byhand) > 0: 1892 ↛ 1894line 1892 didn't jump to line 1894
1893 # There is only one global BYHAND queue
1894 new_queue = (
1895 self.transaction.session.query(PolicyQueue)
1896 .filter_by(queue_name="byhand")
1897 .one()
1898 )
1899 new_suite = new_queue.suite
1901 def binary_component_func(binary: daklib.upload.Binary) -> Component:
1902 component = self._binary_component(suite, binary, only_overrides=False)
1903 assert component is not None
1904 return component
1906 # guess source component
1907 # XXX: should be moved into an extra method
1908 binary_component_names = set()
1909 for binary in binaries:
1910 component = binary_component_func(binary)
1911 binary_component_names.add(component.component_name)
1912 source_component_name = None
1913 for c in self.session.query(Component).order_by(Component.component_id):
1914 guess = c.component_name
1915 if guess in binary_component_names:
1916 source_component_name = guess
1917 break
1918 if source_component_name is None:
1919 source_component = (
1920 self.session.query(Component).order_by(Component.component_id).first()
1921 )
1922 else:
1923 source_component = (
1924 self.session.query(Component)
1925 .filter_by(component_name=source_component_name)
1926 .one()
1927 )
1928 assert source_component is not None
1930 def source_component_func(source: daklib.upload.Source) -> Component:
1931 return source_component
1933 db_changes = self._install_changes()
1934 (db_source, db_binaries) = self._install_to_suite(
1935 suite,
1936 new_suite,
1937 source_component_func,
1938 binary_component_func,
1939 source_suites=True,
1940 extra_source_archives=[suite.archive],
1941 policy_upload=True,
1942 )
1943 policy_upload = self._install_policy(
1944 new_queue, suite, db_changes, db_source, db_binaries
1945 )
1947 for f in byhand: 1947 ↛ 1948line 1947 didn't jump to line 1948 because the loop on line 1947 never started
1948 self._install_byhand(policy_upload, f)
1950 self._do_bts_versiontracking()
1952 def commit(self) -> None:
1953 """commit changes"""
1954 self.transaction.commit()
1956 def rollback(self) -> None:
1957 """rollback changes"""
1958 self.transaction.rollback()
1960 def __enter__(self):
1961 self.prepare()
1962 return self
1964 def __exit__(self, type, value, traceback):
1965 if self._directory is not None: 1965 ↛ 1968line 1965 didn't jump to line 1968 because the condition on line 1965 was always true
1966 shutil.rmtree(self._directory)
1967 self._directory = None
1968 self._changes = None
1969 self.transaction.rollback()
1970 return None