Coverage for daklib/archive.py: 75%
821 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-08-03 16:46 +0000
1# Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU General Public License as published by
5# the Free Software Foundation; either version 2 of the License, or
6# (at your option) any later version.
7#
8# This program is distributed in the hope that it will be useful,
9# but WITHOUT ANY WARRANTY; without even the implied warranty of
10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11# GNU General Public License for more details.
12#
13# You should have received a copy of the GNU General Public License along
14# with this program; if not, write to the Free Software Foundation, Inc.,
15# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17"""module to manipulate the archive
19This module provides classes to manipulate the archive.
20"""
22import os
23import shutil
24import subprocess
25import traceback
26from collections.abc import Callable, Collection, Iterable
27from typing import TYPE_CHECKING
29import sqlalchemy.exc
30from sqlalchemy import select, sql
31from sqlalchemy.orm import object_session
33import daklib.upload
34import daklib.utils
35from daklib import checks, sandbox
36from daklib.config import Config
37from daklib.dbconn import (
38 Archive,
39 ArchiveFile,
40 Component,
41 DBBinary,
42 DBChange,
43 DBConn,
44 DBSource,
45 DSCFile,
46 Fingerprint,
47 Maintainer,
48 Override,
49 OverrideType,
50 PolicyQueue,
51 PolicyQueueByhandFile,
52 PolicyQueueUpload,
53 PoolFile,
54 Suite,
55 VersionCheck,
56 get_architecture,
57 get_mapped_component,
58 get_or_set_maintainer,
59 get_suite_by_name,
60 import_metadata_into_db,
61)
62from daklib.externalsignature import check_upload_for_external_signature_request
63from daklib.fstransactions import FilesystemTransaction
64from daklib.regexes import re_bin_only_nmu, re_changelog_versions
65from daklib.tag2upload import get_tag2upload_info_for_upload, parse_git_tag_info
67if TYPE_CHECKING:
68 import daklib.packagelist
71class ArchiveException(Exception):
72 pass
75class HashMismatchException(ArchiveException):
76 pass
79class ArchiveTransaction:
80 """manipulate the archive in a transaction"""
82 def __init__(self):
83 self.fs = FilesystemTransaction()
84 self.session = DBConn().session()
86 def get_file(
87 self,
88 hashed_file: daklib.upload.HashedFile,
89 source_name: str,
90 check_hashes: bool = True,
91 ) -> PoolFile:
92 """Look for file `hashed_file` in database
94 :param hashed_file: file to look for in the database
95 :param source_name: source package name
96 :param check_hashes: check size and hashes match
97 :return: database entry for the file
98 :raises KeyError: file was not found in the database
99 :raises HashMismatchException: hash mismatch
100 """
101 poolname = os.path.join(daklib.utils.poolify(source_name), hashed_file.filename)
102 poolfile = self.session.execute(
103 select(PoolFile).where(PoolFile.filename == poolname)
104 ).scalar_one_or_none()
105 if poolfile is None:
106 raise KeyError("{0} not found in database.".format(poolname))
107 if check_hashes and ( 107 ↛ 113line 107 didn't jump to line 113 because the condition on line 107 was never true
108 poolfile.filesize != hashed_file.size
109 or poolfile.md5sum != hashed_file.md5sum
110 or poolfile.sha1sum != hashed_file.sha1sum
111 or poolfile.sha256sum != hashed_file.sha256sum
112 ):
113 raise HashMismatchException(
114 "{0}: Does not match file already existing in the pool.".format(
115 hashed_file.filename
116 )
117 )
118 return poolfile
120 def _install_file(
121 self, directory, hashed_file, archive, component, source_name
122 ) -> PoolFile:
123 """Install a file
125 Will not give an error when the file is already present.
127 :return: database object for the new file
128 """
129 session = self.session
131 poolname = os.path.join(daklib.utils.poolify(source_name), hashed_file.filename)
132 try:
133 poolfile = self.get_file(hashed_file, source_name)
134 except KeyError:
135 poolfile = PoolFile(filename=poolname, filesize=hashed_file.size)
136 poolfile.md5sum = hashed_file.md5sum
137 poolfile.sha1sum = hashed_file.sha1sum
138 poolfile.sha256sum = hashed_file.sha256sum
139 session.add(poolfile)
140 session.flush()
142 archive_file_query = select(ArchiveFile).filter_by(
143 archive=archive, component=component, file=poolfile
144 )
145 if session.execute(archive_file_query).scalar_one_or_none() is None:
146 archive_file = ArchiveFile(archive, component, poolfile)
147 session.add(archive_file)
148 session.flush()
150 path = os.path.join(
151 archive.path, "pool", component.component_name, poolname
152 )
153 hashed_file_path = os.path.join(directory, hashed_file.input_filename)
154 self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
156 return poolfile
158 def install_binary(
159 self,
160 directory: str,
161 binary: daklib.upload.Binary,
162 suite: Suite,
163 component: Component,
164 *,
165 allow_tainted: bool = False,
166 fingerprint: Fingerprint | None = None,
167 authorized_by_fingerprint: Fingerprint | None = None,
168 source_suites=None,
169 extra_source_archives: Iterable[Archive] | None = None,
170 ) -> DBBinary:
171 """Install a binary package
173 :param directory: directory the binary package is located in
174 :param binary: binary package to install
175 :param suite: target suite
176 :param component: target component
177 :param allow_tainted: allow to copy additional files from tainted archives
178 :param fingerprint: optional fingerprint
179 :param source_suites: suites to copy the source from if they are not
180 in `suite` or :const:`True` to allow copying from any
181 suite.
182 Can be a SQLAlchemy subquery for :class:`Suite` or :const:`True`.
183 :param extra_source_archives: extra archives to copy Built-Using sources from
184 :return: database object for the new package
185 """
186 session = self.session
187 control = binary.control
188 maintainer = get_or_set_maintainer(control["Maintainer"], session)
189 architecture = get_architecture(control["Architecture"], session)
191 (source_name, source_version) = binary.source
192 source_query = select(DBSource).filter_by(
193 source=source_name, version=source_version
194 )
195 source = session.scalars(
196 source_query.where(DBSource.suites.contains(suite)).limit(1)
197 ).first()
198 if source is None:
199 if source_suites is not True:
200 source_query = source_query.join(DBSource.suites).where(
201 Suite.suite_id == source_suites.c.id
202 )
203 source = session.scalars(source_query.limit(1)).first()
204 if source is None: 204 ↛ 205line 204 didn't jump to line 205 because the condition on line 204 was never true
205 raise ArchiveException(
206 "{0}: trying to install to {1}, but could not find source ({2} {3})".format(
207 binary.hashed_file.filename,
208 suite.suite_name,
209 source_name,
210 source_version,
211 )
212 )
213 self.copy_source(source, suite, source.poolfile.component)
215 db_file = self._install_file(
216 directory, binary.hashed_file, suite.archive, component, source_name
217 )
219 unique = {
220 "package": control["Package"],
221 "version": control["Version"],
222 "architecture": architecture,
223 }
224 rest = {
225 "source": source,
226 "maintainer": maintainer,
227 "poolfile": db_file,
228 "binarytype": binary.type,
229 }
230 # Other attributes that are ignored for purposes of equality with
231 # an existing source
232 rest2 = {
233 "fingerprint": fingerprint,
234 "authorized_by_fingerprint": authorized_by_fingerprint,
235 }
237 db_binary = session.execute(
238 select(DBBinary).filter_by(**unique)
239 ).scalar_one_or_none()
240 if db_binary is not None:
241 for key, value in rest.items():
242 if getattr(db_binary, key) != value: 242 ↛ 243line 242 didn't jump to line 243 because the condition on line 242 was never true
243 raise ArchiveException(
244 "{0}: Does not match binary in database.".format(
245 binary.hashed_file.filename
246 )
247 )
248 else:
249 db_binary = DBBinary(**unique)
250 for key, value in rest.items():
251 setattr(db_binary, key, value)
252 for key, value in rest2.items():
253 setattr(db_binary, key, value)
254 session.add(db_binary)
255 session.flush()
256 import_metadata_into_db(db_binary, session)
258 self._add_built_using(
259 db_binary,
260 binary.hashed_file.filename,
261 control,
262 suite,
263 extra_archives=extra_source_archives,
264 )
266 if suite not in db_binary.suites:
267 db_binary.suites.append(suite)
269 session.flush()
271 return db_binary
273 def _ensure_extra_source_exists(
274 self,
275 filename: str,
276 source: DBSource,
277 archive: Archive,
278 extra_archives: Iterable[Archive] | None = None,
279 ):
280 """ensure source exists in the given archive
282 This is intended to be used to check that Built-Using sources exist.
284 :param filename: filename to use in error messages
285 :param source: source to look for
286 :param archive: archive to look in
287 :param extra_archives: list of archives to copy the source package from
288 if it is not yet present in `archive`
289 """
290 session = self.session
291 db_file = session.scalars(
292 select(ArchiveFile)
293 .filter_by(file=source.poolfile, archive=archive)
294 .limit(1)
295 ).first()
296 if db_file is not None: 296 ↛ 300line 296 didn't jump to line 300 because the condition on line 296 was always true
297 return True
299 # Try to copy file from one extra archive
300 if extra_archives is None:
301 extra_archives = []
302 db_file = session.scalars(
303 select(ArchiveFile)
304 .filter_by(file=source.poolfile)
305 .where(ArchiveFile.archive_id.in_([a.archive_id for a in extra_archives]))
306 .limit(1)
307 ).first()
308 if db_file is None:
309 raise ArchiveException(
310 "{0}: Built-Using refers to package {1} (= {2}) not in target archive {3}.".format(
311 filename, source.source, source.version, archive.archive_name
312 )
313 )
315 source_archive = db_file.archive
316 for dsc_file in source.srcfiles:
317 af = session.execute(
318 select(ArchiveFile).filter_by(
319 file=dsc_file.poolfile,
320 archive=source_archive,
321 component=db_file.component,
322 )
323 ).scalar_one()
324 # We were given an explicit list of archives so it is okay to copy from tainted archives.
325 self._copy_file(af.file, archive, db_file.component, allow_tainted=True)
327 def _add_built_using(
328 self, db_binary, filename, control, suite, extra_archives=None
329 ) -> None:
330 """Add Built-Using sources to ``db_binary.extra_sources``"""
331 session = self.session
333 for bu_source_name, bu_source_version in daklib.utils.parse_built_using(
334 control
335 ):
336 bu_source = session.scalars(
337 select(DBSource)
338 .filter_by(source=bu_source_name, version=bu_source_version)
339 .limit(1)
340 ).first()
341 if bu_source is None: 341 ↛ 342line 341 didn't jump to line 342 because the condition on line 341 was never true
342 raise ArchiveException(
343 "{0}: Built-Using refers to non-existing source package {1} (= {2})".format(
344 filename, bu_source_name, bu_source_version
345 )
346 )
348 self._ensure_extra_source_exists(
349 filename, bu_source, suite.archive, extra_archives=extra_archives
350 )
352 db_binary.extra_sources.append(bu_source)
354 def _add_dsc_files(
355 self,
356 directory: str,
357 archive: Archive,
358 component: Component,
359 source: DBSource,
360 files: Iterable[daklib.upload.HashedFile],
361 *,
362 allow_tainted: bool,
363 extra_file: bool = False,
364 ) -> None:
365 for hashed_file in files:
366 hashed_file_path = os.path.join(directory, hashed_file.input_filename)
367 if os.path.exists(hashed_file_path): 367 ↛ 373line 367 didn't jump to line 373 because the condition on line 367 was always true
368 db_file = self._install_file(
369 directory, hashed_file, archive, component, source.source
370 )
371 self.session.add(db_file)
372 else:
373 db_file = self.get_file(hashed_file, source.source)
374 self._copy_file(
375 db_file, archive, component, allow_tainted=allow_tainted
376 )
378 db_dsc_file = DSCFile()
379 db_dsc_file.source = source
380 db_dsc_file.poolfile = db_file
381 db_dsc_file.extra_file = extra_file
382 self.session.add(db_dsc_file)
384 def install_source_to_archive(
385 self,
386 directory: str,
387 source: daklib.upload.Source,
388 archive: Archive,
389 component: Component,
390 changed_by: Maintainer,
391 *,
392 allow_tainted=False,
393 fingerprint: Fingerprint | None = None,
394 authorized_by_fingerprint: Fingerprint | None = None,
395 extra_source_files: Iterable[daklib.upload.HashedFile] = [],
396 ) -> DBSource:
397 """Install source package to archive"""
398 session = self.session
399 control = source.dsc
400 maintainer = get_or_set_maintainer(control["Maintainer"], session)
401 source_name = control["Source"]
403 ### Add source package to database
405 # We need to install the .dsc first as the DBSource object refers to it.
406 db_file_dsc = self._install_file(
407 directory, source._dsc_file, archive, component, source_name
408 )
410 unique = {
411 "source": source_name,
412 "version": control["Version"],
413 }
414 rest = {
415 "maintainer": maintainer,
416 "poolfile": db_file_dsc,
417 "dm_upload_allowed": (control.get("DM-Upload-Allowed", "no") == "yes"),
418 }
419 # Other attributes that are ignored for purposes of equality with
420 # an existing source
421 rest2 = {
422 "changedby": changed_by,
423 "fingerprint": fingerprint,
424 "authorized_by_fingerprint": authorized_by_fingerprint,
425 }
427 created = False
428 db_source = session.execute(
429 select(DBSource).filter_by(**unique)
430 ).scalar_one_or_none()
431 if db_source is not None:
432 for key, value in rest.items():
433 if getattr(db_source, key) != value: 433 ↛ 434line 433 didn't jump to line 434 because the condition on line 433 was never true
434 raise ArchiveException(
435 "{0}: Does not match source in database.".format(
436 source._dsc_file.filename
437 )
438 )
439 else:
440 created = True
441 db_source = DBSource(**unique)
442 for key, value in rest.items():
443 setattr(db_source, key, value)
444 for key, value in rest2.items():
445 setattr(db_source, key, value)
446 session.add(db_source)
447 session.flush()
449 # Add .dsc file. Other files will be added later.
450 db_dsc_file = DSCFile()
451 db_dsc_file.source = db_source
452 db_dsc_file.poolfile = db_file_dsc
453 session.add(db_dsc_file)
454 session.flush()
456 if not created:
457 for f in db_source.srcfiles:
458 self._copy_file(
459 f.poolfile, archive, component, allow_tainted=allow_tainted
460 )
461 return db_source
463 ### Now add remaining files and copy them to the archive.
464 self._add_dsc_files(
465 directory,
466 archive,
467 component,
468 db_source,
469 source.files.values(),
470 allow_tainted=allow_tainted,
471 )
472 self._add_dsc_files(
473 directory,
474 archive,
475 component,
476 db_source,
477 extra_source_files,
478 allow_tainted=allow_tainted,
479 extra_file=True,
480 )
482 session.flush()
484 # Importing is safe as we only arrive here when we did not find the source already installed earlier.
485 import_metadata_into_db(db_source, session)
487 # Uploaders are the maintainer and co-maintainers from the Uploaders field
488 db_source.uploaders.append(maintainer)
489 if "Uploaders" in control:
490 from daklib.textutils import split_uploaders
492 for u in split_uploaders(control["Uploaders"]):
493 db_source.uploaders.append(get_or_set_maintainer(u, session))
494 session.flush()
496 return db_source
498 def install_source(
499 self,
500 directory: str,
501 source: daklib.upload.Source,
502 suite: Suite,
503 component: Component,
504 changed_by: Maintainer,
505 *,
506 allow_tainted: bool = False,
507 fingerprint: Fingerprint | None = None,
508 authorized_by_fingerprint: Fingerprint | None = None,
509 extra_source_files: Iterable[daklib.upload.HashedFile] = [],
510 ) -> DBSource:
511 """Install a source package
513 :param directory: directory the source package is located in
514 :param source: source package to install
515 :param suite: target suite
516 :param component: target component
517 :param changed_by: person who prepared this version of the package
518 :param allow_tainted: allow to copy additional files from tainted archives
519 :param fingerprint: optional fingerprint
520 :return: database object for the new source
521 """
522 db_source = self.install_source_to_archive(
523 directory,
524 source,
525 suite.archive,
526 component,
527 changed_by,
528 allow_tainted=allow_tainted,
529 fingerprint=fingerprint,
530 authorized_by_fingerprint=authorized_by_fingerprint,
531 extra_source_files=extra_source_files,
532 )
534 if suite in db_source.suites:
535 return db_source
536 db_source.suites.append(suite)
537 self.session.flush()
539 return db_source
541 def _copy_file(
542 self,
543 db_file: PoolFile,
544 archive: Archive,
545 component: Component,
546 allow_tainted: bool = False,
547 ) -> None:
548 """Copy a file to the given archive and component
550 :param db_file: file to copy
551 :param archive: target archive
552 :param component: target component
553 :param allow_tainted: allow to copy from tainted archives (such as NEW)
554 """
555 session = self.session
557 if (
558 session.scalars(
559 select(ArchiveFile)
560 .filter_by(archive=archive, component=component, file=db_file)
561 .limit(1)
562 ).first()
563 is None
564 ):
565 query = select(ArchiveFile).filter_by(file=db_file)
566 if not allow_tainted:
567 query = query.join(Archive).where(~Archive.tainted)
569 source_af = session.scalars(query.limit(1)).first()
570 if source_af is None: 570 ↛ 571line 570 didn't jump to line 571 because the condition on line 570 was never true
571 raise ArchiveException(
572 "cp: Could not find {0} in any archive.".format(db_file.filename)
573 )
574 target_af = ArchiveFile(archive, component, db_file)
575 session.add(target_af)
576 session.flush()
577 self.fs.copy(source_af.path, target_af.path, link=False, mode=archive.mode)
579 def copy_binary(
580 self,
581 db_binary: DBBinary,
582 suite: Suite,
583 component: Component,
584 allow_tainted: bool = False,
585 extra_archives: Iterable[Archive] | None = None,
586 ) -> None:
587 """Copy a binary package to the given suite and component
589 :param db_binary: binary to copy
590 :param suite: target suite
591 :param component: target component
592 :param allow_tainted: allow to copy from tainted archives (such as NEW)
593 :param extra_archives: extra archives to copy Built-Using sources from
594 """
595 session = self.session
596 archive = suite.archive
597 if archive.tainted:
598 allow_tainted = True
600 filename = db_binary.poolfile.filename
602 # make sure source is present in target archive
603 db_source = db_binary.source
604 if ( 604 ↛ 612line 604 didn't jump to line 612
605 session.scalars(
606 select(ArchiveFile)
607 .filter_by(archive=archive, file=db_source.poolfile)
608 .limit(1)
609 ).first()
610 is None
611 ):
612 raise ArchiveException(
613 "{0}: cannot copy to {1}: source is not present in target archive".format(
614 filename, suite.suite_name
615 )
616 )
618 # make sure built-using packages are present in target archive
619 for db_source in db_binary.extra_sources:
620 self._ensure_extra_source_exists(
621 filename, db_source, archive, extra_archives=extra_archives
622 )
624 # copy binary
625 db_file = db_binary.poolfile
626 self._copy_file(db_file, suite.archive, component, allow_tainted=allow_tainted)
627 if suite not in db_binary.suites:
628 db_binary.suites.append(suite)
629 self.session.flush()
631 def copy_source(
632 self,
633 db_source: DBSource,
634 suite: Suite,
635 component: Component,
636 allow_tainted: bool = False,
637 ) -> None:
638 """Copy a source package to the given suite and component
640 :param db_source: source to copy
641 :param suite: target suite
642 :param component: target component
643 :param allow_tainted: allow to copy from tainted archives (such as NEW)
644 """
645 archive = suite.archive
646 if archive.tainted:
647 allow_tainted = True
648 for db_dsc_file in db_source.srcfiles:
649 self._copy_file(
650 db_dsc_file.poolfile, archive, component, allow_tainted=allow_tainted
651 )
652 if suite not in db_source.suites:
653 db_source.suites.append(suite)
654 self.session.flush()
656 def remove_file(
657 self, db_file: PoolFile, archive: Archive, component: Component
658 ) -> None:
659 """Remove a file from a given archive and component
661 :param db_file: file to remove
662 :param archive: archive to remove the file from
663 :param component: component to remove the file from
664 """
665 af: ArchiveFile = self.session.execute(
666 select(ArchiveFile).filter_by(
667 file=db_file, archive=archive, component=component
668 )
669 ).scalar_one()
670 self.fs.unlink(af.path)
671 self.session.delete(af)
673 def remove_binary(self, binary: DBBinary, suite: Suite) -> None:
674 """Remove a binary from a given suite and component
676 :param binary: binary to remove
677 :param suite: suite to remove the package from
678 """
679 binary.suites.remove(suite)
680 self.session.flush()
682 def remove_source(self, source: DBSource, suite: Suite) -> None:
683 """Remove a source from a given suite and component
685 :param source: source to remove
686 :param suite: suite to remove the package from
688 :raises ArchiveException: source package is still referenced by other
689 binaries in the suite
690 """
691 session = self.session
693 query = (
694 select(DBBinary)
695 .filter_by(source=source)
696 .where(DBBinary.suites.contains(suite))
697 .limit(1)
698 )
699 if session.scalars(query).first() is not None: 699 ↛ 700line 699 didn't jump to line 700 because the condition on line 699 was never true
700 raise ArchiveException(
701 "src:{0} is still used by binaries in suite {1}".format(
702 source.source, suite.suite_name
703 )
704 )
706 source.suites.remove(suite)
707 session.flush()
709 def commit(self) -> None:
710 """commit changes"""
711 try:
712 self.session.commit()
713 self.fs.commit()
714 finally:
715 self.session.rollback()
716 self.fs.rollback()
718 def rollback(self) -> None:
719 """rollback changes"""
720 self.session.rollback()
721 self.fs.rollback()
723 def flush(self) -> None:
724 """flush underlying database session"""
725 self.session.flush()
727 def __enter__(self):
728 return self
730 def __exit__(self, type, value, traceback):
731 if type is None:
732 self.commit()
733 else:
734 self.rollback()
737def source_component_from_package_list(
738 package_list: "daklib.packagelist.PackageList", suite: Suite
739) -> Component | None:
740 """Get component for a source package
742 This function will look at the Package-List field to determine the
743 component the source package belongs to. This is the first component
744 the source package provides binaries for (first with respect to the
745 ordering of components).
747 It the source package has no Package-List field, None is returned.
749 :param package_list: package list of the source to get the override for
750 :param suite: suite to consider for binaries produced
751 :return: component for the given source or :const:`None`
752 """
753 if package_list.fallback: 753 ↛ 754line 753 didn't jump to line 754 because the condition on line 753 was never true
754 return None
755 session = object_session(suite)
756 assert session is not None
757 packages = package_list.packages_for_suite(suite)
758 components = {p.component for p in packages}
759 query = (
760 select(Component)
761 .order_by(Component.ordering)
762 .where(Component.component_name.in_(components))
763 .limit(1)
764 )
765 return session.scalars(query).first()
768class ArchiveUpload:
769 """handle an upload
771 This class can be used in a with-statement::
773 with ArchiveUpload(...) as upload:
774 ...
776 Doing so will automatically run any required cleanup and also rollback the
777 transaction if it was not committed.
778 """
780 def __init__(
781 self, directory: str, changes: daklib.upload.Changes, keyrings: Collection[str]
782 ):
783 self.transaction: ArchiveTransaction = ArchiveTransaction()
784 """transaction used to handle the upload"""
786 self.session = self.transaction.session
787 """database session"""
789 self.original_directory: str = directory
790 self.original_changes = changes
792 self._changes: daklib.upload.Changes | None = None
793 """upload to process"""
795 self._extra_source_files: list[daklib.upload.HashedFile] = []
796 """extra source files"""
798 self._directory: str | None = None
799 """directory with temporary copy of files. set by :meth:`prepare`"""
801 self.keyrings = keyrings
803 self.fingerprint: Fingerprint = self.session.execute(
804 select(Fingerprint).filter_by(fingerprint=changes.primary_fingerprint)
805 ).scalar_one()
806 """fingerprint of the key used to sign the upload"""
808 self._authorized_by_fingerprint: Fingerprint | None = None
809 """fingerprint of the key that authorized the upload"""
811 self.reject_reasons: list[str] = []
812 """reasons why the upload cannot by accepted"""
814 self.warnings: list[str] = []
815 """warnings
817 .. note::
819 Not used yet.
820 """
822 self.final_suites: list[Suite] | None = None
824 self.new: bool = False
825 """upload is NEW. set by :meth:`check`"""
827 self._checked: bool = False
828 """checks passes. set by :meth:`check`"""
830 self._new_queue = self.session.execute(
831 select(PolicyQueue).filter_by(queue_name="new")
832 ).scalar_one()
833 self._new = self._new_queue.suite
835 @property
836 def changes(self) -> daklib.upload.Changes:
837 assert self._changes is not None
838 return self._changes
840 @property
841 def directory(self) -> str:
842 assert self._directory is not None
843 return self._directory
845 @property
846 def authorized_by_fingerprint(self) -> Fingerprint:
847 """
848 fingerprint of the key that authorized the upload
849 """
851 return (
852 self._authorized_by_fingerprint
853 if self._authorized_by_fingerprint is not None
854 else self.fingerprint
855 )
857 @authorized_by_fingerprint.setter
858 def authorized_by_fingerprint(self, fingerprint: Fingerprint) -> None:
859 self._authorized_by_fingerprint = fingerprint
861 def warn(self, message: str) -> None:
862 """add a warning message
864 Adds a warning message that can later be seen in :attr:`warnings`
866 :param message: warning message
867 """
868 self.warnings.append(message)
870 def prepare(self) -> None:
871 """prepare upload for further processing
873 This copies the files involved to a temporary directory. If you use
874 this method directly, you have to remove the directory given by the
875 :attr:`directory` attribute later on your own.
877 Instead of using the method directly, you can also use a with-statement::
879 with ArchiveUpload(...) as upload:
880 ...
882 This will automatically handle any required cleanup.
883 """
884 assert self._directory is None
885 assert self.original_changes.valid_signature
887 cnf = Config()
888 session = self.transaction.session
890 group = cnf.get("Dinstall::UnprivGroup") or None
891 self._directory = daklib.utils.temp_dirname(
892 parent=cnf.get("Dir::TempPath"), mode=0o2750, group=group
893 )
894 with FilesystemTransaction() as fs:
895 src = os.path.join(self.original_directory, self.original_changes.filename)
896 dst = os.path.join(self._directory, self.original_changes.filename)
897 fs.copy(src, dst, mode=0o640)
899 self._changes = daklib.upload.Changes(
900 self._directory, self.original_changes.filename, self.keyrings
901 )
903 files = {}
904 try:
905 files = self.changes.files
906 except daklib.upload.InvalidChangesException:
907 # Do not raise an exception; upload will be rejected later
908 # due to the missing files
909 pass
911 for f in files.values():
912 src = os.path.join(self.original_directory, f.filename)
913 dst = os.path.join(self._directory, f.filename)
914 if not os.path.exists(src):
915 continue
916 fs.copy(src, dst, mode=0o640)
918 source = None
919 try:
920 source = self.changes.source
921 except Exception:
922 # Do not raise an exception here if the .dsc is invalid.
923 pass
925 if source is not None:
926 for f in source.files.values():
927 src = os.path.join(self.original_directory, f.filename)
928 dst = os.path.join(self._directory, f.filename)
929 if not os.path.exists(dst):
930 try:
931 db_file = self.transaction.get_file(
932 f, source.dsc["Source"], check_hashes=False
933 )
934 db_archive_file = session.scalars(
935 select(ArchiveFile).filter_by(file=db_file).limit(1)
936 ).first()
937 assert db_archive_file is not None
938 fs.copy(db_archive_file.path, dst, mode=0o640)
939 except KeyError:
940 # Ignore if get_file could not find it. Upload will
941 # probably be rejected later.
942 pass
944 def unpacked_source(self) -> str | None:
945 """Path to unpacked source
947 Get path to the unpacked source. This method does unpack the source
948 into a temporary directory under :attr:`directory` if it has not
949 been done so already.
951 :return: string giving the path to the unpacked source directory
952 or :const:`None` if no source was included in the upload.
953 """
954 source = self.changes.source
955 if source is None:
956 return None
957 dsc_path = os.path.join(self.directory, source._dsc_file.filename)
959 sourcedir = os.path.join(self.directory, "source")
960 if not os.path.exists(sourcedir):
961 sandbox.run(
962 ["dpkg-source", "--no-copy", "--no-check", "-x", dsc_path, sourcedir],
963 sandbox=sandbox.Sandbox(
964 extra_read_write_paths=[
965 self.directory,
966 os.environ.get("TMPDIR", "/tmp"),
967 ],
968 ),
969 stdout=subprocess.DEVNULL,
970 check=True,
971 )
972 daklib.utils.remove_unsafe_symlinks(sourcedir)
973 if not os.path.isdir(sourcedir):
974 raise Exception(
975 "{0} is not a directory after extracting source package".format(
976 sourcedir
977 )
978 )
979 return sourcedir
981 def _map_suite(self, suite_name: str) -> set[str]:
982 suite_names = {suite_name}
983 for rule in Config().value_list("SuiteMappings"):
984 fields = rule.split()
985 rtype = fields[0]
986 if rtype == "map" or rtype == "silent-map": 986 ↛ 987line 986 didn't jump to line 987 because the condition on line 986 was never true
987 (src, dst) = fields[1:3]
988 if src in suite_names:
989 suite_names.remove(src)
990 suite_names.add(dst)
991 if rtype != "silent-map":
992 self.warnings.append("Mapping {0} to {1}.".format(src, dst))
993 elif rtype == "copy" or rtype == "silent-copy": 993 ↛ 994line 993 didn't jump to line 994 because the condition on line 993 was never true
994 (src, dst) = fields[1:3]
995 if src in suite_names:
996 suite_names.add(dst)
997 if rtype != "silent-copy":
998 self.warnings.append("Copy {0} to {1}.".format(src, dst))
999 elif rtype == "ignore": 999 ↛ 1000line 999 didn't jump to line 1000 because the condition on line 999 was never true
1000 ignored = fields[1]
1001 if ignored in suite_names:
1002 suite_names.remove(ignored)
1003 self.warnings.append("Ignoring target suite {0}.".format(ignored))
1004 elif rtype == "reject": 1004 ↛ 1005line 1004 didn't jump to line 1005 because the condition on line 1004 was never true
1005 rejected = fields[1]
1006 if rejected in suite_names:
1007 raise checks.Reject(
1008 "Uploads to {0} are not accepted.".format(rejected)
1009 )
1010 ## XXX: propup-version and map-unreleased not yet implemented
1011 return suite_names
1013 def _mapped_suites(self) -> list[Suite]:
1014 """Get target suites after mappings
1016 :return: list giving the mapped target suites of this upload
1017 """
1018 session = self.session
1020 suite_names = set()
1021 for dist in self.changes.distributions:
1022 suite_names.update(self._map_suite(dist))
1024 suites = select(Suite).where(Suite.suite_name.in_(suite_names))
1025 return list(session.scalars(suites))
1027 def _check_new_binary_overrides(self, suite: Suite, overridesuite: Suite) -> bool:
1028 new = False
1029 source = self.changes.source
1031 # Check binaries listed in the source package's Package-List field:
1032 if source is not None and not source.package_list.fallback:
1033 packages = source.package_list.packages_for_suite(suite)
1034 for b in packages:
1035 override = self._binary_override(overridesuite, b)
1036 if override is None:
1037 self.warnings.append("binary:{0} is NEW.".format(b.name))
1038 new = True
1040 # Check all uploaded packages.
1041 # This is necessary to account for packages without a Package-List
1042 # field, really late binary-only uploads (where an unused override
1043 # was already removed), and for debug packages uploaded to a suite
1044 # without a debug suite (which are then considered as NEW).
1045 for b2 in self.changes.binaries:
1046 if (
1047 daklib.utils.is_in_debug_section(b2.control)
1048 and suite.debug_suite is not None
1049 ):
1050 continue
1051 override = self._binary_override(overridesuite, b2)
1052 if override is None:
1053 self.warnings.append("binary:{0} is NEW.".format(b2.name))
1054 new = True
1056 return new
1058 def _check_new(self, suite: Suite, overridesuite: Suite) -> bool:
1059 """Check if upload is NEW
1061 An upload is NEW if it has binary or source packages that do not have
1062 an override in `overridesuite` OR if it references files ONLY in a
1063 tainted archive (eg. when it references files in NEW).
1065 Debug packages (*-dbgsym in Section: debug) are not considered as NEW
1066 if `suite` has a separate debug suite.
1068 :return: :const:`True` if the upload is NEW, :const:`False` otherwise
1069 """
1070 session = self.session
1071 new = False
1073 # Check for missing overrides
1074 if self._check_new_binary_overrides(suite, overridesuite):
1075 new = True
1076 if self.changes.source is not None:
1077 override = self._source_override(overridesuite, self.changes.source)
1078 if override is None:
1079 self.warnings.append(
1080 "source:{0} is NEW.".format(self.changes.source.dsc["Source"])
1081 )
1082 new = True
1084 # Check if we reference a file only in a tainted archive
1085 files = list(self.changes.files.values())
1086 if self.changes.source is not None:
1087 files.extend(self.changes.source.files.values())
1088 for f in files:
1089 query = (
1090 select(ArchiveFile).join(PoolFile).where(PoolFile.sha1sum == f.sha1sum)
1091 )
1092 query_untainted = query.join(Archive).where(~Archive.tainted)
1094 in_archive = session.scalars(query.limit(1)).first() is not None
1095 in_untainted_archive = (
1096 session.scalars(query_untainted.limit(1)).first() is not None
1097 )
1099 if in_archive and not in_untainted_archive:
1100 self.warnings.append("{0} is only available in NEW.".format(f.filename))
1101 new = True
1103 return new
1105 def _final_suites(self) -> list[Suite]:
1106 session = self.session
1108 mapped_suites = self._mapped_suites()
1109 final_suites: list[Suite] = []
1111 for suite in mapped_suites:
1112 overridesuite = suite
1113 if suite.overridesuite is not None:
1114 overridesuite = get_suite_by_name(suite.overridesuite, session)
1115 if self._check_new(suite, overridesuite):
1116 self.new = True
1117 if suite not in final_suites: 1117 ↛ 1111line 1117 didn't jump to line 1111 because the condition on line 1117 was always true
1118 final_suites.append(suite)
1120 return final_suites
1122 def _binary_override(
1123 self,
1124 suite: Suite,
1125 binary: "daklib.upload.Binary | daklib.packagelist.PackageListEntry",
1126 ) -> Override | None:
1127 """Get override entry for a binary
1129 :param suite: suite to get override for
1130 :param binary: binary to get override for
1131 :return: override for the given binary or :const:`None`
1132 """
1133 if suite.overridesuite is not None:
1134 suite = get_suite_by_name(suite.overridesuite, self.session)
1136 if binary.component is None: 1136 ↛ 1137line 1136 didn't jump to line 1137 because the condition on line 1136 was never true
1137 return None
1138 mapped_component = get_mapped_component(binary.component)
1139 if mapped_component is None: 1139 ↛ 1140line 1139 didn't jump to line 1140 because the condition on line 1139 was never true
1140 return None
1142 query = (
1143 select(Override)
1144 .filter_by(suite=suite, package=binary.name)
1145 .join(Component)
1146 .where(Component.component_name == mapped_component.component_name)
1147 .join(OverrideType)
1148 .where(OverrideType.overridetype == binary.type)
1149 )
1151 return self.session.execute(query).scalar_one_or_none()
1153 def _source_override(
1154 self, suite: Suite, source: daklib.upload.Source
1155 ) -> Override | None:
1156 """Get override entry for a source
1158 :param suite: suite to get override for
1159 :param source: source to get override for
1160 :return: override for the given source or :const:`None`
1161 """
1162 if suite.overridesuite is not None: 1162 ↛ 1163line 1162 didn't jump to line 1163 because the condition on line 1162 was never true
1163 suite = get_suite_by_name(suite.overridesuite, self.session)
1165 query = (
1166 select(Override)
1167 .filter_by(suite=suite, package=source.dsc["Source"])
1168 .join(OverrideType)
1169 .where(OverrideType.overridetype == "dsc")
1170 )
1172 component = source_component_from_package_list(source.package_list, suite)
1173 if component is not None:
1174 query = query.where(Override.component == component)
1176 return self.session.execute(query).scalar_one_or_none()
1178 def _binary_component(
1179 self, suite: Suite, binary: daklib.upload.Binary, only_overrides: bool = True
1180 ) -> Component | None:
1181 """get component for a binary
1183 By default this will only look at overrides to get the right component;
1184 if `only_overrides` is :const:`False` this method will also look at the
1185 Section field.
1187 :param only_overrides: only use overrides to get the right component
1188 """
1189 override = self._binary_override(suite, binary)
1190 if override is not None:
1191 return override.component
1192 if only_overrides: 1192 ↛ 1193line 1192 didn't jump to line 1193 because the condition on line 1192 was never true
1193 return None
1194 return get_mapped_component(binary.component, self.session)
1196 def _source_component(
1197 self, suite: Suite, source: daklib.upload.Source, only_overrides: bool = True
1198 ) -> Component | None:
1199 """get component for a source
1201 By default this will only look at overrides to get the right component;
1202 if `only_overrides` is :const:`False` this method will also look at the
1203 Section field.
1205 :param only_overrides: only use overrides to get the right component
1206 """
1207 override = self._source_override(suite, source)
1208 if override is not None: 1208 ↛ 1210line 1208 didn't jump to line 1210 because the condition on line 1208 was always true
1209 return override.component
1210 if only_overrides:
1211 return None
1212 return get_mapped_component(source.component, self.session)
1214 def _run_checks(
1215 self,
1216 force: bool,
1217 simple_checks: Iterable[type[checks.Check]],
1218 per_suite_checks: Collection[type[checks.Check]],
1219 suites: Collection[Suite],
1220 ) -> bool:
1221 try:
1222 for check in simple_checks:
1223 check().check(self)
1225 if per_suite_checks and not suites: 1225 ↛ 1226line 1225 didn't jump to line 1226 because the condition on line 1225 was never true
1226 raise ValueError(
1227 "Per-suite checks should be called, but no suites given."
1228 )
1229 for check in per_suite_checks:
1230 for suite in suites:
1231 check().per_suite_check(self, suite)
1232 except checks.Reject as e: 1232 ↛ 1235line 1232 didn't jump to line 1235
1233 self.reject_reasons.append(str(e))
1234 return False
1235 except Exception as e:
1236 self.reject_reasons.append(
1237 "Processing raised an exception: {0}.\n{1}".format(
1238 e, traceback.format_exc()
1239 )
1240 )
1241 return False
1243 return len(self.reject_reasons) == 0
1245 def _run_checks_very_early(self, force: bool) -> bool:
1246 """
1247 run very early checks
1249 These check validate signatures on .changes and hashes.
1250 """
1251 return self._run_checks(
1252 force=force,
1253 simple_checks=[
1254 checks.SignatureAndHashesCheck,
1255 checks.WeakSignatureCheck,
1256 checks.SignatureTimestampCheck,
1257 ],
1258 per_suite_checks=[],
1259 suites=[],
1260 )
1262 def _run_checks_early(self, force: bool) -> bool:
1263 """
1264 run early checks
1266 These are checks that run after checking signatures, but
1267 before deciding the target suite.
1269 This should cover archive-wide policies, sanity checks, ...
1270 """
1271 return self._run_checks(
1272 force=force,
1273 simple_checks=[
1274 checks.ChangesCheck,
1275 checks.ExternalHashesCheck,
1276 checks.SourceCheck,
1277 checks.BinaryCheck,
1278 checks.BinaryMembersCheck,
1279 checks.BinaryTimestampCheck,
1280 checks.SingleDistributionCheck,
1281 checks.ArchAllBinNMUCheck,
1282 ],
1283 per_suite_checks=[],
1284 suites=[],
1285 )
1287 def _run_checks_late(self, force: bool, suites: Collection[Suite]) -> bool:
1288 """
1289 run late checks
1291 These are checks that run after the target suites are known.
1293 This should cover permission checks, suite-specific polices
1294 (e.g., lintian), version constraints, ...
1295 """
1296 return self._run_checks(
1297 force=force,
1298 simple_checks=[
1299 checks.TransitionCheck,
1300 checks.ACLCheck,
1301 checks.NewOverrideCheck,
1302 checks.NoSourceOnlyCheck,
1303 checks.LintianCheck,
1304 ],
1305 per_suite_checks=[
1306 checks.SuiteCheck,
1307 checks.ACLCheck,
1308 checks.SourceFormatCheck,
1309 checks.SuiteArchitectureCheck,
1310 checks.VersionCheck,
1311 ],
1312 suites=suites,
1313 )
1315 def _handle_tag2upload(self) -> bool:
1316 """
1317 check if upload is via tag2upload
1319 if so, determine who authorized the upload to notify them of
1320 rejections and for ACL checks
1321 """
1323 if not (keyring := self.fingerprint.keyring) or not keyring.tag2upload:
1324 return True
1326 source = self.changes.source
1327 if not source: 1327 ↛ 1328line 1327 didn't jump to line 1328 because the condition on line 1327 was never true
1328 self.reject_reasons.append("tag2upload: upload missing source")
1329 return False
1331 try:
1332 tag2upload_file, info = get_tag2upload_info_for_upload(self)
1333 except Exception as e:
1334 self.reject_reasons.append(f"tag2upload: invalid metadata: {e}")
1335 return False
1336 self._extra_source_files.append(tag2upload_file)
1338 success = True
1340 if self.changes.binaries: 1340 ↛ 1341line 1340 didn't jump to line 1341 because the condition on line 1340 was never true
1341 success = False
1342 self.reject_reasons.append("tag2upload: upload includes binaries")
1343 if self.changes.byhand_files: 1343 ↛ 1344line 1343 didn't jump to line 1344 because the condition on line 1343 was never true
1344 success = False
1345 self.reject_reasons.append("tag2upload: upload included by-hand files")
1347 if not info.signed_file.valid: 1347 ↛ 1348line 1347 didn't jump to line 1348 because the condition on line 1347 was never true
1348 success = False
1349 self.reject_reasons.append("tag2upload: no valid signature on tag")
1350 else:
1351 # Only set with a valid signature, but also when we reject
1352 # the upload so the signer might get included in the
1353 # rejection mail.
1354 self.authorized_by_fingerprint = self.session.execute(
1355 select(Fingerprint).filter_by(
1356 fingerprint=info.signed_file.primary_fingerprint
1357 )
1358 ).scalar_one()
1359 if info.signed_file.weak_signature: 1359 ↛ 1360line 1359 didn't jump to line 1360 because the condition on line 1359 was never true
1360 success = False
1361 self.reject_reasons.append(
1362 "tag2upload: tag was signed using a weak algorithm (such as SHA-1)"
1363 )
1364 try:
1365 checks.check_signature_timestamp("tag2upload", info.signed_file)
1366 except checks.Reject as e:
1367 success = False
1368 self.reject_reasons.append(str(e))
1370 if info.metadata.get("distro") != "debian": 1370 ↛ 1371line 1370 didn't jump to line 1371 because the condition on line 1370 was never true
1371 success = False
1372 self.reject_reasons.append("tag2upload: upload not targeted at Debian.")
1373 if info.metadata.get("source") != source.dsc["Source"]: 1373 ↛ 1374line 1373 didn't jump to line 1374 because the condition on line 1373 was never true
1374 success = False
1375 self.reject_reasons.append(
1376 "tag2upload: source from tag metadata does not match upload"
1377 )
1378 if info.metadata.get("version") != source.dsc["Version"]: 1378 ↛ 1379line 1378 didn't jump to line 1379 because the condition on line 1378 was never true
1379 success = False
1380 self.reject_reasons.append(
1381 "tag2upload: version from tag metadata does not match upload"
1382 )
1384 tag_info_field = source.dsc.get("Git-Tag-Info")
1385 if not tag_info_field: 1385 ↛ 1386line 1385 didn't jump to line 1386 because the condition on line 1385 was never true
1386 success = False
1387 self.reject_reasons.append("tag2upload: source misses Git-Tag-Info field")
1388 else:
1389 try:
1390 tag_info = parse_git_tag_info(tag_info_field)
1391 except ValueError:
1392 success = False
1393 self.reject_reasons.append("tag2upload: could not parse Git-Tag-Info")
1394 else:
1395 if tag_info.fp.upper() != info.signed_file.fingerprint: 1395 ↛ 1396line 1395 didn't jump to line 1396 because the condition on line 1395 was never true
1396 success = False
1397 self.reject_reasons.append(
1398 "tag2upload: signing key from Git and Git-Tag-Info differ"
1399 )
1401 return success
1403 def check(self, force: bool = False) -> bool:
1404 """run checks against the upload
1406 :param force: ignore failing forcable checks
1407 :return: :const:`True` if all checks passed, :const:`False` otherwise
1408 """
1409 # XXX: needs to be better structured.
1410 assert self.changes.valid_signature
1412 # Validate signatures and hashes before we do any real work:
1413 if not self._run_checks_very_early(force):
1414 return False
1416 if not self._handle_tag2upload(): 1416 ↛ 1417line 1416 didn't jump to line 1417 because the condition on line 1416 was never true
1417 return False
1419 if not self._run_checks_early(force): 1419 ↛ 1420line 1419 didn't jump to line 1420 because the condition on line 1419 was never true
1420 return False
1422 try:
1423 final_suites = self._final_suites()
1424 except Exception as e:
1425 self.reject_reasons.append(
1426 "Processing raised an exception: {0}.\n{1}".format(
1427 e, traceback.format_exc()
1428 )
1429 )
1430 return False
1431 if len(final_suites) == 0:
1432 self.reject_reasons.append(
1433 "No target suite found. Please check your target distribution and that you uploaded to the right archive."
1434 )
1435 return False
1437 self.final_suites = final_suites
1439 if not self._run_checks_late(force, final_suites):
1440 return False
1442 if len(self.reject_reasons) != 0: 1442 ↛ 1443line 1442 didn't jump to line 1443 because the condition on line 1442 was never true
1443 return False
1445 self._checked = True
1446 return True
1448 def _install_to_suite(
1449 self,
1450 target_suite: Suite,
1451 suite: Suite,
1452 source_component_func: Callable[[daklib.upload.Source], Component],
1453 binary_component_func: Callable[[daklib.upload.Binary], Component],
1454 source_suites=None,
1455 extra_source_archives: Iterable[Archive] | None = None,
1456 policy_upload: bool = False,
1457 ) -> tuple[DBSource | None, list[DBBinary]]:
1458 """Install upload to the given suite
1460 :param target_suite: target suite (before redirection to policy queue or NEW)
1461 :param suite: suite to install the package into. This is the real suite,
1462 ie. after any redirection to NEW or a policy queue
1463 :param source_component_func: function to get the :class:`daklib.dbconn.Component`
1464 for a :class:`daklib.upload.Source` object
1465 :param binary_component_func: function to get the :class:`daklib.dbconn.Component`
1466 for a :class:`daklib.upload.Binary` object
1467 :param source_suites: see :meth:`daklib.archive.ArchiveTransaction.install_binary`
1468 :param extra_source_archives: see :meth:`daklib.archive.ArchiveTransaction.install_binary`
1469 :param policy_upload: Boolean indicating upload to policy queue (including NEW)
1470 :return: tuple with two elements. The first is a :class:`daklib.dbconn.DBSource`
1471 object for the install source or :const:`None` if no source was
1472 included. The second is a list of :class:`daklib.dbconn.DBBinary`
1473 objects for the installed binary packages.
1474 """
1475 # XXX: move this function to ArchiveTransaction?
1477 control = self.changes.changes
1478 changed_by = get_or_set_maintainer(
1479 control.get("Changed-By", control["Maintainer"]), self.session
1480 )
1482 if source_suites is None: 1482 ↛ 1483line 1482 didn't jump to line 1483
1483 source_suites = (
1484 select(Suite)
1485 .join(VersionCheck, VersionCheck.reference_id == Suite.suite_id)
1486 .where(VersionCheck.check == "Enhances")
1487 .where(VersionCheck.suite == suite)
1488 .subquery()
1489 )
1491 source = self.changes.source
1492 if source is not None:
1493 component = source_component_func(source)
1494 db_source = self.transaction.install_source(
1495 self.directory,
1496 source,
1497 suite,
1498 component,
1499 changed_by,
1500 fingerprint=self.fingerprint,
1501 authorized_by_fingerprint=self.authorized_by_fingerprint,
1502 extra_source_files=self._extra_source_files,
1503 )
1504 else:
1505 db_source = None
1507 db_binaries = []
1508 for binary in sorted(self.changes.binaries, key=lambda x: x.name):
1509 copy_to_suite = suite
1510 if (
1511 daklib.utils.is_in_debug_section(binary.control)
1512 and suite.debug_suite is not None
1513 ):
1514 copy_to_suite = suite.debug_suite
1516 component = binary_component_func(binary)
1517 db_binary = self.transaction.install_binary(
1518 self.directory,
1519 binary,
1520 copy_to_suite,
1521 component,
1522 fingerprint=self.fingerprint,
1523 authorized_by_fingerprint=self.authorized_by_fingerprint,
1524 source_suites=source_suites,
1525 extra_source_archives=extra_source_archives,
1526 )
1527 db_binaries.append(db_binary)
1529 if not policy_upload:
1530 check_upload_for_external_signature_request(
1531 self.session, target_suite, copy_to_suite, db_binary
1532 )
1534 if suite.copychanges: 1534 ↛ 1535line 1534 didn't jump to line 1535 because the condition on line 1534 was never true
1535 src = os.path.join(self.directory, self.changes.filename)
1536 dst = os.path.join(
1537 suite.archive.path, "dists", suite.suite_name, self.changes.filename
1538 )
1539 self.transaction.fs.copy(src, dst, mode=suite.archive.mode)
1541 suite.update_last_changed()
1543 return (db_source, db_binaries)
1545 def _install_changes(self) -> DBChange:
1546 assert self.changes.valid_signature
1547 control = self.changes.changes
1548 session = self.transaction.session
1550 changelog_id = None
1551 # Only add changelog for sourceful uploads and binNMUs
1552 if self.changes.sourceful or re_bin_only_nmu.search(control["Version"]):
1553 query = "INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id"
1554 changelog_id = session.execute(
1555 sql.text(query), {"changelog": control["Changes"]}
1556 ).scalar()
1557 assert changelog_id is not None
1559 db_changes = DBChange()
1560 db_changes.changesname = self.changes.filename
1561 db_changes.source = control["Source"]
1562 db_changes.binaries = control.get("Binary", None)
1563 db_changes.architecture = control["Architecture"]
1564 db_changes.version = control["Version"]
1565 db_changes.distribution = control["Distribution"]
1566 db_changes.urgency = control["Urgency"]
1567 db_changes.maintainer = control["Maintainer"]
1568 db_changes.changedby = control.get("Changed-By", control["Maintainer"])
1569 db_changes.date = control["Date"]
1570 db_changes.fingerprint = self.fingerprint.fingerprint
1571 db_changes.authorized_by_fingerprint = (
1572 self.authorized_by_fingerprint.fingerprint
1573 )
1574 db_changes.changelog_id = changelog_id
1575 db_changes.closes = self.changes.closed_bugs
1577 try:
1578 self.transaction.session.add(db_changes)
1579 self.transaction.session.flush()
1580 except sqlalchemy.exc.IntegrityError:
1581 raise ArchiveException(
1582 "{0} is already known.".format(self.changes.filename)
1583 )
1585 return db_changes
1587 def _install_policy(
1588 self, policy_queue, target_suite, db_changes, db_source, db_binaries
1589 ) -> PolicyQueueUpload:
1590 """install upload to policy queue"""
1591 u = PolicyQueueUpload()
1592 u.policy_queue = policy_queue
1593 u.target_suite = target_suite
1594 u.changes = db_changes
1595 u.source = db_source
1596 u.binaries = db_binaries
1597 self.transaction.session.add(u)
1598 self.transaction.session.flush()
1600 queue_files = [self.changes.filename]
1601 queue_files.extend(f.filename for f in self.changes.buildinfo_files)
1602 for fn in queue_files:
1603 src = os.path.join(self.changes.directory, fn)
1604 dst = os.path.join(policy_queue.path, fn)
1605 self.transaction.fs.copy(src, dst, mode=policy_queue.change_perms)
1607 return u
1609 def try_autobyhand(self) -> bool:
1610 """Try AUTOBYHAND
1612 Try to handle byhand packages automatically.
1613 """
1614 assert len(self.reject_reasons) == 0
1615 assert self.changes.valid_signature
1616 assert self.final_suites is not None
1617 assert self._checked
1619 byhand = self.changes.byhand_files
1620 if len(byhand) == 0: 1620 ↛ 1623line 1620 didn't jump to line 1623 because the condition on line 1620 was always true
1621 return True
1623 suites = list(self.final_suites)
1624 assert len(suites) == 1, "BYHAND uploads must be to a single suite"
1625 suite = suites[0]
1627 cnf = Config()
1628 control = self.changes.changes
1629 automatic_byhand_packages = cnf.subtree("AutomaticByHandPackages")
1631 remaining = []
1632 for f in byhand:
1633 if "_" in f.filename:
1634 parts = f.filename.split("_", 2)
1635 if len(parts) != 3:
1636 print(
1637 "W: unexpected byhand filename {0}. No automatic processing.".format(
1638 f.filename
1639 )
1640 )
1641 remaining.append(f)
1642 continue
1644 package, _, archext = parts
1645 arch, ext = archext.split(".", 1)
1646 else:
1647 parts = f.filename.split(".")
1648 if len(parts) < 2:
1649 print(
1650 "W: unexpected byhand filename {0}. No automatic processing.".format(
1651 f.filename
1652 )
1653 )
1654 remaining.append(f)
1655 continue
1657 package = parts[0]
1658 arch = "all"
1659 ext = parts[-1]
1661 try:
1662 rule = automatic_byhand_packages.subtree(package)
1663 except KeyError:
1664 remaining.append(f)
1665 continue
1667 if (
1668 rule["Source"] != self.changes.source_name
1669 or rule["Section"] != f.section
1670 or ("Extension" in rule and rule["Extension"] != ext)
1671 ):
1672 remaining.append(f)
1673 continue
1675 script = rule["Script"]
1676 retcode = subprocess.call(
1677 [
1678 script,
1679 os.path.join(self.directory, f.filename),
1680 control["Version"],
1681 arch,
1682 os.path.join(self.directory, self.changes.filename),
1683 suite.suite_name,
1684 ],
1685 shell=False,
1686 )
1687 if retcode != 0:
1688 print("W: error processing {0}.".format(f.filename))
1689 remaining.append(f)
1691 return len(remaining) == 0
1693 def _install_byhand(
1694 self,
1695 policy_queue_upload: PolicyQueueUpload,
1696 hashed_file: daklib.upload.HashedFile,
1697 ) -> PolicyQueueByhandFile:
1698 """install byhand file"""
1699 fs = self.transaction.fs
1700 session = self.transaction.session
1701 policy_queue = policy_queue_upload.policy_queue
1703 byhand_file = PolicyQueueByhandFile()
1704 byhand_file.upload = policy_queue_upload
1705 byhand_file.filename = hashed_file.filename
1706 session.add(byhand_file)
1707 session.flush()
1709 src = os.path.join(self.directory, hashed_file.filename)
1710 dst = os.path.join(policy_queue.path, hashed_file.filename)
1711 fs.copy(src, dst, mode=policy_queue.change_perms)
1713 return byhand_file
1715 def _do_bts_versiontracking(self) -> None:
1716 cnf = Config()
1717 fs = self.transaction.fs
1719 btsdir = cnf.get("Dir::BTSVersionTrack")
1720 if btsdir is None or btsdir == "": 1720 ↛ 1723line 1720 didn't jump to line 1723 because the condition on line 1720 was always true
1721 return
1723 base = os.path.join(btsdir, self.changes.filename[:-8])
1725 # version history
1726 sourcedir = self.unpacked_source()
1727 if sourcedir is not None:
1728 dch_path = os.path.join(sourcedir, "debian", "changelog")
1729 with open(dch_path, "r") as fh:
1730 versions = fs.create("{0}.versions".format(base), mode=0o644)
1731 for line in fh:
1732 if re_changelog_versions.match(line):
1733 versions.write(line)
1734 versions.close()
1736 # binary -> source mapping
1737 if self.changes.binaries:
1738 debinfo = fs.create("{0}.debinfo".format(base), mode=0o644)
1739 for binary in self.changes.binaries:
1740 control = binary.control
1741 source_package, source_version = binary.source
1742 line = " ".join(
1743 [
1744 control["Package"],
1745 control["Version"],
1746 control["Architecture"],
1747 source_package,
1748 source_version,
1749 ]
1750 )
1751 print(line, file=debinfo)
1752 debinfo.close()
1754 def _policy_queue(self, suite) -> PolicyQueue | None:
1755 if suite.policy_queue is not None:
1756 return suite.policy_queue
1757 return None
1759 def install(self) -> None:
1760 """install upload
1762 Install upload to a suite or policy queue. This method does **not**
1763 handle uploads to NEW.
1765 You need to have called the :meth:`check` method before calling this method.
1766 """
1767 assert len(self.reject_reasons) == 0
1768 assert self.changes.valid_signature
1769 assert self.final_suites is not None
1770 assert self._checked
1771 assert not self.new
1773 db_changes = self._install_changes()
1775 for suite in self.final_suites:
1776 overridesuite = suite
1777 if suite.overridesuite is not None:
1778 overridesuite = get_suite_by_name(suite.overridesuite, self.session)
1780 policy_queue = self._policy_queue(suite)
1781 policy_upload = False
1783 redirected_suite = suite
1784 if policy_queue is not None:
1785 redirected_suite = policy_queue.suite
1786 policy_upload = True
1788 # source can be in the suite we install to or any suite we enhance
1789 source_suite_ids = {suite.suite_id, redirected_suite.suite_id}
1790 for (enhanced_suite_id,) in self.session.execute(
1791 select(VersionCheck.reference_id)
1792 .where(VersionCheck.suite_id.in_(source_suite_ids))
1793 .where(VersionCheck.check == "Enhances")
1794 ):
1795 source_suite_ids.add(enhanced_suite_id)
1797 source_suites = (
1798 select(Suite).where(Suite.suite_id.in_(source_suite_ids)).subquery()
1799 )
1801 def source_component_func(source: daklib.upload.Source) -> Component:
1802 component = self._source_component(
1803 overridesuite, source, only_overrides=False
1804 )
1805 assert component is not None
1806 return component
1808 def binary_component_func(binary: daklib.upload.Binary) -> Component:
1809 component = self._binary_component(
1810 overridesuite, binary, only_overrides=False
1811 )
1812 assert component is not None
1813 return component
1815 (db_source, db_binaries) = self._install_to_suite(
1816 suite,
1817 redirected_suite,
1818 source_component_func,
1819 binary_component_func,
1820 source_suites=source_suites,
1821 extra_source_archives=[suite.archive],
1822 policy_upload=policy_upload,
1823 )
1825 if policy_queue is not None:
1826 self._install_policy(
1827 policy_queue, suite, db_changes, db_source, db_binaries
1828 )
1830 # copy to build queues
1831 if policy_queue is None or policy_queue.send_to_build_queues: 1831 ↛ 1775line 1831 didn't jump to line 1775 because the condition on line 1831 was always true
1832 for build_queue in suite.copy_queues:
1833 self._install_to_suite(
1834 suite,
1835 build_queue.suite,
1836 source_component_func,
1837 binary_component_func,
1838 source_suites=source_suites,
1839 extra_source_archives=[suite.archive],
1840 )
1842 self._do_bts_versiontracking()
1844 def install_to_new(self) -> None:
1845 """install upload to NEW
1847 Install upload to NEW. This method does **not** handle regular uploads
1848 to suites or policy queues.
1850 You need to have called the :meth:`check` method before calling this method.
1851 """
1852 # Uploads to NEW are special as we don't have overrides.
1853 assert len(self.reject_reasons) == 0
1854 assert self.changes.valid_signature
1855 assert self.final_suites is not None
1857 binaries = self.changes.binaries
1858 byhand = self.changes.byhand_files
1860 # we need a suite to guess components
1861 suites = list(self.final_suites)
1862 assert len(suites) == 1, "NEW uploads must be to a single suite"
1863 suite = suites[0]
1865 # decide which NEW queue to use
1866 if suite.new_queue is None: 1866 ↛ 1871line 1866 didn't jump to line 1871 because the condition on line 1866 was always true
1867 new_queue = self.transaction.session.execute(
1868 select(PolicyQueue).filter_by(queue_name="new")
1869 ).scalar_one()
1870 else:
1871 new_queue = suite.new_queue
1872 if len(byhand) > 0: 1872 ↛ 1874line 1872 didn't jump to line 1874 because the condition on line 1872 was never true
1873 # There is only one global BYHAND queue
1874 new_queue = self.transaction.session.execute(
1875 select(PolicyQueue).filter_by(queue_name="byhand")
1876 ).scalar_one()
1877 new_suite = new_queue.suite
1879 def binary_component_func(binary: daklib.upload.Binary) -> Component:
1880 component = self._binary_component(suite, binary, only_overrides=False)
1881 assert component is not None
1882 return component
1884 # guess source component
1885 # XXX: should be moved into an extra method
1886 binary_component_names = set()
1887 for binary in binaries:
1888 component = binary_component_func(binary)
1889 binary_component_names.add(component.component_name)
1890 source_component_name = None
1891 for c in self.session.scalars(
1892 select(Component).order_by(Component.component_id)
1893 ):
1894 guess = c.component_name
1895 if guess in binary_component_names:
1896 source_component_name = guess
1897 break
1898 if source_component_name is None:
1899 source_component = self.session.scalars(
1900 select(Component).order_by(Component.component_id).limit(1)
1901 ).first()
1902 else:
1903 source_component = self.session.execute(
1904 select(Component).filter_by(component_name=source_component_name)
1905 ).scalar_one()
1906 assert source_component is not None
1908 def source_component_func(source: daklib.upload.Source) -> Component:
1909 return source_component
1911 db_changes = self._install_changes()
1912 (db_source, db_binaries) = self._install_to_suite(
1913 suite,
1914 new_suite,
1915 source_component_func,
1916 binary_component_func,
1917 source_suites=True,
1918 extra_source_archives=[suite.archive],
1919 policy_upload=True,
1920 )
1921 policy_upload = self._install_policy(
1922 new_queue, suite, db_changes, db_source, db_binaries
1923 )
1925 for f in byhand: 1925 ↛ 1926line 1925 didn't jump to line 1926 because the loop on line 1925 never started
1926 self._install_byhand(policy_upload, f)
1928 self._do_bts_versiontracking()
1930 def commit(self) -> None:
1931 """commit changes"""
1932 self.transaction.commit()
1934 def rollback(self) -> None:
1935 """rollback changes"""
1936 self.transaction.rollback()
1938 def __enter__(self):
1939 self.prepare()
1940 return self
1942 def __exit__(self, type, value, traceback):
1943 if self._directory is not None: 1943 ↛ 1946line 1943 didn't jump to line 1946 because the condition on line 1943 was always true
1944 shutil.rmtree(self._directory)
1945 self._directory = None
1946 self._changes = None
1947 self.transaction.rollback()