1""" DB access class
3@contact: Debian FTPMaster <ftpmaster@debian.org>
4@copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
5@copyright: 2008-2009 Mark Hymers <mhy@debian.org>
6@copyright: 2009, 2010 Joerg Jaspert <joerg@debian.org>
7@copyright: 2009 Mike O'Connor <stew@debian.org>
8@license: GNU General Public License version 2 or later
9"""
11# This program is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation; either version 2 of the License, or
14# (at your option) any later version.
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
21# You should have received a copy of the GNU General Public License
22# along with this program; if not, write to the Free Software
23# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25################################################################################
27# < mhy> I need a funny comment
28# < sgran> two peanuts were walking down a dark street
29# < sgran> one was a-salted
30# * mhy looks up the definition of "funny"
32################################################################################
34import apt_pkg
35import functools
36import inspect
37import os
38from os.path import normpath
39import re
40import subprocess
41import warnings
42from collections.abc import Iterable
43from typing import Optional, TYPE_CHECKING, Union
45from debian.debfile import Deb822
46from tarfile import TarFile
48import sqlalchemy
49from sqlalchemy import create_engine, Table, desc
50from sqlalchemy.orm import sessionmaker, mapper, relation, object_session, \
51 backref, object_mapper
52import sqlalchemy.types
53from sqlalchemy.orm.collections import attribute_mapped_collection
54from sqlalchemy.ext.associationproxy import association_proxy
56# Don't remove this, we re-export the exceptions to scripts which import us
57from sqlalchemy.exc import *
58from sqlalchemy.orm.exc import NoResultFound
60import daklib.gpg
61from .aptversion import AptVersion
62# Only import Config until Queue stuff is changed to store its config
63# in the database
64from .config import Config
65from .textutils import fix_maintainer
67# suppress some deprecation warnings in squeeze related to sqlalchemy
68warnings.filterwarnings('ignore',
69 "Predicate of partial index .* ignored during reflection",
70 SAWarning)
72# (Debian 12 "bookworm") Silence warning targeted at SQLAlchemy dialect maintainers
73warnings.filterwarnings(
74 "ignore",
75 "Dialect postgresql:psycopg2 will not make use of SQL compilation caching.*",
76 SAWarning)
78from .database.base import Base
80if TYPE_CHECKING: 80 ↛ 81line 80 didn't jump to line 81, because the condition on line 80 was never true
81 import sqlalchemy.orm.query
84################################################################################
86# Patch in support for the debversion field type so that it works during
87# reflection
89class DebVersion(sqlalchemy.types.UserDefinedType):
90 def get_col_spec(self):
91 return "DEBVERSION"
93 def bind_processor(self, dialect):
94 return None
96 def result_processor(self, dialect, coltype):
97 return None
100from sqlalchemy.databases import postgresql
101postgresql.ischema_names['debversion'] = DebVersion
103################################################################################
105__all__ = ['IntegrityError', 'SQLAlchemyError', 'DebVersion']
107################################################################################
110def session_wrapper(fn):
111 """
112 Wrapper around common ".., session=None):" handling. If the wrapped
113 function is called without passing 'session', we create a local one
114 and destroy it when the function ends.
116 Also attaches a commit_or_flush method to the session; if we created a
117 local session, this is a synonym for session.commit(), otherwise it is a
118 synonym for session.flush().
119 """
121 @functools.wraps(fn)
122 def wrapped(*args, **kwargs):
123 private_transaction = False
125 # Find the session object
126 session = kwargs.get('session')
128 if session is None:
129 if len(args) < len(inspect.getfullargspec(fn).args):
130 # No session specified as last argument or in kwargs
131 private_transaction = True
132 session = kwargs['session'] = DBConn().session()
133 else:
134 # Session is last argument in args
135 session = args[-1]
136 if session is None: 136 ↛ 137line 136 didn't jump to line 137, because the condition on line 136 was never true
137 args = list(args)
138 session = args[-1] = DBConn().session()
139 private_transaction = True
141 if private_transaction:
142 session.commit_or_flush = session.commit
143 else:
144 session.commit_or_flush = session.flush
146 try:
147 return fn(*args, **kwargs)
148 finally:
149 if private_transaction:
150 # We created a session; close it.
151 session.close()
153 return wrapped
156__all__.append('session_wrapper')
158################################################################################
161class ORMObject:
162 """
163 ORMObject is a base class for all ORM classes mapped by SQLalchemy. All
164 derived classes must implement the properties() method.
165 """
167 def properties(self) -> list[str]:
168 '''
169 This method should be implemented by all derived classes and returns a
170 list of the important properties. The properties 'created' and
171 'modified' will be added automatically. A suffix '_count' should be
172 added to properties that are lists or query objects. The most important
173 property name should be returned as the first element in the list
174 because it is used by repr().
175 '''
176 return []
178 def classname(self) -> str:
179 '''
180 Returns the name of the class.
181 '''
182 return type(self).__name__
184 def __repr__(self):
185 '''
186 Returns a short string representation of the object using the first
187 element from the properties() method.
188 '''
189 primary_property = self.properties()[0]
190 value = getattr(self, primary_property)
191 return '<%s %s>' % (self.classname(), str(value))
193 def __str__(self):
194 '''
195 Returns a human readable form of the object using the properties()
196 method.
197 '''
198 return '<%s(...)>' % (self.classname())
200 @classmethod
201 @session_wrapper
202 def get(cls, primary_key, session=None):
203 '''
204 This is a support function that allows getting an object by its primary
205 key.
207 Architecture.get(3[, session])
209 instead of the more verbose
211 session.query(Architecture).get(3)
212 '''
213 return session.query(cls).get(primary_key)
215 def session(self):
216 '''
217 Returns the current session that is associated with the object. May
218 return None is object is in detached state.
219 '''
221 return object_session(self)
224__all__.append('ORMObject')
226################################################################################
229class ACL(ORMObject):
230 def __repr__(self):
231 return "<ACL {0}>".format(self.name)
234__all__.append('ACL')
237class ACLPerSource(ORMObject):
238 def __repr__(self):
239 return "<ACLPerSource acl={0} fingerprint={1} source={2} reason={3}>".format(self.acl.name, self.fingerprint.fingerprint, self.source, self.reason)
242__all__.append('ACLPerSource')
245class ACLPerSuite(ORMObject):
246 def __repr__(self):
247 return "<ACLPerSuite acl={0} fingerprint={1} suite={2} reason={3}>".format(self.acl.name, self.fingerprint.fingerprint, self.suite.suite_name, self.reason)
250__all__.append('ACLPerSuite')
252################################################################################
255from .database.architecture import Architecture
257__all__.append('Architecture')
260@session_wrapper
261def get_architecture(architecture: str, session=None) -> Optional[Architecture]:
262 """
263 Returns database id for given `architecture`.
265 :param architecture: The name of the architecture
266 :param session: Optional SQLA session object (a temporary one will be
267 generated if not supplied)
268 :return: Architecture object for the given arch (None if not present)
269 """
271 q = session.query(Architecture).filter_by(arch_string=architecture)
272 return q.one_or_none()
275__all__.append('get_architecture')
277################################################################################
280class Archive:
281 def __init__(self, *args, **kwargs):
282 pass
284 def __repr__(self):
285 return '<Archive %s>' % self.archive_name
288__all__.append('Archive')
291@session_wrapper
292def get_archive(archive: str, session=None) -> Optional[Archive]:
293 """
294 returns database id for given `archive`.
296 :param archive: the name of the arhive
297 :param session: Optional SQLA session object (a temporary one will be
298 generated if not supplied)
299 :return: Archive object for the given name (None if not present)
300 """
301 archive = archive.lower()
303 q = session.query(Archive).filter_by(archive_name=archive)
304 return q.one_or_none()
307__all__.append('get_archive')
309################################################################################
312class ArchiveFile:
313 def __init__(self, archive=None, component=None, file=None):
314 self.archive = archive
315 self.component = component
316 self.file = file
318 @property
319 def path(self):
320 return os.path.join(self.archive.path, 'pool', self.component.component_name, self.file.filename)
323__all__.append('ArchiveFile')
325################################################################################
328class BinContents(ORMObject):
329 def __init__(self, file=None, binary=None):
330 self.file = file
331 self.binary = binary
333 def properties(self) -> list[str]:
334 return ['file', 'binary']
337__all__.append('BinContents')
339################################################################################
342class DBBinary(ORMObject):
343 def __init__(self, package=None, source=None, version=None,
344 maintainer=None, architecture=None, poolfile=None,
345 binarytype='deb', fingerprint=None):
346 self.package = package
347 self.source = source
348 self.version = version
349 self.maintainer = maintainer
350 self.architecture = architecture
351 self.poolfile = poolfile
352 self.binarytype = binarytype
353 self.fingerprint = fingerprint
355 @property
356 def pkid(self) -> int:
357 return self.binary_id
359 @property
360 def name(self) -> str:
361 return self.package
363 @property
364 def arch_string(self) -> str:
365 return "%s" % self.architecture
367 def properties(self) -> list[str]:
368 return ['package', 'version', 'maintainer', 'source', 'architecture',
369 'poolfile', 'binarytype', 'fingerprint', 'install_date',
370 'suites_count', 'binary_id', 'contents_count', 'extra_sources']
372 metadata = association_proxy('key', 'value')
374 def scan_contents(self) -> Iterable[str]:
375 '''
376 Yields the contents of the package. Only regular files are yielded and
377 the path names are normalized after converting them from either utf-8
378 or iso8859-1 encoding. It yields the string ' <EMPTY PACKAGE>' if the
379 package does not contain any regular file.
380 '''
381 fullpath = self.poolfile.fullpath
382 dpkg_cmd = ('dpkg-deb', '--fsys-tarfile', fullpath)
383 dpkg = subprocess.Popen(dpkg_cmd, stdout=subprocess.PIPE)
384 tar = TarFile.open(fileobj=dpkg.stdout, mode='r|')
385 for member in tar.getmembers():
386 if not member.isdir():
387 name = normpath(member.name)
388 yield name
389 tar.close()
390 dpkg.stdout.close()
391 dpkg.wait()
393 def read_control(self) -> bytes:
394 '''
395 Reads the control information from a binary.
397 :return: stanza text of the control section.
398 '''
399 from . import utils
400 fullpath = self.poolfile.fullpath
401 return utils.deb_extract_control(fullpath)
403 def read_control_fields(self) -> apt_pkg.TagSection:
404 '''
405 Reads the control information from a binary and return
406 as a dictionary.
408 :return: fields of the control section as a dictionary.
409 '''
410 stanza = self.read_control()
411 return apt_pkg.TagSection(stanza)
413 @property
414 def proxy(self) -> "MetadataProxy":
415 session = object_session(self)
416 query = session.query(BinaryMetadata).filter_by(binary=self)
417 return MetadataProxy(session, query)
420__all__.append('DBBinary')
423@session_wrapper
424def get_suites_binary_in(package: str, session=None) -> 'list[Suite]':
425 """
426 Returns list of Suite objects which given `package` name is in
428 :param package: DBBinary package name to search for
429 :return: list of Suite objects for the given package
430 """
432 return session.query(Suite).filter(Suite.binaries.any(DBBinary.package == package)).all()
435__all__.append('get_suites_binary_in')
438@session_wrapper
439def get_component_by_package_suite(package: str, suite_list: list[str], arch_list: Optional[str] = None, session=None) -> Optional[str]:
440 '''
441 Returns the component name of the newest binary package in suite_list or
442 None if no package is found. The result can be optionally filtered by a list
443 of architecture names.
445 :param package: DBBinary package name to search for
446 :param suite_list: list of suite_name items
447 :param arch_list: optional list of arch_string items that defaults to []
448 :return: name of component or None
449 '''
451 q = session.query(DBBinary).filter_by(package=package). \
452 join(DBBinary.suites).filter(Suite.suite_name.in_(suite_list))
453 if arch_list:
454 q = q.join(DBBinary.architecture). \
455 filter(Architecture.arch_string.in_(arch_list))
456 binary = q.order_by(desc(DBBinary.version)).first()
457 if binary is None:
458 return None
459 else:
460 return binary.poolfile.component.component_name
463__all__.append('get_component_by_package_suite')
465################################################################################
468class BuildQueue:
469 def __init__(self, *args, **kwargs):
470 pass
472 def __repr__(self):
473 return '<BuildQueue %s>' % self.queue_name
476__all__.append('BuildQueue')
478################################################################################
481class Component(ORMObject):
482 def __init__(self, component_name=None):
483 self.component_name = component_name
485 def __eq__(self, val):
486 if isinstance(val, str): 486 ↛ 487line 486 didn't jump to line 487, because the condition on line 486 was never true
487 warnings.warn("comparison with a `str` is deprecated", DeprecationWarning, stacklevel=2)
488 return (self.component_name == val)
489 # This signals to use the normal comparison operator
490 return NotImplemented
492 def __ne__(self, val):
493 if isinstance(val, str):
494 warnings.warn("comparison with a `str` is deprecated", DeprecationWarning, stacklevel=2)
495 return (self.component_name != val)
496 # This signals to use the normal comparison operator
497 return NotImplemented
499 __hash__ = ORMObject.__hash__
501 def properties(self) -> list[str]:
502 return ['component_name', 'component_id', 'description',
503 'meets_dfsg', 'overrides_count']
506__all__.append('Component')
509@session_wrapper
510def get_component(component: str, session=None) -> Optional[Component]:
511 """
512 Returns database id for given `component`.
514 :param component: The name of the override type
515 :return: the database id for the given component
516 """
517 component = component.lower()
519 q = session.query(Component).filter_by(component_name=component)
521 return q.one_or_none()
524__all__.append('get_component')
527def get_mapped_component_name(component_name):
528 cnf = Config()
529 for m in cnf.value_list("ComponentMappings"): 529 ↛ 530line 529 didn't jump to line 530, because the loop on line 529 never started
530 (src, dst) = m.split()
531 if component_name == src:
532 component_name = dst
533 return component_name
536__all__.append('get_mapped_component_name')
539@session_wrapper
540def get_mapped_component(component_name: str, session=None) -> Optional[Component]:
541 """get component after mappings
543 Evaluate component mappings from ComponentMappings in dak.conf for the
544 given component name.
546 .. todo::
548 ansgar wants to get rid of this. It's currently only used for
549 the security archive
551 :param component_name: component name
552 :param session: database session
553 :return: component after applying maps or :const:`None`
554 """
555 component_name = get_mapped_component_name(component_name)
556 component = session.query(Component).filter_by(component_name=component_name).first()
557 return component
560__all__.append('get_mapped_component')
563@session_wrapper
564def get_component_names(session=None) -> list[str]:
565 """
566 Returns list of strings of component names.
568 :return: list of strings of component names
569 """
571 return [x.component_name for x in session.query(Component).all()]
574__all__.append('get_component_names')
576################################################################################
579class DBConfig:
580 def __init__(self, *args, **kwargs):
581 pass
583 def __repr__(self):
584 return '<DBConfig %s>' % self.name
587__all__.append('DBConfig')
589################################################################################
592class DSCFile:
593 def __init__(self, *args, **kwargs):
594 pass
596 def __repr__(self):
597 return '<DSCFile %s>' % self.dscfile_id
600__all__.append('DSCFile')
603@session_wrapper
604def get_dscfiles(
605 dscfile_id: Optional[int] = None,
606 source_id: Optional[int] = None,
607 poolfile_id: Optional[int] = None,
608 session=None
609) -> list[DSCFile]:
610 """
611 Returns a list of DSCFiles which may be empty
613 :param dscfile_id: the dscfile_id of the DSCFiles to find
614 :param source_id: the source id related to the DSCFiles to find
615 :param poolfile_id: the poolfile id related to the DSCFiles to find
616 :return: Possibly empty list of DSCFiles
617 """
619 q = session.query(DSCFile)
621 if dscfile_id is not None:
622 q = q.filter_by(dscfile_id=dscfile_id)
624 if source_id is not None:
625 q = q.filter_by(source_id=source_id)
627 if poolfile_id is not None:
628 q = q.filter_by(poolfile_id=poolfile_id)
630 return q.all()
633__all__.append('get_dscfiles')
635################################################################################
638class ExternalOverride(ORMObject):
639 def __init__(self, *args, **kwargs):
640 pass
642 def __repr__(self):
643 return '<ExternalOverride %s = %s: %s>' % (self.package, self.key, self.value)
646__all__.append('ExternalOverride')
648################################################################################
651class PoolFile(ORMObject):
652 def __init__(self, filename=None, filesize=-1,
653 md5sum=None):
654 self.filename = filename
655 self.filesize = filesize
656 self.md5sum = md5sum
658 @property
659 def fullpath(self) -> str:
660 session = DBConn().session().object_session(self)
661 af = session.query(ArchiveFile).join(Archive) \
662 .filter(ArchiveFile.file == self) \
663 .order_by(Archive.tainted.desc()).first()
664 return af.path
666 @property
667 def component(self) -> Component:
668 session = DBConn().session().object_session(self)
669 component_id = session.query(ArchiveFile.component_id).filter(ArchiveFile.file == self) \
670 .group_by(ArchiveFile.component_id).one()
671 return session.query(Component).get(component_id)
673 @property
674 def basename(self) -> str:
675 return os.path.basename(self.filename)
677 def properties(self) -> list[str]:
678 return ['filename', 'file_id', 'filesize', 'md5sum', 'sha1sum',
679 'sha256sum', 'source', 'binary', 'last_used']
682__all__.append('PoolFile')
684################################################################################
687class Fingerprint(ORMObject):
688 def __init__(self, fingerprint=None):
689 self.fingerprint = fingerprint
691 def properties(self) -> list[str]:
692 return ['fingerprint', 'fingerprint_id', 'keyring', 'uid',
693 'binary_reject']
696__all__.append('Fingerprint')
699@session_wrapper
700def get_fingerprint(fpr: str, session=None) -> Optional[Fingerprint]:
701 """
702 Returns Fingerprint object for given fpr.
704 :param fpr: The fpr to find / add
705 :param session: Optional SQL session object (a temporary one will be
706 generated if not supplied).
707 :return: the Fingerprint object for the given fpr or None
708 """
710 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
711 return q.one_or_none()
714__all__.append('get_fingerprint')
717@session_wrapper
718def get_or_set_fingerprint(fpr: str, session=None) -> Fingerprint:
719 """
720 Returns Fingerprint object for given fpr.
722 If no matching fpr is found, a row is inserted.
724 :param fpr: The fpr to find / add
725 :param session: Optional SQL session object (a temporary one will be
726 generated if not supplied). If not passed, a commit will be performed at
727 the end of the function, otherwise the caller is responsible for commiting.
728 A flush will be performed either way.
729 :return: the Fingerprint object for the given fpr
730 """
732 q = session.query(Fingerprint).filter_by(fingerprint=fpr)
734 try:
735 ret = q.one()
736 except NoResultFound:
737 fingerprint = Fingerprint()
738 fingerprint.fingerprint = fpr
739 session.add(fingerprint)
740 session.commit_or_flush()
741 ret = fingerprint
743 return ret
746__all__.append('get_or_set_fingerprint')
748################################################################################
750# Helper routine for Keyring class
753def get_ldap_name(entry) -> str:
754 name = []
755 for k in ["cn", "mn", "sn"]:
756 ret = entry.get(k)
757 if not ret:
758 continue
759 value = ret[0].decode()
760 if value and value[0] != "-":
761 name.append(value)
762 return " ".join(name)
764################################################################################
767class Keyring:
768 keys = {}
769 fpr_lookup: dict[str, str] = {}
771 def __init__(self, *args, **kwargs):
772 pass
774 def __repr__(self):
775 return '<Keyring %s>' % self.keyring_name
777 def de_escape_gpg_str(self, txt: str) -> str:
778 esclist = re.split(r'(\\x..)', txt)
779 for x in range(1, len(esclist), 2): 779 ↛ 780line 779 didn't jump to line 780, because the loop on line 779 never started
780 esclist[x] = "%c" % (int(esclist[x][2:], 16))
781 return "".join(esclist)
783 def parse_address(self, uid: str) -> tuple[str, str]:
784 """parses uid and returns a tuple of real name and email address"""
785 import email.utils
786 (name, address) = email.utils.parseaddr(uid)
787 name = re.sub(r"\s*[(].*[)]", "", name)
788 name = self.de_escape_gpg_str(name)
789 if name == "":
790 name = uid
791 return (name, address)
793 def load_keys(self, keyring: str) -> None:
794 if not self.keyring_id: 794 ↛ 795line 794 didn't jump to line 795, because the condition on line 794 was never true
795 raise Exception('Must be initialized with database information')
797 cmd = ["gpg", "--no-default-keyring", "--keyring", keyring,
798 "--with-colons", "--fingerprint", "--fingerprint"]
799 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
801 key = None
802 need_fingerprint = False
804 for line_raw in p.stdout:
805 try:
806 line = line_raw.decode()
807 except UnicodeDecodeError:
808 # Some old UIDs might not use UTF-8 encoding. We assume they
809 # use latin1.
810 line = line_raw.decode('latin1')
811 field = line.split(":")
812 if field[0] == "pub":
813 key = field[4]
814 self.keys[key] = {}
815 (name, addr) = self.parse_address(field[9])
816 if "@" in addr: 816 ↛ 817line 816 didn't jump to line 817, because the condition on line 816 was never true
817 self.keys[key]["email"] = addr
818 self.keys[key]["name"] = name
819 need_fingerprint = True
820 elif key and field[0] == "uid":
821 (name, addr) = self.parse_address(field[9])
822 if "email" not in self.keys[key] and "@" in addr: 822 ↛ 804line 822 didn't jump to line 804, because the condition on line 822 was never false
823 self.keys[key]["email"] = addr
824 self.keys[key]["name"] = name
825 elif need_fingerprint and field[0] == "fpr":
826 self.keys[key]["fingerprints"] = [field[9]]
827 self.fpr_lookup[field[9]] = key
828 need_fingerprint = False
830 (out, err) = p.communicate()
831 r = p.returncode
832 if r != 0: 832 ↛ 833line 832 didn't jump to line 833, because the condition on line 832 was never true
833 raise daklib.gpg.GpgException("command failed: %s\nstdout: %s\nstderr: %s\n" % (cmd, out, err))
835 def import_users_from_ldap(self, session) -> tuple[dict[str, tuple[int, str]], dict[int, tuple[str, str]]]:
836 from .utils import open_ldap_connection
837 import ldap # type: ignore
838 l = open_ldap_connection()
839 cnf = Config()
840 LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
841 Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
842 "(&(keyfingerprint=*)(supplementaryGid=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
843 ["uid", "keyfingerprint", "cn", "mn", "sn"])
845 byuid: dict[int, tuple[str, str]] = {}
846 byname: dict[str, tuple[int, str]] = {}
848 for i in Attrs:
849 entry = i[1]
850 uid = entry["uid"][0].decode()
851 name = get_ldap_name(entry)
852 fingerprints = entry["keyFingerPrint"]
853 keyid = None
854 for f_raw in fingerprints:
855 f = f_raw.decode()
856 key = self.fpr_lookup.get(f, None)
857 if key not in self.keys:
858 continue
859 self.keys[key]["uid"] = uid
861 if keyid is not None:
862 continue
863 keyid = get_or_set_uid(uid, session).uid_id
864 byuid[keyid] = (uid, name)
865 byname[uid] = (keyid, name)
867 return (byname, byuid)
869 def generate_users_from_keyring(self, format: str, session) -> tuple[dict[str, tuple[int, str]], dict[int, tuple[str, str]]]:
870 byuid: dict[int, tuple[str, str]] = {}
871 byname: dict[str, tuple[int, str]] = {}
872 any_invalid = False
873 for x in list(self.keys.keys()):
874 if "email" not in self.keys[x]: 874 ↛ 875line 874 didn't jump to line 875, because the condition on line 874 was never true
875 any_invalid = True
876 self.keys[x]["uid"] = format % "invalid-uid"
877 else:
878 uid = format % self.keys[x]["email"]
879 keyid = get_or_set_uid(uid, session).uid_id
880 byuid[keyid] = (uid, self.keys[x]["name"])
881 byname[uid] = (keyid, self.keys[x]["name"])
882 self.keys[x]["uid"] = uid
884 if any_invalid: 884 ↛ 885line 884 didn't jump to line 885, because the condition on line 884 was never true
885 uid = format % "invalid-uid"
886 keyid = get_or_set_uid(uid, session).uid_id
887 byuid[keyid] = (uid, "ungeneratable user id")
888 byname[uid] = (keyid, "ungeneratable user id")
890 return (byname, byuid)
893__all__.append('Keyring')
896@session_wrapper
897def get_keyring(keyring: str, session=None) -> Optional[Keyring]:
898 """
899 If `keyring` does not have an entry in the `keyrings` table yet, return None
900 If `keyring` already has an entry, simply return the existing :class:`Keyring`
902 :param keyring: the keyring name
903 :return: the :class:`Keyring` object for this keyring
904 """
906 q = session.query(Keyring).filter_by(keyring_name=keyring)
907 return q.one_or_none()
910__all__.append('get_keyring')
913@session_wrapper
914def get_active_keyring_paths(session=None) -> list[str]:
915 """
916 :return: list of active keyring paths
917 """
918 return [x.keyring_name for x in session.query(Keyring).filter(Keyring.active == True).order_by(desc(Keyring.priority)).all()] # noqa:E712
921__all__.append('get_active_keyring_paths')
923################################################################################
926class DBChange:
927 def __init__(self, *args, **kwargs):
928 pass
930 def __repr__(self):
931 return '<DBChange %s>' % self.changesname
934__all__.append('DBChange')
937@session_wrapper
938def get_dbchange(filename: str, session=None) -> Optional[DBChange]:
939 """
940 returns DBChange object for given `filename`.
942 :param filename: the name of the file
943 :param session: Optional SQLA session object (a temporary one will be
944 generated if not supplied)
945 :return: DBChange object for the given filename (:const:`None` if not present)
946 """
947 q = session.query(DBChange).filter_by(changesname=filename)
948 return q.one_or_none()
951__all__.append('get_dbchange')
953################################################################################
956class Maintainer(ORMObject):
957 def __init__(self, name=None):
958 self.name = name
960 def properties(self) -> list[str]:
961 return ['name', 'maintainer_id']
963 def get_split_maintainer(self) -> tuple[str, str, str, str]:
964 if not hasattr(self, 'name') or self.name is None:
965 return ('', '', '', '')
967 return fix_maintainer(self.name.strip())
970__all__.append('Maintainer')
973@session_wrapper
974def get_or_set_maintainer(name: str, session=None) -> Maintainer:
975 """
976 Returns Maintainer object for given maintainer name.
978 If no matching maintainer name is found, a row is inserted.
980 :param name: The maintainer name to add
981 :param session: Optional SQL session object (a temporary one will be
982 generated if not supplied). If not passed, a commit will be performed at
983 the end of the function, otherwise the caller is responsible for commiting.
984 A flush will be performed either way.
985 :return: the Maintainer object for the given maintainer
986 """
988 q = session.query(Maintainer).filter_by(name=name)
989 try:
990 ret = q.one()
991 except NoResultFound:
992 maintainer = Maintainer()
993 maintainer.name = name
994 session.add(maintainer)
995 session.commit_or_flush()
996 ret = maintainer
998 return ret
1001__all__.append('get_or_set_maintainer')
1004@session_wrapper
1005def get_maintainer(maintainer_id: int, session=None) -> Optional[Maintainer]:
1006 """
1007 Return the name of the maintainer behind `maintainer_id` or :const:`None`
1008 if that `maintainer_id` is invalid.
1010 :param maintainer_id: the id of the maintainer
1011 :return: the Maintainer with this `maintainer_id`
1012 """
1014 return session.query(Maintainer).get(maintainer_id)
1017__all__.append('get_maintainer')
1019################################################################################
1022class NewComment:
1023 def __init__(self, *args, **kwargs):
1024 pass
1026 def __repr__(self):
1027 return '''<NewComment for '%s %s' (%s)>''' % (self.package, self.version, self.comment_id)
1030__all__.append('NewComment')
1033@session_wrapper
1034def has_new_comment(policy_queue: "PolicyQueue", package: str, version: str, session=None) -> bool:
1035 """
1036 Returns :const:`True` if the given combination of `package`, `version` has a comment.
1038 :param package: name of the package
1039 :param version: package version
1040 :param session: Optional SQLA session object (a temporary one will be
1041 generated if not supplied)
1042 """
1044 q = session.query(NewComment).filter_by(policy_queue=policy_queue)
1045 q = q.filter_by(package=package)
1046 q = q.filter_by(version=version)
1048 return bool(q.count() > 0)
1051__all__.append('has_new_comment')
1054@session_wrapper
1055def get_new_comments(
1056 policy_queue: "PolicyQueue",
1057 package: Optional[str] = None,
1058 version: Optional[str] = None,
1059 comment_id: Optional[int] = None,
1060 session=None
1061) -> list[NewComment]:
1062 """
1063 Returns (possibly empty) list of NewComment objects for the given
1064 parameters
1066 :param package: name of the package
1067 :param version: package version
1068 :param comment_id: An id of a comment
1069 :param session: Optional SQLA session object (a temporary one will be
1070 generated if not supplied)
1071 :return: A (possibly empty) list of NewComment objects will be returned
1072 """
1074 q = session.query(NewComment).filter_by(policy_queue=policy_queue)
1075 if package is not None: 1075 ↛ 1077line 1075 didn't jump to line 1077, because the condition on line 1075 was never false
1076 q = q.filter_by(package=package)
1077 if version is not None: 1077 ↛ 1078line 1077 didn't jump to line 1078, because the condition on line 1077 was never true
1078 q = q.filter_by(version=version)
1079 if comment_id is not None: 1079 ↛ 1080line 1079 didn't jump to line 1080, because the condition on line 1079 was never true
1080 q = q.filter_by(comment_id=comment_id)
1082 return q.all()
1085__all__.append('get_new_comments')
1087################################################################################
1090class Override(ORMObject):
1091 def __init__(self, package=None, suite=None, component=None, overridetype=None,
1092 section=None, priority=None):
1093 self.package = package
1094 self.suite = suite
1095 self.component = component
1096 self.overridetype = overridetype
1097 self.section = section
1098 self.priority = priority
1100 def properties(self) -> list[str]:
1101 return ['package', 'suite', 'component', 'overridetype', 'section',
1102 'priority']
1105__all__.append('Override')
1108@session_wrapper
1109def get_override(
1110 package: str,
1111 suite: Union[str, list[str], None] = None,
1112 component: Union[str, list[str], None] = None,
1113 overridetype: Union[str, list[str], None] = None,
1114 session=None
1115) -> list[Override]:
1116 """
1117 Returns Override object for the given parameters
1119 :param package: The name of the package
1120 :param suite: The name of the suite (or suites if a list) to limit to. If
1121 None, don't limit. Defaults to None.
1122 :param component: The name of the component (or components if a list) to
1123 limit to. If None, don't limit. Defaults to None.
1124 :param overridetype: The name of the overridetype (or overridetypes if a list) to
1125 limit to. If None, don't limit. Defaults to None.
1126 :param session: Optional SQLA session object (a temporary one will be
1127 generated if not supplied)
1128 :return: A (possibly empty) list of Override objects will be returned
1129 """
1131 q = session.query(Override)
1132 q = q.filter_by(package=package)
1134 if suite is not None:
1135 if not isinstance(suite, list):
1136 suite = [suite]
1137 q = q.join(Suite).filter(Suite.suite_name.in_(suite))
1139 if component is not None:
1140 if not isinstance(component, list):
1141 component = [component]
1142 q = q.join(Component).filter(Component.component_name.in_(component))
1144 if overridetype is not None:
1145 if not isinstance(overridetype, list):
1146 overridetype = [overridetype]
1147 q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype))
1149 return q.all()
1152__all__.append('get_override')
1155################################################################################
1157class OverrideType(ORMObject):
1158 def __init__(self, overridetype=None):
1159 self.overridetype = overridetype
1161 def properties(self) -> list[str]:
1162 return ['overridetype', 'overridetype_id', 'overrides_count']
1165__all__.append('OverrideType')
1168@session_wrapper
1169def get_override_type(override_type: str, session=None) -> Optional[OverrideType]:
1170 """
1171 Returns OverrideType object for given `override_type`.
1173 :param override_type: The name of the override type
1174 :param session: Optional SQLA session object (a temporary one will be
1175 generated if not supplied)
1176 :return: the database id for the given override type
1177 """
1179 q = session.query(OverrideType).filter_by(overridetype=override_type)
1180 return q.one_or_none()
1183__all__.append('get_override_type')
1185################################################################################
1188class PolicyQueue:
1189 def __init__(self, *args, **kwargs):
1190 pass
1192 def __repr__(self):
1193 return '<PolicyQueue %s>' % self.queue_name
1196__all__.append('PolicyQueue')
1199@session_wrapper
1200def get_policy_queue(queuename: str, session=None) -> Optional[PolicyQueue]:
1201 """
1202 Returns PolicyQueue object for given `queuename`
1204 :param queuename: The name of the queue
1205 :param session: Optional SQLA session object (a temporary one will be
1206 generated if not supplied)
1207 :return: PolicyQueue object for the given queue
1208 """
1210 q = session.query(PolicyQueue).filter_by(queue_name=queuename)
1211 return q.one_or_none()
1214__all__.append('get_policy_queue')
1216################################################################################
1219@functools.total_ordering
1220class PolicyQueueUpload:
1221 def _key(self):
1222 return (
1223 self.changes.source,
1224 AptVersion(self.changes.version),
1225 self.source is None,
1226 self.changes.changesname
1227 )
1229 def __eq__(self, other: object) -> bool:
1230 if not isinstance(other, PolicyQueueUpload): 1230 ↛ 1231line 1230 didn't jump to line 1231, because the condition on line 1230 was never true
1231 return NotImplemented
1232 return self._key() == other._key()
1234 def __lt__(self, other):
1235 return self._key() < other._key()
1238__all__.append('PolicyQueueUpload')
1240################################################################################
1243class PolicyQueueByhandFile:
1244 pass
1247__all__.append('PolicyQueueByhandFile')
1249################################################################################
1252class Priority(ORMObject):
1253 def __init__(self, priority=None, level=None):
1254 self.priority = priority
1255 self.level = level
1257 def properties(self) -> list[str]:
1258 return ['priority', 'priority_id', 'level', 'overrides_count']
1260 def __eq__(self, val):
1261 if isinstance(val, str):
1262 warnings.warn("comparison with a `str` is deprecated", DeprecationWarning, stacklevel=2)
1263 return (self.priority == val)
1264 # This signals to use the normal comparison operator
1265 return NotImplemented
1267 def __ne__(self, val):
1268 if isinstance(val, str): 1268 ↛ 1272line 1268 didn't jump to line 1272, because the condition on line 1268 was never false
1269 warnings.warn("comparison with a `str` is deprecated", DeprecationWarning, stacklevel=2)
1270 return (self.priority != val)
1271 # This signals to use the normal comparison operator
1272 return NotImplemented
1274 __hash__ = ORMObject.__hash__
1277__all__.append('Priority')
1280@session_wrapper
1281def get_priority(priority: str, session=None) -> Optional[Priority]:
1282 """
1283 Returns Priority object for given `priority` name.
1285 :param priority: The name of the priority
1286 :param session: Optional SQLA session object (a temporary one will be
1287 generated if not supplied)
1288 :return: Priority object for the given priority
1289 """
1291 q = session.query(Priority).filter_by(priority=priority)
1292 return q.one_or_none()
1295__all__.append('get_priority')
1298@session_wrapper
1299def get_priorities(session=None) -> dict[str, int]:
1300 """
1301 Returns dictionary of priority names -> id mappings
1303 :param session: Optional SQL session object (a temporary one will be
1304 generated if not supplied)
1305 :return: dictionary of priority names -> id mappings
1306 """
1308 ret = {}
1309 q = session.query(Priority)
1310 for x in q.all():
1311 ret[x.priority] = x.priority_id
1313 return ret
1316__all__.append('get_priorities')
1318################################################################################
1321from .database.section import Section
1323__all__.append('Section')
1326@session_wrapper
1327def get_section(section: str, session=None) -> Optional[Section]:
1328 """
1329 Returns Section object for given `section` name.
1331 :param section: The name of the section
1332 :param session: Optional SQLA session object (a temporary one will be
1333 generated if not supplied)
1334 :return: Section object for the given section name
1335 """
1337 q = session.query(Section).filter_by(section=section)
1338 return q.one_or_none()
1341__all__.append('get_section')
1344@session_wrapper
1345def get_sections(session=None) -> dict[str, int]:
1346 """
1347 Returns dictionary of section names -> id mappings
1349 :param session: Optional SQL session object (a temporary one will be
1350 generated if not supplied)
1351 :return: dictionary of section names -> id mappings
1352 """
1354 ret = {}
1355 q = session.query(Section)
1356 for x in q.all():
1357 ret[x.section] = x.section_id
1359 return ret
1362__all__.append('get_sections')
1364################################################################################
1367class SignatureHistory(ORMObject):
1368 @classmethod
1369 def from_signed_file(cls, signed_file: 'daklib.gpg.SignedFile') -> 'SignatureHistory':
1370 """signature history entry from signed file
1372 :param signed_file: signed file
1373 """
1374 self = cls()
1375 self.fingerprint = signed_file.primary_fingerprint
1376 self.signature_timestamp = signed_file.signature_timestamp
1377 self.contents_sha1 = signed_file.contents_sha1
1378 return self
1380 def query(self, session):
1381 return session.query(SignatureHistory).filter_by(fingerprint=self.fingerprint, signature_timestamp=self.signature_timestamp, contents_sha1=self.contents_sha1).first()
1384__all__.append('SignatureHistory')
1386################################################################################
1389class SrcContents(ORMObject):
1390 def __init__(self, file=None, source=None):
1391 self.file = file
1392 self.source = source
1394 def properties(self) -> list[str]:
1395 return ['file', 'source']
1398__all__.append('SrcContents')
1400################################################################################
1403class DBSource(ORMObject):
1404 def __init__(self, source=None, version=None, maintainer=None,
1405 changedby=None, poolfile=None, install_date=None, fingerprint=None):
1406 self.source = source
1407 self.version = version
1408 self.maintainer = maintainer
1409 self.changedby = changedby
1410 self.poolfile = poolfile
1411 self.install_date = install_date
1412 self.fingerprint = fingerprint
1414 @property
1415 def pkid(self) -> int:
1416 return self.source_id
1418 @property
1419 def name(self) -> str:
1420 return self.source
1422 @property
1423 def arch_string(self) -> str:
1424 return 'source'
1426 def properties(self) -> list[str]:
1427 return ['source', 'source_id', 'maintainer', 'changedby',
1428 'fingerprint', 'poolfile', 'version', 'suites_count',
1429 'install_date', 'binaries_count', 'uploaders_count']
1431 def read_control_fields(self) -> Deb822:
1432 '''
1433 Reads the control information from a dsc
1435 :return: fields is the dsc information in a dictionary form
1436 '''
1437 with open(self.poolfile.fullpath, 'r') as fd:
1438 fields = Deb822(fd)
1439 return fields
1441 metadata = association_proxy('key', 'value')
1443 def scan_contents(self) -> set[str]:
1444 '''
1445 Returns a set of names for non directories. The path names are
1446 normalized after converting them from either utf-8 or iso8859-1
1447 encoding.
1448 '''
1449 fullpath = self.poolfile.fullpath
1450 from daklib.contents import UnpackedSource
1451 unpacked = UnpackedSource(fullpath)
1452 fileset = set()
1453 for name in unpacked.get_all_filenames():
1454 fileset.add(name)
1455 return fileset
1457 @property
1458 def proxy(self) -> "MetadataProxy":
1459 session = object_session(self)
1460 query = session.query(SourceMetadata).filter_by(source=self)
1461 return MetadataProxy(session, query)
1464__all__.append('DBSource')
1467@session_wrapper
1468def get_suites_source_in(source: str, session=None) -> 'list[Suite]':
1469 """
1470 Returns list of Suite objects which given `source` name is in
1472 :param source: DBSource package name to search for
1473 :return: list of Suite objects for the given source
1474 """
1476 return session.query(Suite).filter(Suite.sources.any(source=source)).all()
1479__all__.append('get_suites_source_in')
1481# FIXME: This function fails badly if it finds more than 1 source package and
1482# its implementation is trivial enough to be inlined.
1485@session_wrapper
1486def get_source_in_suite(source: str, suite_name: Optional[str], session=None) -> Optional[DBSource]:
1487 """
1488 Returns a DBSource object for a combination of `source` and `suite_name`.
1490 :param source: source package name
1491 :param suite_name: the suite name
1492 :return: the version for `source` in `suite`
1493 """
1494 suite = get_suite(suite_name, session)
1495 if suite is None: 1495 ↛ 1496line 1495 didn't jump to line 1496, because the condition on line 1495 was never true
1496 return None
1497 return suite.get_sources(source).one_or_none()
1500__all__.append('get_source_in_suite')
1503@session_wrapper
1504def import_metadata_into_db(obj: Union[DBBinary, DBSource], session=None) -> None:
1505 """
1506 This routine works on either DBBinary or DBSource objects and imports
1507 their metadata into the database
1508 """
1509 fields = obj.read_control_fields()
1510 for k in fields.keys():
1511 try:
1512 # Try raw ASCII
1513 val = str(fields[k])
1514 except UnicodeEncodeError:
1515 # Fall back to UTF-8
1516 try:
1517 val = fields[k].encode('utf-8')
1518 except UnicodeEncodeError:
1519 # Finally try iso8859-1
1520 val = fields[k].encode('iso8859-1')
1521 # Otherwise we allow the exception to percolate up and we cause
1522 # a reject as someone is playing silly buggers
1524 obj.metadata[get_or_set_metadatakey(k, session)] = val
1526 session.commit_or_flush()
1529__all__.append('import_metadata_into_db')
1531################################################################################
1534class SrcFormat:
1535 def __init__(self, *args, **kwargs):
1536 pass
1538 def __repr__(self):
1539 return '<SrcFormat %s>' % (self.format_name)
1542__all__.append('SrcFormat')
1544################################################################################
1546SUITE_FIELDS = [('SuiteName', 'suite_name'),
1547 ('SuiteID', 'suite_id'),
1548 ('Version', 'version'),
1549 ('Origin', 'origin'),
1550 ('Label', 'label'),
1551 ('Description', 'description'),
1552 ('Untouchable', 'untouchable'),
1553 ('Announce', 'announce'),
1554 ('Codename', 'codename'),
1555 ('OverrideCodename', 'overridecodename'),
1556 ('ValidTime', 'validtime'),
1557 ('Priority', 'priority'),
1558 ('NotAutomatic', 'notautomatic'),
1559 ('CopyChanges', 'copychanges'),
1560 ('OverrideSuite', 'overridesuite')]
1562# Why the heck don't we have any UNIQUE constraints in table suite?
1563# TODO: Add UNIQUE constraints for appropriate columns.
1566class Suite(ORMObject):
1567 def __init__(self, suite_name=None, version=None):
1568 self.suite_name = suite_name
1569 self.version = version
1571 def properties(self) -> list[str]:
1572 return ['suite_name', 'version', 'sources_count', 'binaries_count',
1573 'overrides_count']
1575 def __eq__(self, val):
1576 if isinstance(val, str): 1576 ↛ 1577line 1576 didn't jump to line 1577, because the condition on line 1576 was never true
1577 warnings.warn("comparison with a `str` is deprecated", DeprecationWarning, stacklevel=2)
1578 return (self.suite_name == val)
1579 # This signals to use the normal comparison operator
1580 return NotImplemented
1582 def __ne__(self, val):
1583 if isinstance(val, str): 1583 ↛ 1584line 1583 didn't jump to line 1584, because the condition on line 1583 was never true
1584 warnings.warn("comparison with a `str` is deprecated", DeprecationWarning, stacklevel=2)
1585 return (self.suite_name != val)
1586 # This signals to use the normal comparison operator
1587 return NotImplemented
1589 __hash__ = ORMObject.__hash__
1591 def details(self) -> str:
1592 ret = []
1593 for disp, field in SUITE_FIELDS:
1594 val = getattr(self, field, None)
1595 if val is not None:
1596 ret.append("%s: %s" % (disp, val))
1598 return "\n".join(ret)
1600 def get_architectures(self, skipsrc: bool = False, skipall: bool = False) -> list[Architecture]:
1601 """
1602 Returns list of Architecture objects
1604 :param skipsrc: Whether to skip returning the 'source' architecture entry
1605 :param skipall: Whether to skip returning the 'all' architecture entry
1606 :return: list of Architecture objects for the given name (may be empty)
1607 """
1609 q = object_session(self).query(Architecture).with_parent(self)
1610 if skipsrc:
1611 q = q.filter(Architecture.arch_string != 'source')
1612 if skipall:
1613 q = q.filter(Architecture.arch_string != 'all')
1614 return q.order_by(Architecture.arch_string).all()
1616 def get_sources(self, source: str) -> sqlalchemy.orm.query.Query:
1617 """
1618 Returns a query object representing DBSource that is part of this suite.
1620 :param source: source package name
1621 :return: a query of DBSource
1622 """
1624 session = object_session(self)
1625 return session.query(DBSource).filter_by(source=source). \
1626 with_parent(self)
1628 def get_overridesuite(self) -> "Suite":
1629 if self.overridesuite is None:
1630 return self
1631 else:
1632 return object_session(self).query(Suite).filter_by(suite_name=self.overridesuite).one()
1634 def update_last_changed(self) -> None:
1635 self.last_changed = sqlalchemy.func.now()
1637 @property
1638 def path(self) -> str:
1639 return os.path.join(self.archive.path, 'dists', self.suite_name)
1641 @property
1642 def release_suite_output(self) -> str:
1643 if self.release_suite is not None: 1643 ↛ 1644line 1643 didn't jump to line 1644, because the condition on line 1643 was never true
1644 return self.release_suite
1645 return self.suite_name
1648__all__.append('Suite')
1651@session_wrapper
1652def get_suite(suite: str, session=None) -> Optional[Suite]:
1653 """
1654 Returns Suite object for given `suite` name.
1656 :param suite: The name of the suite
1657 :param session: Optional SQLA session object (a temporary one will be
1658 generated if not supplied)
1659 :return: Suite object for the requested suite name (None if not present)
1660 """
1662 # Start by looking for the dak internal name
1663 q = session.query(Suite).filter_by(suite_name=suite)
1664 try:
1665 return q.one()
1666 except NoResultFound:
1667 pass
1669 # Now try codename
1670 q = session.query(Suite).filter_by(codename=suite)
1671 try:
1672 return q.one()
1673 except NoResultFound:
1674 pass
1676 # Finally give release_suite a try
1677 q = session.query(Suite).filter_by(release_suite=suite)
1678 return q.one_or_none()
1681__all__.append('get_suite')
1683################################################################################
1686@session_wrapper
1687def get_suite_architectures(suite: str, skipsrc: bool = False, skipall: bool = False, session=None) -> list[Architecture]:
1688 """
1689 Returns list of Architecture objects for given `suite` name. The list is
1690 empty if `suite` does not exist.
1692 :param suite: Suite name to search for
1693 :param skipsrc: Whether to skip returning the 'source' architecture entry
1694 :param skipall: Whether to skip returning the 'all' architecture entry
1695 :param session: Optional SQL session object (a temporary one will be
1696 generated if not supplied)
1697 :return: list of Architecture objects for the given name (may be empty)
1698 """
1700 try:
1701 return get_suite(suite, session).get_architectures(skipsrc, skipall)
1702 except AttributeError:
1703 return []
1706__all__.append('get_suite_architectures')
1708################################################################################
1711class Uid(ORMObject):
1712 def __init__(self, uid=None, name=None):
1713 self.uid = uid
1714 self.name = name
1716 def __eq__(self, val):
1717 if isinstance(val, str):
1718 warnings.warn("comparison with a `str` is deprecated", DeprecationWarning, stacklevel=2)
1719 return (self.uid == val)
1720 # This signals to use the normal comparison operator
1721 return NotImplemented
1723 def __ne__(self, val):
1724 if isinstance(val, str):
1725 warnings.warn("comparison with a `str` is deprecated", DeprecationWarning, stacklevel=2)
1726 return (self.uid != val)
1727 # This signals to use the normal comparison operator
1728 return NotImplemented
1730 __hash__ = ORMObject.__hash__
1732 def properties(self) -> list[str]:
1733 return ['uid', 'name', 'fingerprint']
1736__all__.append('Uid')
1739@session_wrapper
1740def get_or_set_uid(uidname: str, session=None) -> Uid:
1741 """
1742 Returns uid object for given uidname.
1744 If no matching uidname is found, a row is inserted.
1746 :param uidname: The uid to add
1747 :param session: Optional SQL session object (a temporary one will be
1748 generated if not supplied). If not passed, a commit will be performed at
1749 the end of the function, otherwise the caller is responsible for commiting.
1750 :return: the uid object for the given uidname
1751 """
1753 q = session.query(Uid).filter_by(uid=uidname)
1755 try:
1756 ret = q.one()
1757 except NoResultFound:
1758 uid = Uid()
1759 uid.uid = uidname
1760 session.add(uid)
1761 session.commit_or_flush()
1762 ret = uid
1764 return ret
1767__all__.append('get_or_set_uid')
1770@session_wrapper
1771def get_uid_from_fingerprint(fpr: str, session=None) -> Optional[Uid]:
1772 q = session.query(Uid)
1773 q = q.join(Fingerprint).filter_by(fingerprint=fpr)
1775 return q.one_or_none()
1778__all__.append('get_uid_from_fingerprint')
1780################################################################################
1783class MetadataKey(ORMObject):
1784 def __init__(self, key=None):
1785 self.key = key
1787 def properties(self) -> list[str]:
1788 return ['key']
1791__all__.append('MetadataKey')
1794@session_wrapper
1795def get_or_set_metadatakey(keyname: str, session=None) -> MetadataKey:
1796 """
1797 Returns MetadataKey object for given uidname.
1799 If no matching keyname is found, a row is inserted.
1801 :param keyname: The keyname to add
1802 :param session: Optional SQL session object (a temporary one will be
1803 generated if not supplied). If not passed, a commit will be performed at
1804 the end of the function, otherwise the caller is responsible for commiting.
1805 :return: the metadatakey object for the given keyname
1806 """
1808 q = session.query(MetadataKey).filter_by(key=keyname)
1810 try:
1811 ret = q.one()
1812 except NoResultFound:
1813 ret = MetadataKey(keyname)
1814 session.add(ret)
1815 session.commit_or_flush()
1817 return ret
1820__all__.append('get_or_set_metadatakey')
1822################################################################################
1825class BinaryMetadata(ORMObject):
1826 def __init__(self, key=None, value=None, binary=None):
1827 self.key = key
1828 self.value = value
1829 if binary is not None:
1830 self.binary = binary
1832 def properties(self) -> list[str]:
1833 return ['binary', 'key', 'value']
1836__all__.append('BinaryMetadata')
1838################################################################################
1841class SourceMetadata(ORMObject):
1842 def __init__(self, key=None, value=None, source=None):
1843 self.key = key
1844 self.value = value
1845 if source is not None:
1846 self.source = source
1848 def properties(self) -> list[str]:
1849 return ['source', 'key', 'value']
1852__all__.append('SourceMetadata')
1854################################################################################
1857class MetadataProxy:
1858 def __init__(self, session, query):
1859 self.session = session
1860 self.query = query
1862 def _get(self, key):
1863 metadata_key = self.session.query(MetadataKey).filter_by(key=key).first()
1864 if metadata_key is None:
1865 return None
1866 metadata = self.query.filter_by(key=metadata_key).first()
1867 return metadata
1869 def __contains__(self, key: str) -> bool:
1870 if self._get(key) is not None: 1870 ↛ 1872line 1870 didn't jump to line 1872, because the condition on line 1870 was never false
1871 return True
1872 return False
1874 def __getitem__(self, key: str) -> str:
1875 metadata = self._get(key)
1876 if metadata is None:
1877 raise KeyError
1878 return metadata.value
1880 def get(self, key: str, default: Optional[str] = None) -> Optional[str]:
1881 try:
1882 return self[key]
1883 except KeyError:
1884 return default
1886################################################################################
1889class VersionCheck(ORMObject):
1890 def __init__(self, *args, **kwargs):
1891 pass
1893 def properties(self) -> list[str]:
1894 return ['check']
1897__all__.append('VersionCheck')
1900@session_wrapper
1901def get_version_checks(suite_name: str, check: Optional[str] = None, session=None) -> list[VersionCheck]:
1902 suite = get_suite(suite_name, session)
1903 if not suite: 1903 ↛ 1906line 1903 didn't jump to line 1906, because the condition on line 1903 was never true
1904 # Make sure that what we return is iterable so that list comprehensions
1905 # involving this don't cause a traceback
1906 return []
1907 q = session.query(VersionCheck).filter_by(suite=suite)
1908 if check: 1908 ↛ 1910line 1908 didn't jump to line 1910, because the condition on line 1908 was never false
1909 q = q.filter_by(check=check)
1910 return q.all()
1913__all__.append('get_version_checks')
1915################################################################################
1918class DBConn:
1919 """
1920 database module init.
1921 """
1922 __shared_state = {}
1924 db_meta = None
1926 tbl_architecture = Architecture.__table__
1928 tables = (
1929 'acl',
1930 'acl_architecture_map',
1931 'acl_fingerprint_map',
1932 'acl_per_source',
1933 'acl_per_suite',
1934 'archive',
1935 'bin_associations',
1936 'bin_contents',
1937 'binaries',
1938 'binaries_metadata',
1939 'build_queue',
1940 'changelogs_text',
1941 'changes',
1942 'component',
1943 'component_suite',
1944 'config',
1945 'dsc_files',
1946 'external_files',
1947 'external_overrides',
1948 'external_signature_requests',
1949 'extra_src_references',
1950 'files',
1951 'files_archive_map',
1952 'fingerprint',
1953 'hashfile',
1954 'keyrings',
1955 'maintainer',
1956 'metadata_keys',
1957 'new_comments',
1958 # TODO: the maintainer column in table override should be removed.
1959 'override',
1960 'override_type',
1961 'policy_queue',
1962 'policy_queue_upload',
1963 'policy_queue_upload_binaries_map',
1964 'policy_queue_byhand_file',
1965 'priority',
1966 'signature_history',
1967 'source',
1968 'source_metadata',
1969 'src_associations',
1970 'src_contents',
1971 'src_format',
1972 'src_uploaders',
1973 'suite',
1974 'suite_acl_map',
1975 'suite_architectures',
1976 'suite_build_queue_copy',
1977 'suite_permission',
1978 'suite_src_formats',
1979 'uid',
1980 'version_check',
1981 )
1983 views = (
1984 'bin_associations_binaries',
1985 'changelogs',
1986 'newest_source',
1987 'newest_src_association',
1988 'package_list',
1989 'source_suite',
1990 'src_associations_src',
1991 )
1993 def __init__(self, *args, **kwargs):
1994 self.__dict__ = self.__shared_state
1996 if not getattr(self, 'initialised', False):
1997 self.initialised = True
1998 self.debug = 'debug' in kwargs
1999 self.__createconn()
2001 def __setuptables(self):
2002 for table_name in self.tables:
2003 table = Table(table_name, self.db_meta,
2004 autoload=True, extend_existing=True)
2005 setattr(self, 'tbl_%s' % table_name, table)
2007 for view_name in self.views:
2008 view = Table(view_name, self.db_meta, autoload=True)
2009 setattr(self, 'view_%s' % view_name, view)
2011 def __setupmappers(self):
2012 mapper(ACL, self.tbl_acl,
2013 properties=dict(
2014 architectures=relation(Architecture, secondary=self.tbl_acl_architecture_map, collection_class=set),
2015 fingerprints=relation(Fingerprint, secondary=self.tbl_acl_fingerprint_map, collection_class=set),
2016 match_keyring=relation(Keyring, primaryjoin=(self.tbl_acl.c.match_keyring_id == self.tbl_keyrings.c.id)),
2017 per_source=relation(ACLPerSource, collection_class=set, back_populates="acl"),
2018 per_suite=relation(ACLPerSuite, collection_class=set, back_populates="acl"),
2019 ))
2021 mapper(ACLPerSource, self.tbl_acl_per_source,
2022 properties=dict(
2023 acl=relation(ACL, back_populates="per_source"),
2024 fingerprint=relation(Fingerprint, primaryjoin=(self.tbl_acl_per_source.c.fingerprint_id == self.tbl_fingerprint.c.id)),
2025 created_by=relation(Fingerprint, primaryjoin=(self.tbl_acl_per_source.c.created_by_id == self.tbl_fingerprint.c.id)),
2026 ))
2028 mapper(ACLPerSuite, self.tbl_acl_per_suite,
2029 properties=dict(
2030 acl=relation(ACL, back_populates="per_suite"),
2031 fingerprint=relation(Fingerprint, primaryjoin=(self.tbl_acl_per_suite.c.fingerprint_id == self.tbl_fingerprint.c.id)),
2032 suite=relation(Suite, primaryjoin=(self.tbl_acl_per_suite.c.suite_id == self.tbl_suite.c.id)),
2033 created_by=relation(Fingerprint, primaryjoin=(self.tbl_acl_per_suite.c.created_by_id == self.tbl_fingerprint.c.id)),
2034 ))
2036 mapper(Archive, self.tbl_archive,
2037 properties=dict(archive_id=self.tbl_archive.c.id,
2038 archive_name=self.tbl_archive.c.name))
2040 mapper(ArchiveFile, self.tbl_files_archive_map,
2041 properties=dict(archive=relation(Archive, backref='files'),
2042 component=relation(Component),
2043 file=relation(PoolFile, backref='archives')))
2045 mapper(BuildQueue, self.tbl_build_queue,
2046 properties=dict(queue_id=self.tbl_build_queue.c.id,
2047 suite=relation(Suite, primaryjoin=(self.tbl_build_queue.c.suite_id == self.tbl_suite.c.id))))
2049 mapper(DBBinary, self.tbl_binaries,
2050 properties=dict(binary_id=self.tbl_binaries.c.id,
2051 package=self.tbl_binaries.c.package,
2052 version=self.tbl_binaries.c.version,
2053 maintainer_id=self.tbl_binaries.c.maintainer,
2054 maintainer=relation(Maintainer),
2055 source_id=self.tbl_binaries.c.source,
2056 source=relation(DBSource, backref='binaries'),
2057 arch_id=self.tbl_binaries.c.architecture,
2058 architecture=relation(Architecture),
2059 poolfile_id=self.tbl_binaries.c.file,
2060 poolfile=relation(PoolFile),
2061 binarytype=self.tbl_binaries.c.type,
2062 fingerprint_id=self.tbl_binaries.c.sig_fpr,
2063 fingerprint=relation(Fingerprint),
2064 install_date=self.tbl_binaries.c.install_date,
2065 suites=relation(Suite, secondary=self.tbl_bin_associations,
2066 backref=backref('binaries', lazy='dynamic')),
2067 extra_sources=relation(DBSource, secondary=self.tbl_extra_src_references,
2068 backref=backref('extra_binary_references', lazy='dynamic')),
2069 key=relation(BinaryMetadata, cascade='all',
2070 collection_class=attribute_mapped_collection('key'), back_populates="binary")),
2071 )
2073 mapper(Component, self.tbl_component,
2074 properties=dict(component_id=self.tbl_component.c.id,
2075 component_name=self.tbl_component.c.name),
2076 )
2078 mapper(DBConfig, self.tbl_config,
2079 properties=dict(config_id=self.tbl_config.c.id))
2081 mapper(DSCFile, self.tbl_dsc_files,
2082 properties=dict(dscfile_id=self.tbl_dsc_files.c.id,
2083 source_id=self.tbl_dsc_files.c.source,
2084 source=relation(DBSource, back_populates="srcfiles"),
2085 poolfile_id=self.tbl_dsc_files.c.file,
2086 poolfile=relation(PoolFile)))
2088 mapper(ExternalOverride, self.tbl_external_overrides,
2089 properties=dict(
2090 suite_id=self.tbl_external_overrides.c.suite,
2091 suite=relation(Suite),
2092 component_id=self.tbl_external_overrides.c.component,
2093 component=relation(Component)))
2095 mapper(PoolFile, self.tbl_files,
2096 properties=dict(file_id=self.tbl_files.c.id,
2097 filesize=self.tbl_files.c.size),
2098 )
2100 mapper(Fingerprint, self.tbl_fingerprint,
2101 properties=dict(fingerprint_id=self.tbl_fingerprint.c.id,
2102 uid_id=self.tbl_fingerprint.c.uid,
2103 uid=relation(Uid, back_populates="fingerprint"),
2104 keyring_id=self.tbl_fingerprint.c.keyring,
2105 keyring=relation(Keyring),
2106 acl=relation(ACL)),
2107 )
2109 mapper(Keyring, self.tbl_keyrings,
2110 properties=dict(keyring_name=self.tbl_keyrings.c.name,
2111 keyring_id=self.tbl_keyrings.c.id,
2112 acl=relation(ACL, primaryjoin=(self.tbl_keyrings.c.acl_id == self.tbl_acl.c.id)))),
2114 mapper(DBChange, self.tbl_changes,
2115 properties=dict(change_id=self.tbl_changes.c.id,
2116 seen=self.tbl_changes.c.seen,
2117 source=self.tbl_changes.c.source,
2118 binaries=self.tbl_changes.c.binaries,
2119 architecture=self.tbl_changes.c.architecture,
2120 distribution=self.tbl_changes.c.distribution,
2121 urgency=self.tbl_changes.c.urgency,
2122 maintainer=self.tbl_changes.c.maintainer,
2123 changedby=self.tbl_changes.c.changedby,
2124 date=self.tbl_changes.c.date,
2125 version=self.tbl_changes.c.version))
2127 mapper(Maintainer, self.tbl_maintainer,
2128 properties=dict(maintainer_id=self.tbl_maintainer.c.id,
2129 maintains_sources=relation(DBSource, backref='maintainer',
2130 primaryjoin=(self.tbl_maintainer.c.id == self.tbl_source.c.maintainer)),
2131 changed_sources=relation(DBSource, backref='changedby',
2132 primaryjoin=(self.tbl_maintainer.c.id == self.tbl_source.c.changedby))),
2133 )
2135 mapper(NewComment, self.tbl_new_comments,
2136 properties=dict(comment_id=self.tbl_new_comments.c.id,
2137 policy_queue=relation(PolicyQueue)))
2139 mapper(Override, self.tbl_override,
2140 properties=dict(suite_id=self.tbl_override.c.suite,
2141 suite=relation(Suite,
2142 backref=backref('overrides', lazy='dynamic')),
2143 package=self.tbl_override.c.package,
2144 component_id=self.tbl_override.c.component,
2145 component=relation(Component,
2146 backref=backref('overrides', lazy='dynamic')),
2147 priority_id=self.tbl_override.c.priority,
2148 priority=relation(Priority,
2149 backref=backref('overrides', lazy='dynamic')),
2150 section_id=self.tbl_override.c.section,
2151 section=relation(Section,
2152 backref=backref('overrides', lazy='dynamic')),
2153 overridetype_id=self.tbl_override.c.type,
2154 overridetype=relation(OverrideType,
2155 backref=backref('overrides', lazy='dynamic'))))
2157 mapper(OverrideType, self.tbl_override_type,
2158 properties=dict(overridetype=self.tbl_override_type.c.type,
2159 overridetype_id=self.tbl_override_type.c.id))
2161 mapper(PolicyQueue, self.tbl_policy_queue,
2162 properties=dict(policy_queue_id=self.tbl_policy_queue.c.id,
2163 suite=relation(Suite, primaryjoin=(self.tbl_policy_queue.c.suite_id == self.tbl_suite.c.id))))
2165 mapper(PolicyQueueUpload, self.tbl_policy_queue_upload,
2166 properties=dict(
2167 changes=relation(DBChange),
2168 policy_queue=relation(PolicyQueue, backref='uploads'),
2169 target_suite=relation(Suite),
2170 source=relation(DBSource),
2171 binaries=relation(DBBinary, secondary=self.tbl_policy_queue_upload_binaries_map),
2172 ))
2174 mapper(PolicyQueueByhandFile, self.tbl_policy_queue_byhand_file,
2175 properties=dict(
2176 upload=relation(PolicyQueueUpload, backref='byhand'),
2177 ))
2179 mapper(Priority, self.tbl_priority,
2180 properties=dict(priority_id=self.tbl_priority.c.id))
2182 mapper(SignatureHistory, self.tbl_signature_history)
2184 mapper(DBSource, self.tbl_source,
2185 properties=dict(source_id=self.tbl_source.c.id,
2186 version=self.tbl_source.c.version,
2187 maintainer_id=self.tbl_source.c.maintainer,
2188 poolfile_id=self.tbl_source.c.file,
2189 poolfile=relation(PoolFile),
2190 fingerprint_id=self.tbl_source.c.sig_fpr,
2191 fingerprint=relation(Fingerprint),
2192 changedby_id=self.tbl_source.c.changedby,
2193 srcfiles=relation(DSCFile,
2194 primaryjoin=(self.tbl_source.c.id == self.tbl_dsc_files.c.source),
2195 back_populates="source"),
2196 suites=relation(Suite, secondary=self.tbl_src_associations,
2197 backref=backref('sources', lazy='dynamic')),
2198 uploaders=relation(Maintainer,
2199 secondary=self.tbl_src_uploaders),
2200 key=relation(SourceMetadata, cascade='all',
2201 collection_class=attribute_mapped_collection('key'), back_populates="source")),
2202 )
2204 mapper(SrcFormat, self.tbl_src_format,
2205 properties=dict(src_format_id=self.tbl_src_format.c.id,
2206 format_name=self.tbl_src_format.c.format_name))
2208 mapper(Suite, self.tbl_suite,
2209 properties=dict(suite_id=self.tbl_suite.c.id,
2210 policy_queue=relation(PolicyQueue, primaryjoin=(self.tbl_suite.c.policy_queue_id == self.tbl_policy_queue.c.id)),
2211 new_queue=relation(PolicyQueue, primaryjoin=(self.tbl_suite.c.new_queue_id == self.tbl_policy_queue.c.id)),
2212 debug_suite=relation(Suite, remote_side=[self.tbl_suite.c.id]),
2213 copy_queues=relation(BuildQueue,
2214 secondary=self.tbl_suite_build_queue_copy),
2215 srcformats=relation(SrcFormat, secondary=self.tbl_suite_src_formats,
2216 backref=backref('suites', lazy='dynamic')),
2217 archive=relation(Archive, backref='suites'),
2218 acls=relation(ACL, secondary=self.tbl_suite_acl_map, collection_class=set),
2219 components=relation(Component, secondary=self.tbl_component_suite,
2220 order_by=self.tbl_component.c.ordering,
2221 backref=backref('suites')),
2222 architectures=relation(Architecture, secondary=self.tbl_suite_architectures,
2223 backref=backref('suites'))),
2224 )
2226 mapper(Uid, self.tbl_uid,
2227 properties=dict(uid_id=self.tbl_uid.c.id,
2228 fingerprint=relation(Fingerprint, back_populates="uid")),
2229 )
2231 mapper(BinContents, self.tbl_bin_contents,
2232 properties=dict(
2233 binary=relation(DBBinary,
2234 backref=backref('contents', lazy='dynamic', cascade='all')),
2235 file=self.tbl_bin_contents.c.file))
2237 mapper(SrcContents, self.tbl_src_contents,
2238 properties=dict(
2239 source=relation(DBSource,
2240 backref=backref('contents', lazy='dynamic', cascade='all')),
2241 file=self.tbl_src_contents.c.file))
2243 mapper(MetadataKey, self.tbl_metadata_keys,
2244 properties=dict(
2245 key_id=self.tbl_metadata_keys.c.key_id,
2246 key=self.tbl_metadata_keys.c.key))
2248 mapper(BinaryMetadata, self.tbl_binaries_metadata,
2249 properties=dict(
2250 binary_id=self.tbl_binaries_metadata.c.bin_id,
2251 binary=relation(DBBinary, back_populates="key"),
2252 key_id=self.tbl_binaries_metadata.c.key_id,
2253 key=relation(MetadataKey),
2254 value=self.tbl_binaries_metadata.c.value))
2256 mapper(SourceMetadata, self.tbl_source_metadata,
2257 properties=dict(
2258 source_id=self.tbl_source_metadata.c.src_id,
2259 source=relation(DBSource, back_populates="key"),
2260 key_id=self.tbl_source_metadata.c.key_id,
2261 key=relation(MetadataKey),
2262 value=self.tbl_source_metadata.c.value))
2264 mapper(VersionCheck, self.tbl_version_check,
2265 properties=dict(
2266 suite_id=self.tbl_version_check.c.suite,
2267 suite=relation(Suite, primaryjoin=self.tbl_version_check.c.suite == self.tbl_suite.c.id),
2268 reference_id=self.tbl_version_check.c.reference,
2269 reference=relation(Suite, primaryjoin=self.tbl_version_check.c.reference == self.tbl_suite.c.id, lazy='joined')))
2271 ## Connection functions
2272 def __createconn(self):
2273 from .config import Config
2274 cnf = Config()
2275 if "DB::Service" in cnf: 2275 ↛ 2276line 2275 didn't jump to line 2276, because the condition on line 2275 was never true
2276 connstr = "postgresql://service=%s" % cnf["DB::Service"]
2277 elif "DB::Host" in cnf:
2278 # TCP/IP
2279 connstr = "postgresql://%s" % cnf["DB::Host"]
2280 if "DB::Port" in cnf and cnf["DB::Port"] != "-1": 2280 ↛ 2281line 2280 didn't jump to line 2281, because the condition on line 2280 was never true
2281 connstr += ":%s" % cnf["DB::Port"]
2282 connstr += "/%s" % cnf["DB::Name"]
2283 else:
2284 # Unix Socket
2285 connstr = "postgresql:///%s" % cnf["DB::Name"]
2286 if "DB::Port" in cnf and cnf["DB::Port"] != "-1": 2286 ↛ 2287line 2286 didn't jump to line 2287, because the condition on line 2286 was never true
2287 connstr += "?port=%s" % cnf["DB::Port"]
2289 engine_args = {'echo': self.debug}
2290 if 'DB::PoolSize' in cnf:
2291 engine_args['pool_size'] = int(cnf['DB::PoolSize'])
2292 if 'DB::MaxOverflow' in cnf:
2293 engine_args['max_overflow'] = int(cnf['DB::MaxOverflow'])
2294 # we don't support non-utf-8 connections
2295 engine_args['client_encoding'] = 'utf-8'
2297 # Monkey patch a new dialect in in order to support service= syntax
2298 import sqlalchemy.dialects.postgresql
2299 from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2
2301 class PGDialect_psycopg2_dak(PGDialect_psycopg2):
2302 def create_connect_args(self, url):
2303 if str(url).startswith('postgresql://service='): 2303 ↛ 2305line 2303 didn't jump to line 2305, because the condition on line 2303 was never true
2304 # Eww
2305 servicename = str(url)[21:]
2306 return (['service=%s' % servicename], {})
2307 else:
2308 return PGDialect_psycopg2.create_connect_args(self, url)
2310 sqlalchemy.dialects.postgresql.base.dialect = PGDialect_psycopg2_dak
2312 try:
2313 self.db_pg = create_engine(connstr, **engine_args)
2314 self.db_smaker = sessionmaker(bind=self.db_pg,
2315 autoflush=True,
2316 autocommit=False)
2318 if self.db_meta is None:
2319 self.__class__.db_meta = Base.metadata
2320 self.__class__.db_meta.bind = self.db_pg
2321 self.__setuptables()
2322 self.__setupmappers()
2324 except OperationalError as e:
2325 from . import utils
2326 utils.fubar("Cannot connect to database (%s)" % str(e))
2328 self.pid = os.getpid()
2330 def session(self, work_mem=0):
2331 '''
2332 Returns a new session object. If a work_mem parameter is provided a new
2333 transaction is started and the work_mem parameter is set for this
2334 transaction. The work_mem parameter is measured in MB. A default value
2335 will be used if the parameter is not set.
2336 '''
2337 # reinitialize DBConn in new processes
2338 if self.pid != os.getpid():
2339 self.__createconn()
2340 session = self.db_smaker()
2341 if work_mem > 0:
2342 session.execute("SET LOCAL work_mem TO '%d MB'" % work_mem)
2343 return session
2346__all__.append('DBConn')