Coverage for daklib/policy.py: 88%
200 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-08-03 16:46 +0000
1# Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU General Public License as published by
5# the Free Software Foundation; either version 2 of the License, or
6# (at your option) any later version.
7#
8# This program is distributed in the hope that it will be useful,
9# but WITHOUT ANY WARRANTY; without even the implied warranty of
10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11# GNU General Public License for more details.
12#
13# You should have received a copy of the GNU General Public License along
14# with this program; if not, write to the Free Software Foundation, Inc.,
15# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17"""module to process policy queue uploads"""
19import errno
20import os
21import shutil
22from typing import TYPE_CHECKING, NotRequired, TypedDict
24from sqlalchemy import select
26from daklib import utils
28from .config import Config
29from .dbconn import (
30 Component,
31 Override,
32 OverrideType,
33 PolicyQueueUpload,
34 Priority,
35 Section,
36 Suite,
37 get_mapped_component,
38 get_mapped_component_name,
39 get_suite_by_name,
40)
41from .fstransactions import FilesystemTransaction
42from .packagelist import PackageList
43from .regexes import re_file_changes, re_file_safe
45if TYPE_CHECKING:
46 from collections.abc import Iterable
48 from sqlalchemy.orm import Session
51class UploadCopy:
52 """export a policy queue upload
54 This class can be used in a with-statement::
56 with UploadCopy(...) as copy:
57 ...
59 Doing so will provide a temporary copy of the upload in the directory
60 given by the :attr:`directory` attribute. The copy will be removed
61 on leaving the with-block.
62 """
64 def __init__(self, upload: PolicyQueueUpload, group: str | None = None):
65 """initializer
67 :param upload: upload to handle
68 """
70 self._directory: str | None = None
71 self.upload = upload
72 self.group = group
74 @property
75 def directory(self) -> str:
76 assert self._directory is not None
77 return self._directory
79 def export(
80 self,
81 directory: str,
82 mode: int | None = None,
83 symlink: bool = True,
84 ignore_existing: bool = False,
85 ) -> None:
86 """export a copy of the upload
88 :param directory: directory to export to
89 :param mode: permissions to use for the copied files
90 :param symlink: use symlinks instead of copying the files
91 :param ignore_existing: ignore already existing files
92 """
93 with FilesystemTransaction() as fs:
94 source = self.upload.source
95 queue = self.upload.policy_queue
97 if source is not None:
98 for dsc_file in source.srcfiles:
99 f = dsc_file.poolfile
100 dst = os.path.join(directory, os.path.basename(f.filename))
101 if not os.path.exists(dst) or not ignore_existing: 101 ↛ 98line 101 didn't jump to line 98 because the condition on line 101 was always true
102 fs.copy(f.fullpath, dst, mode=mode, symlink=symlink)
104 for binary in self.upload.binaries:
105 f = binary.poolfile
106 dst = os.path.join(directory, os.path.basename(f.filename))
107 if not os.path.exists(dst) or not ignore_existing: 107 ↛ 104line 107 didn't jump to line 104 because the condition on line 107 was always true
108 fs.copy(f.fullpath, dst, mode=mode, symlink=symlink)
110 # copy byhand files
111 for byhand in self.upload.byhand: 111 ↛ 112line 111 didn't jump to line 112 because the loop on line 111 never started
112 src = os.path.join(queue.path, byhand.filename)
113 dst = os.path.join(directory, byhand.filename)
114 if os.path.exists(src) and (
115 not os.path.exists(dst) or not ignore_existing
116 ):
117 fs.copy(src, dst, mode=mode, symlink=symlink)
119 # copy .changes
120 src = os.path.join(queue.path, self.upload.changes.changesname)
121 dst = os.path.join(directory, self.upload.changes.changesname)
122 if not os.path.exists(dst) or not ignore_existing: 122 ↛ 93line 122 didn't jump to line 93
123 fs.copy(src, dst, mode=mode, symlink=symlink)
125 def __enter__(self):
126 assert self._directory is None
128 mode = 0o0700
129 symlink = True
130 if self.group is not None: 130 ↛ 131line 130 didn't jump to line 131 because the condition on line 130 was never true
131 mode = 0o2750
132 symlink = False
134 cnf = Config()
135 self._directory = utils.temp_dirname(
136 parent=cnf.get("Dir::TempPath"), mode=mode, group=self.group
137 )
138 self.export(self.directory, symlink=symlink)
139 return self
141 def __exit__(self, *args):
142 if self._directory is not None: 142 ↛ exitline 142 didn't return from function '__exit__' because the condition on line 142 was always true
143 shutil.rmtree(self._directory)
144 self._directory = None
147class MissingOverride(TypedDict):
148 package: str
149 priority: str
150 section: str
151 component: str
152 type: str
153 included: NotRequired[bool]
154 valid: NotRequired[bool]
157def section_matches_component(section: str, component: str) -> bool:
158 """check whether the component embedded in the section matches component
160 The section carries the component as a prefix ("misc" for main,
161 "contrib/misc" for contrib, ...). Both are compared in upload form;
162 ComponentMappings are applied only when the override is stored.
163 """
164 return utils.extract_component_from_section(section)[1] == component
167class PolicyQueueUploadHandler:
168 """process uploads to policy queues
170 This class allows to accept or reject uploads and to get a list of missing
171 overrides (for NEW processing).
172 """
174 def __init__(self, upload: PolicyQueueUpload, session: "Session"):
175 """initializer
177 :param upload: upload to process
178 :param session: database session
179 """
180 self.upload = upload
181 self.session = session
183 @property
184 def _overridesuite(self) -> Suite:
185 overridesuite = self.upload.target_suite
186 if overridesuite.overridesuite is not None:
187 overridesuite = get_suite_by_name(overridesuite.overridesuite, self.session)
188 return overridesuite
190 def _source_override(self, component_name: str) -> Override | None:
191 assert self.upload.source is not None
192 package = self.upload.source.source
193 suite = self._overridesuite
194 component = get_mapped_component(component_name, self.session)
195 query = (
196 select(Override)
197 .filter_by(package=package, suite=suite)
198 .join(OverrideType)
199 .where(OverrideType.overridetype == "dsc")
200 .where(Override.component == component)
201 .limit(1)
202 )
203 return self.session.scalars(query).first()
205 def _binary_override(
206 self, name: str, binarytype, component_name: str
207 ) -> Override | None:
208 suite = self._overridesuite
209 component = get_mapped_component(component_name, self.session)
210 query = (
211 select(Override)
212 .filter_by(package=name, suite=suite)
213 .join(OverrideType)
214 .where(OverrideType.overridetype == binarytype)
215 .where(Override.component == component)
216 .limit(1)
217 )
218 return self.session.scalars(query).first()
220 @property
221 def _changes_prefix(self) -> str:
222 changesname = self.upload.changes.changesname
223 assert changesname.endswith(".changes")
224 assert re_file_changes.match(changesname)
225 return changesname[0:-8]
227 def accept(self) -> None:
228 """mark upload as accepted"""
229 assert len(self.missing_overrides()) == 0
231 fn1 = "ACCEPT.{0}".format(self._changes_prefix)
232 fn = os.path.join(self.upload.policy_queue.path, "COMMENTS", fn1)
233 try:
234 fh = os.open(fn, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
235 with os.fdopen(fh, "wt") as f:
236 f.write("OK\n")
237 except OSError as e:
238 if e.errno == errno.EEXIST:
239 pass
240 else:
241 raise
243 def reject(self, reason: str, *, rejected_by: str | None) -> None:
244 """mark upload as rejected
246 :param reason: reason for the rejection
247 """
248 fn1 = "REJECT.{0}".format(self._changes_prefix)
249 assert re_file_safe.match(fn1)
251 fn = os.path.join(self.upload.policy_queue.path, "COMMENTS", fn1)
252 try:
253 fh = os.open(fn, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
254 with os.fdopen(fh, "wt") as f:
255 f.write("NOTOK\n")
256 if rejected_by: 256 ↛ 258line 256 didn't jump to line 258 because the condition on line 256 was always true
257 f.write(f"From: {rejected_by}\n\n")
258 f.write(reason)
259 except OSError as e:
260 if e.errno == errno.EEXIST:
261 pass
262 else:
263 raise
265 def get_action(self) -> str | None:
266 """get current action
268 :return: string giving the current action, one of 'ACCEPT', 'ACCEPTED', 'REJECT'
269 """
270 changes_prefix = self._changes_prefix
272 for action in ("ACCEPT", "ACCEPTED", "REJECT"):
273 fn1 = "{0}.{1}".format(action, changes_prefix)
274 fn = os.path.join(self.upload.policy_queue.path, "COMMENTS", fn1)
275 if os.path.exists(fn):
276 return action
278 return None
280 def missing_overrides(
281 self, hints: list[MissingOverride] | None = None
282 ) -> list[MissingOverride]:
283 """get missing override entries for the upload
285 :param hints: suggested hints for new overrides in the same format as
286 the return value
287 :return: list of dicts with the following keys:
289 - package: package name
290 - priority: default priority (from upload)
291 - section: default section (from upload)
292 - component: default component (from upload)
293 - type: type of required override ('dsc', 'deb' or 'udeb')
295 All values are strings.
296 """
297 # TODO: use Package-List field
298 missing: list[MissingOverride] = []
299 components = set()
301 source = self.upload.source
303 if hints is None:
304 hints = []
305 hints_map = dict([((o["type"] or "", o["package"]), o) for o in hints])
307 def check_override(
308 name: str,
309 type: str | None,
310 priority: str | None,
311 section: str | None,
312 included: bool,
313 ) -> None:
314 type = type or ""
315 section = section or ""
316 component = "main"
317 if section.find("/") != -1:
318 component = section.split("/", 1)[0]
319 override = self._binary_override(name, type, component)
320 if override is None and not any(
321 o["package"] == name and o["type"] == type for o in missing
322 ):
323 hint = hints_map.get((type, name))
324 if hint is not None:
325 missing.append(hint)
326 component = hint["component"]
327 else:
328 missing.append(
329 {
330 "package": name,
331 "priority": priority or "",
332 "section": section,
333 "component": component or "",
334 "type": type or "",
335 "included": included,
336 }
337 )
338 components.add(component)
340 for binary in self.upload.binaries:
341 binary_proxy = binary.proxy
342 priority = binary_proxy.get("Priority", "optional")
343 section = binary_proxy["Section"]
344 check_override(
345 binary.package, binary.binarytype, priority, section, included=True
346 )
348 if source is not None:
349 source_proxy = source.proxy
350 package_list = PackageList(source_proxy)
351 if not package_list.fallback: 351 ↛ 361line 351 didn't jump to line 361 because the condition on line 351 was always true
352 packages = package_list.packages_for_suite(self.upload.target_suite)
353 for p in packages:
354 check_override(
355 p.name, p.type, p.priority, p.section, included=False
356 )
358 # see daklib.archive.source_component_from_package_list
359 # which we cannot use here as we might not have a Package-List
360 # field for old packages
361 mapping = {c: get_mapped_component_name(c) for c in components}
362 source_component_query = (
363 select(Component)
364 .order_by(Component.ordering)
365 .where(Component.component_name.in_(list(mapping.values())))
366 .limit(1)
367 )
368 source_component_db = self.session.scalars(source_component_query).first()
369 assert source_component_db is not None
370 # use the unmapped component name so the suggested section (built
371 # from it below) matches the sections stored in the database; on
372 # the security archive the component is mapped (e.g. "non-free" ->
373 # "updates/non-free") but sections keep the plain prefix
374 source_component = min(
375 c
376 for c, mapped in mapping.items()
377 if mapped == source_component_db.component_name
378 )
380 override = self._source_override(source_component)
381 if override is None:
382 hint = hints_map.get(("dsc", source.source))
383 if hint is not None:
384 missing.append(hint)
385 else:
386 section = "misc"
387 if source_component != "main":
388 section = "{0}/{1}".format(source_component, section)
389 missing.append(
390 {
391 "package": source.source,
392 "priority": "optional",
393 "section": section,
394 "component": source_component,
395 "type": "dsc",
396 "included": True,
397 }
398 )
400 return missing
402 def add_overrides(
403 self, new_overrides: "Iterable[MissingOverride]", suite: Suite
404 ) -> None:
405 if suite.overridesuite is not None:
406 suite = get_suite_by_name(suite.overridesuite, self.session)
408 for override in new_overrides:
409 package = override["package"]
410 if not section_matches_component(
411 override["section"], override["component"]
412 ):
413 raise Exception(
414 f"Section {override['section']} does not match component "
415 f"{override['component']} for package {package}"
416 )
417 priority = self.session.scalars(
418 select(Priority)
419 .where(Priority.priority == override["priority"])
420 .limit(1)
421 ).first()
422 section = self.session.scalars(
423 select(Section).where(Section.section == override["section"]).limit(1)
424 ).first()
425 component = get_mapped_component(override["component"], self.session)
426 overridetype = self.session.execute(
427 select(OverrideType).where(
428 OverrideType.overridetype == override["type"]
429 )
430 ).scalar_one()
432 if priority is None: 432 ↛ 433line 432 didn't jump to line 433 because the condition on line 432 was never true
433 raise Exception(
434 "Invalid priority {0} for package {1}".format(priority, package)
435 )
436 if section is None: 436 ↛ 437line 436 didn't jump to line 437 because the condition on line 436 was never true
437 raise Exception(
438 "Invalid section {0} for package {1}".format(section, package)
439 )
440 if component is None: 440 ↛ 441line 440 didn't jump to line 441 because the condition on line 440 was never true
441 raise Exception(
442 "Invalid component {0} for package {1}".format(component, package)
443 )
445 o = Override(
446 package=package,
447 suite=suite,
448 component=component,
449 priority=priority,
450 section=section,
451 overridetype=overridetype,
452 )
453 self.session.add(o)
455 self.session.commit()