1# Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU General Public License as published by
5# the Free Software Foundation; either version 2 of the License, or
6# (at your option) any later version.
7#
8# This program is distributed in the hope that it will be useful,
9# but WITHOUT ANY WARRANTY; without even the implied warranty of
10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11# GNU General Public License for more details.
12#
13# You should have received a copy of the GNU General Public License along
14# with this program; if not, write to the Free Software Foundation, Inc.,
15# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17"""module to process policy queue uploads"""
19import errno
20import os
21import shutil
22from typing import Optional
24import daklib.utils as utils
26from .config import Config
27from .dbconn import (
28 Component,
29 Override,
30 OverrideType,
31 PolicyQueueUpload,
32 Priority,
33 Section,
34 Suite,
35 get_mapped_component,
36 get_mapped_component_name,
37)
38from .fstransactions import FilesystemTransaction
39from .packagelist import PackageList
40from .regexes import re_file_changes, re_file_safe
43class UploadCopy:
44 """export a policy queue upload
46 This class can be used in a with-statement::
48 with UploadCopy(...) as copy:
49 ...
51 Doing so will provide a temporary copy of the upload in the directory
52 given by the :attr:`directory` attribute. The copy will be removed
53 on leaving the with-block.
54 """
56 def __init__(self, upload: PolicyQueueUpload, group: Optional[str] = None):
57 """initializer
59 :param upload: upload to handle
60 """
62 self.directory: Optional[str] = None
63 self.upload = upload
64 self.group = group
66 def export(
67 self,
68 directory: str,
69 mode: Optional[int] = None,
70 symlink: bool = True,
71 ignore_existing: bool = False,
72 ) -> None:
73 """export a copy of the upload
75 :param directory: directory to export to
76 :param mode: permissions to use for the copied files
77 :param symlink: use symlinks instead of copying the files
78 :param ignore_existing: ignore already existing files
79 """
80 with FilesystemTransaction() as fs:
81 source = self.upload.source
82 queue = self.upload.policy_queue
84 if source is not None:
85 for dsc_file in source.srcfiles:
86 f = dsc_file.poolfile
87 dst = os.path.join(directory, os.path.basename(f.filename))
88 if not os.path.exists(dst) or not ignore_existing: 88 ↛ 85line 88 didn't jump to line 85, because the condition on line 88 was never false
89 fs.copy(f.fullpath, dst, mode=mode, symlink=symlink)
91 for binary in self.upload.binaries:
92 f = binary.poolfile
93 dst = os.path.join(directory, os.path.basename(f.filename))
94 if not os.path.exists(dst) or not ignore_existing: 94 ↛ 91line 94 didn't jump to line 91, because the condition on line 94 was never false
95 fs.copy(f.fullpath, dst, mode=mode, symlink=symlink)
97 # copy byhand files
98 for byhand in self.upload.byhand: 98 ↛ 99line 98 didn't jump to line 99, because the loop on line 98 never started
99 src = os.path.join(queue.path, byhand.filename)
100 dst = os.path.join(directory, byhand.filename)
101 if os.path.exists(src) and (
102 not os.path.exists(dst) or not ignore_existing
103 ):
104 fs.copy(src, dst, mode=mode, symlink=symlink)
106 # copy .changes
107 src = os.path.join(queue.path, self.upload.changes.changesname)
108 dst = os.path.join(directory, self.upload.changes.changesname)
109 if not os.path.exists(dst) or not ignore_existing: 109 ↛ exitline 109 didn't return from function 'export', because the condition on line 109 was never false
110 fs.copy(src, dst, mode=mode, symlink=symlink)
112 def __enter__(self):
113 assert self.directory is None
115 mode = 0o0700
116 symlink = True
117 if self.group is not None: 117 ↛ 118line 117 didn't jump to line 118, because the condition on line 117 was never true
118 mode = 0o2750
119 symlink = False
121 cnf = Config()
122 self.directory = utils.temp_dirname(
123 parent=cnf.get("Dir::TempPath"), mode=mode, group=self.group
124 )
125 self.export(self.directory, symlink=symlink)
126 return self
128 def __exit__(self, *args):
129 if self.directory is not None: 129 ↛ 132line 129 didn't jump to line 132, because the condition on line 129 was never false
130 shutil.rmtree(self.directory)
131 self.directory = None
132 return None
135class PolicyQueueUploadHandler:
136 """process uploads to policy queues
138 This class allows to accept or reject uploads and to get a list of missing
139 overrides (for NEW processing).
140 """
142 def __init__(self, upload: PolicyQueueUpload, session):
143 """initializer
145 :param upload: upload to process
146 :param session: database session
147 """
148 self.upload = upload
149 self.session = session
151 @property
152 def _overridesuite(self) -> Suite:
153 overridesuite = self.upload.target_suite
154 if overridesuite.overridesuite is not None:
155 overridesuite = (
156 self.session.query(Suite)
157 .filter_by(suite_name=overridesuite.overridesuite)
158 .one()
159 )
160 return overridesuite
162 def _source_override(self, component_name: str) -> Override:
163 package = self.upload.source.source
164 suite = self._overridesuite
165 component = get_mapped_component(component_name, self.session)
166 query = (
167 self.session.query(Override)
168 .filter_by(package=package, suite=suite)
169 .join(OverrideType)
170 .filter(OverrideType.overridetype == "dsc")
171 .filter(Override.component == component)
172 )
173 return query.first()
175 def _binary_override(self, name: str, binarytype, component_name: str) -> Override:
176 suite = self._overridesuite
177 component = get_mapped_component(component_name, self.session)
178 query = (
179 self.session.query(Override)
180 .filter_by(package=name, suite=suite)
181 .join(OverrideType)
182 .filter(OverrideType.overridetype == binarytype)
183 .filter(Override.component == component)
184 )
185 return query.first()
187 @property
188 def _changes_prefix(self) -> str:
189 changesname = self.upload.changes.changesname
190 assert changesname.endswith(".changes")
191 assert re_file_changes.match(changesname)
192 return changesname[0:-8]
194 def accept(self) -> None:
195 """mark upload as accepted"""
196 assert len(self.missing_overrides()) == 0
198 fn1 = "ACCEPT.{0}".format(self._changes_prefix)
199 fn = os.path.join(self.upload.policy_queue.path, "COMMENTS", fn1)
200 try:
201 fh = os.open(fn, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
202 with os.fdopen(fh, "wt") as f:
203 f.write("OK\n")
204 except OSError as e:
205 if e.errno == errno.EEXIST:
206 pass
207 else:
208 raise
210 def reject(self, reason: str) -> None:
211 """mark upload as rejected
213 :param reason: reason for the rejection
214 """
215 cnf = Config()
217 fn1 = "REJECT.{0}".format(self._changes_prefix)
218 assert re_file_safe.match(fn1)
220 fn = os.path.join(self.upload.policy_queue.path, "COMMENTS", fn1)
221 try:
222 fh = os.open(fn, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
223 with os.fdopen(fh, "wt") as f:
224 f.write("NOTOK\n")
225 f.write(
226 "From: {0} <{1}>\n\n".format(
227 utils.whoami(), cnf["Dinstall::MyAdminAddress"]
228 )
229 )
230 f.write(reason)
231 except OSError as e:
232 if e.errno == errno.EEXIST:
233 pass
234 else:
235 raise
237 def get_action(self) -> Optional[str]:
238 """get current action
240 :return: string giving the current action, one of 'ACCEPT', 'ACCEPTED', 'REJECT'
241 """
242 changes_prefix = self._changes_prefix
244 for action in ("ACCEPT", "ACCEPTED", "REJECT"):
245 fn1 = "{0}.{1}".format(action, changes_prefix)
246 fn = os.path.join(self.upload.policy_queue.path, "COMMENTS", fn1)
247 if os.path.exists(fn):
248 return action
250 return None
252 def missing_overrides(self, hints: Optional[list[dict]] = None) -> list[dict]:
253 """get missing override entries for the upload
255 :param hints: suggested hints for new overrides in the same format as
256 the return value
257 :return: list of dicts with the following keys:
259 - package: package name
260 - priority: default priority (from upload)
261 - section: default section (from upload)
262 - component: default component (from upload)
263 - type: type of required override ('dsc', 'deb' or 'udeb')
265 All values are strings.
266 """
267 # TODO: use Package-List field
268 missing = []
269 components = set()
271 source = self.upload.source
273 if hints is None:
274 hints = []
275 hints_map = dict([((o["type"], o["package"]), o) for o in hints])
277 def check_override(name, type, priority, section, included):
278 component = "main"
279 if section.find("/") != -1:
280 component = section.split("/", 1)[0]
281 override = self._binary_override(name, type, component)
282 if override is None and not any(
283 o["package"] == name and o["type"] == type for o in missing
284 ):
285 hint = hints_map.get((type, name))
286 if hint is not None:
287 missing.append(hint)
288 component = hint["component"]
289 else:
290 missing.append(
291 dict(
292 package=name,
293 priority=priority,
294 section=section,
295 component=component,
296 type=type,
297 included=included,
298 )
299 )
300 components.add(component)
302 for binary in self.upload.binaries:
303 binary_proxy = binary.proxy
304 priority = binary_proxy.get("Priority", "optional")
305 section = binary_proxy["Section"]
306 check_override(
307 binary.package, binary.binarytype, priority, section, included=True
308 )
310 if source is not None:
311 source_proxy = source.proxy
312 package_list = PackageList(source_proxy)
313 if not package_list.fallback: 313 ↛ 323line 313 didn't jump to line 323, because the condition on line 313 was never false
314 packages = package_list.packages_for_suite(self.upload.target_suite)
315 for p in packages:
316 check_override(
317 p.name, p.type, p.priority, p.section, included=False
318 )
320 # see daklib.archive.source_component_from_package_list
321 # which we cannot use here as we might not have a Package-List
322 # field for old packages
323 mapped_components = [get_mapped_component_name(c) for c in components]
324 query = (
325 self.session.query(Component)
326 .order_by(Component.ordering)
327 .filter(Component.component_name.in_(mapped_components))
328 )
329 source_component = query.first().component_name
331 override = self._source_override(source_component)
332 if override is None:
333 hint = hints_map.get(("dsc", source.source))
334 if hint is not None:
335 missing.append(hint)
336 else:
337 section = "misc"
338 if source_component != "main":
339 section = "{0}/{1}".format(source_component, section)
340 missing.append(
341 dict(
342 package=source.source,
343 priority="optional",
344 section=section,
345 component=source_component,
346 type="dsc",
347 included=True,
348 )
349 )
351 return missing
353 def add_overrides(self, new_overrides, suite: Suite) -> None:
354 if suite.overridesuite is not None:
355 suite = (
356 self.session.query(Suite)
357 .filter_by(suite_name=suite.overridesuite)
358 .one()
359 )
361 for override in new_overrides:
362 package = override["package"]
363 priority = (
364 self.session.query(Priority)
365 .filter_by(priority=override["priority"])
366 .first()
367 )
368 section = (
369 self.session.query(Section)
370 .filter_by(section=override["section"])
371 .first()
372 )
373 component = get_mapped_component(override["component"], self.session)
374 overridetype = (
375 self.session.query(OverrideType)
376 .filter_by(overridetype=override["type"])
377 .one()
378 )
380 if priority is None: 380 ↛ 381line 380 didn't jump to line 381, because the condition on line 380 was never true
381 raise Exception(
382 "Invalid priority {0} for package {1}".format(priority, package)
383 )
384 if section is None: 384 ↛ 385line 384 didn't jump to line 385, because the condition on line 384 was never true
385 raise Exception(
386 "Invalid section {0} for package {1}".format(section, package)
387 )
388 if component is None: 388 ↛ 389line 388 didn't jump to line 389, because the condition on line 388 was never true
389 raise Exception(
390 "Invalid component {0} for package {1}".format(component, package)
391 )
393 o = Override(
394 package=package,
395 suite=suite,
396 component=component,
397 priority=priority,
398 section=section,
399 overridetype=overridetype,
400 )
401 self.session.add(o)
403 self.session.commit()