1# Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU General Public License as published by
5# the Free Software Foundation; either version 2 of the License, or
6# (at your option) any later version.
7#
8# This program is distributed in the hope that it will be useful,
9# but WITHOUT ANY WARRANTY; without even the implied warranty of
10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11# GNU General Public License for more details.
12#
13# You should have received a copy of the GNU General Public License along
14# with this program; if not, write to the Free Software Foundation, Inc.,
15# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17"""module to process policy queue uploads"""
19from .config import Config
20from .dbconn import Component, Override, OverrideType, PolicyQueueUpload, Priority, Section, Suite, get_mapped_component, get_mapped_component_name
21from .fstransactions import FilesystemTransaction
22from .regexes import re_file_changes, re_file_safe
23from .packagelist import PackageList
24import daklib.utils as utils
26import errno
27import os
28import shutil
29from typing import Optional
32class UploadCopy:
33 """export a policy queue upload
35 This class can be used in a with-statement::
37 with UploadCopy(...) as copy:
38 ...
40 Doing so will provide a temporary copy of the upload in the directory
41 given by the :attr:`directory` attribute. The copy will be removed
42 on leaving the with-block.
43 """
45 def __init__(self, upload: PolicyQueueUpload, group: Optional[str] = None):
46 """initializer
48 :param upload: upload to handle
49 """
51 self.directory: Optional[str] = None
52 self.upload = upload
53 self.group = group
55 def export(self, directory: str, mode: Optional[int] = None, symlink: bool = True, ignore_existing: bool = False) -> None:
56 """export a copy of the upload
58 :param directory: directory to export to
59 :param mode: permissions to use for the copied files
60 :param symlink: use symlinks instead of copying the files
61 :param ignore_existing: ignore already existing files
62 """
63 with FilesystemTransaction() as fs:
64 source = self.upload.source
65 queue = self.upload.policy_queue
67 if source is not None:
68 for dsc_file in source.srcfiles:
69 f = dsc_file.poolfile
70 dst = os.path.join(directory, os.path.basename(f.filename))
71 if not os.path.exists(dst) or not ignore_existing: 71 ↛ 68line 71 didn't jump to line 68, because the condition on line 71 was never false
72 fs.copy(f.fullpath, dst, mode=mode, symlink=symlink)
74 for binary in self.upload.binaries:
75 f = binary.poolfile
76 dst = os.path.join(directory, os.path.basename(f.filename))
77 if not os.path.exists(dst) or not ignore_existing: 77 ↛ 74line 77 didn't jump to line 74, because the condition on line 77 was never false
78 fs.copy(f.fullpath, dst, mode=mode, symlink=symlink)
80 # copy byhand files
81 for byhand in self.upload.byhand: 81 ↛ 82line 81 didn't jump to line 82, because the loop on line 81 never started
82 src = os.path.join(queue.path, byhand.filename)
83 dst = os.path.join(directory, byhand.filename)
84 if os.path.exists(src) and (not os.path.exists(dst) or not ignore_existing):
85 fs.copy(src, dst, mode=mode, symlink=symlink)
87 # copy .changes
88 src = os.path.join(queue.path, self.upload.changes.changesname)
89 dst = os.path.join(directory, self.upload.changes.changesname)
90 if not os.path.exists(dst) or not ignore_existing: 90 ↛ exitline 90 didn't return from function 'export', because the condition on line 90 was never false
91 fs.copy(src, dst, mode=mode, symlink=symlink)
93 def __enter__(self):
94 assert self.directory is None
96 mode = 0o0700
97 symlink = True
98 if self.group is not None: 98 ↛ 99line 98 didn't jump to line 99, because the condition on line 98 was never true
99 mode = 0o2750
100 symlink = False
102 cnf = Config()
103 self.directory = utils.temp_dirname(parent=cnf.get('Dir::TempPath'),
104 mode=mode,
105 group=self.group)
106 self.export(self.directory, symlink=symlink)
107 return self
109 def __exit__(self, *args):
110 if self.directory is not None: 110 ↛ 113line 110 didn't jump to line 113, because the condition on line 110 was never false
111 shutil.rmtree(self.directory)
112 self.directory = None
113 return None
116class PolicyQueueUploadHandler:
117 """process uploads to policy queues
119 This class allows to accept or reject uploads and to get a list of missing
120 overrides (for NEW processing).
121 """
123 def __init__(self, upload: PolicyQueueUpload, session):
124 """initializer
126 :param upload: upload to process
127 :param session: database session
128 """
129 self.upload = upload
130 self.session = session
132 @property
133 def _overridesuite(self) -> Suite:
134 overridesuite = self.upload.target_suite
135 if overridesuite.overridesuite is not None:
136 overridesuite = self.session.query(Suite).filter_by(suite_name=overridesuite.overridesuite).one()
137 return overridesuite
139 def _source_override(self, component_name: str) -> Override:
140 package = self.upload.source.source
141 suite = self._overridesuite
142 component = get_mapped_component(component_name, self.session)
143 query = self.session.query(Override).filter_by(package=package, suite=suite) \
144 .join(OverrideType).filter(OverrideType.overridetype == 'dsc') \
145 .filter(Override.component == component)
146 return query.first()
148 def _binary_override(self, name: str, binarytype, component_name: str) -> Override:
149 suite = self._overridesuite
150 component = get_mapped_component(component_name, self.session)
151 query = self.session.query(Override).filter_by(package=name, suite=suite) \
152 .join(OverrideType).filter(OverrideType.overridetype == binarytype) \
153 .filter(Override.component == component)
154 return query.first()
156 @property
157 def _changes_prefix(self) -> str:
158 changesname = self.upload.changes.changesname
159 assert changesname.endswith('.changes')
160 assert re_file_changes.match(changesname)
161 return changesname[0:-8]
163 def accept(self) -> None:
164 """mark upload as accepted"""
165 assert len(self.missing_overrides()) == 0
167 fn1 = 'ACCEPT.{0}'.format(self._changes_prefix)
168 fn = os.path.join(self.upload.policy_queue.path, 'COMMENTS', fn1)
169 try:
170 fh = os.open(fn, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
171 with os.fdopen(fh, 'wt') as f:
172 f.write('OK\n')
173 except OSError as e:
174 if e.errno == errno.EEXIST:
175 pass
176 else:
177 raise
179 def reject(self, reason: str) -> None:
180 """mark upload as rejected
182 :param reason: reason for the rejection
183 """
184 cnf = Config()
186 fn1 = 'REJECT.{0}'.format(self._changes_prefix)
187 assert re_file_safe.match(fn1)
189 fn = os.path.join(self.upload.policy_queue.path, 'COMMENTS', fn1)
190 try:
191 fh = os.open(fn, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
192 with os.fdopen(fh, 'wt') as f:
193 f.write('NOTOK\n')
194 f.write('From: {0} <{1}>\n\n'.format(utils.whoami(), cnf['Dinstall::MyAdminAddress']))
195 f.write(reason)
196 except OSError as e:
197 if e.errno == errno.EEXIST:
198 pass
199 else:
200 raise
202 def get_action(self) -> Optional[str]:
203 """get current action
205 :return: string giving the current action, one of 'ACCEPT', 'ACCEPTED', 'REJECT'
206 """
207 changes_prefix = self._changes_prefix
209 for action in ('ACCEPT', 'ACCEPTED', 'REJECT'):
210 fn1 = '{0}.{1}'.format(action, changes_prefix)
211 fn = os.path.join(self.upload.policy_queue.path, 'COMMENTS', fn1)
212 if os.path.exists(fn):
213 return action
215 return None
217 def missing_overrides(self, hints: Optional[list[dict]] = None) -> list[dict]:
218 """get missing override entries for the upload
220 :param hints: suggested hints for new overrides in the same format as
221 the return value
222 :return: list of dicts with the following keys:
224 - package: package name
225 - priority: default priority (from upload)
226 - section: default section (from upload)
227 - component: default component (from upload)
228 - type: type of required override ('dsc', 'deb' or 'udeb')
230 All values are strings.
231 """
232 # TODO: use Package-List field
233 missing = []
234 components = set()
236 source = self.upload.source
238 if hints is None:
239 hints = []
240 hints_map = dict([((o['type'], o['package']), o) for o in hints])
242 def check_override(name, type, priority, section, included):
243 component = 'main'
244 if section.find('/') != -1:
245 component = section.split('/', 1)[0]
246 override = self._binary_override(name, type, component)
247 if override is None and not any(o['package'] == name and o['type'] == type for o in missing):
248 hint = hints_map.get((type, name))
249 if hint is not None:
250 missing.append(hint)
251 component = hint['component']
252 else:
253 missing.append(
254 dict(
255 package=name,
256 priority=priority,
257 section=section,
258 component=component,
259 type=type,
260 included=included
261 ))
262 components.add(component)
264 for binary in self.upload.binaries:
265 binary_proxy = binary.proxy
266 priority = binary_proxy.get('Priority', 'optional')
267 section = binary_proxy['Section']
268 check_override(binary.package, binary.binarytype, priority, section, included=True)
270 if source is not None:
271 source_proxy = source.proxy
272 package_list = PackageList(source_proxy)
273 if not package_list.fallback: 273 ↛ 281line 273 didn't jump to line 281, because the condition on line 273 was never false
274 packages = package_list.packages_for_suite(self.upload.target_suite)
275 for p in packages:
276 check_override(p.name, p.type, p.priority, p.section, included=False)
278 # see daklib.archive.source_component_from_package_list
279 # which we cannot use here as we might not have a Package-List
280 # field for old packages
281 mapped_components = [get_mapped_component_name(c) for c in components]
282 query = self.session.query(Component).order_by(Component.ordering) \
283 .filter(Component.component_name.in_(mapped_components))
284 source_component = query.first().component_name
286 override = self._source_override(source_component)
287 if override is None:
288 hint = hints_map.get(('dsc', source.source))
289 if hint is not None:
290 missing.append(hint)
291 else:
292 section = 'misc'
293 if source_component != 'main':
294 section = "{0}/{1}".format(source_component, section)
295 missing.append(
296 dict(
297 package=source.source,
298 priority='optional',
299 section=section,
300 component=source_component,
301 type='dsc',
302 included=True,
303 ))
305 return missing
307 def add_overrides(self, new_overrides, suite: Suite) -> None:
308 if suite.overridesuite is not None:
309 suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
311 for override in new_overrides:
312 package = override['package']
313 priority = self.session.query(Priority).filter_by(priority=override['priority']).first()
314 section = self.session.query(Section).filter_by(section=override['section']).first()
315 component = get_mapped_component(override['component'], self.session)
316 overridetype = self.session.query(OverrideType).filter_by(overridetype=override['type']).one()
318 if priority is None: 318 ↛ 319line 318 didn't jump to line 319, because the condition on line 318 was never true
319 raise Exception('Invalid priority {0} for package {1}'.format(priority, package))
320 if section is None: 320 ↛ 321line 320 didn't jump to line 321, because the condition on line 320 was never true
321 raise Exception('Invalid section {0} for package {1}'.format(section, package))
322 if component is None: 322 ↛ 323line 322 didn't jump to line 323, because the condition on line 322 was never true
323 raise Exception('Invalid component {0} for package {1}'.format(component, package))
325 o = Override(package=package, suite=suite, component=component, priority=priority, section=section, overridetype=overridetype)
326 self.session.add(o)
328 self.session.commit()