1#! /usr/bin/env python3
3"""
4Checks Debian packages from Incoming
5@contact: Debian FTP Master <ftpmaster@debian.org>
6@copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
7@copyright: 2009 Joerg Jaspert <joerg@debian.org>
8@copyright: 2009 Mark Hymers <mhy@debian.org>
9@copyright: 2009 Frank Lichtenheld <djpig@debian.org>
10@license: GNU General Public License version 2 or later
11"""
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27# based on process-unchecked and process-accepted
29## pu|pa: locking (daily.lock)
30## pu|pa: parse arguments -> list of changes files
31## pa: initialize urgency log
32## pu|pa: sort changes list
34## foreach changes:
35### pa: load dak file
36## pu: copy CHG to tempdir
37## pu: check CHG signature
38## pu: parse changes file
39## pu: checks:
40## pu: check distribution (mappings, rejects)
41## pu: copy FILES to tempdir
42## pu: check whether CHG already exists in CopyChanges
43## pu: check whether FILES already exist in one of the policy queues
44## for deb in FILES:
45## pu: extract control information
46## pu: various checks on control information
47## pu|pa: search for source (in CHG, projectb, policy queues)
48## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
49## pu|pa: check whether deb already exists in the pool
50## for src in FILES:
51## pu: various checks on filenames and CHG consistency
52## pu: if isdsc: check signature
53## for file in FILES:
54## pu: various checks
55## pu: NEW?
56## //pu: check whether file already exists in the pool
57## pu: store what "Component" the package is currently in
58## pu: check whether we found everything we were looking for in CHG
59## pu: check the DSC:
60## pu: check whether we need and have ONE DSC
61## pu: parse the DSC
62## pu: various checks //maybe drop some of the in favor of lintian
63## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
64## pu: check whether DSC_FILES is consistent with "Format"
65## for src in DSC_FILES:
66## pu|pa: check whether file already exists in the pool (with special handling for .orig.tar.gz)
67## pu: create new tempdir
68## pu: create symlink mirror of source
69## pu: unpack source
70## pu: extract changelog information for BTS
71## //pu: create missing .orig symlink
72## pu: check with lintian
73## for file in FILES:
74## pu: check checksums and sizes
75## for file in DSC_FILES:
76## pu: check checksums and sizes
77## pu: CHG: check urgency
78## for deb in FILES:
79## pu: extract contents list and check for dubious timestamps
80## pu: check that the uploader is actually allowed to upload the package
81### pa: install:
82### if stable_install:
83### pa: remove from p-u
84### pa: add to stable
85### pa: move CHG to morgue
86### pa: append data to ChangeLog
87### pa: send mail
88### pa: remove .dak file
89### else:
90### pa: add dsc to db:
91### for file in DSC_FILES:
92### pa: add file to file
93### pa: add file to dsc_files
94### pa: create source entry
95### pa: update source associations
96### pa: update src_uploaders
97### for deb in FILES:
98### pa: add deb to db:
99### pa: add file to file
100### pa: find source entry
101### pa: create binaries entry
102### pa: update binary associations
103### pa: .orig component move
104### pa: move files to pool
105### pa: save CHG
106### pa: move CHG to done/
107### pa: change entry in queue_build
108## pu: use dispatch table to choose target queue:
109## if NEW:
110## pu: write .dak file
111## pu: move to NEW
112## pu: send mail
113## elsif AUTOBYHAND:
114## pu: run autobyhand script
115## pu: if stuff left, do byhand or accept
116## elsif targetqueue in (oldstable, stable, embargo, unembargo):
117## pu: write .dak file
118## pu: check overrides
119## pu: move to queue
120## pu: send mail
121## else:
122## pu: write .dak file
123## pu: move to ACCEPTED
124## pu: send mails
125## pu: create files for BTS
126## pu: create entry in queue_build
127## pu: check overrides
129# Integrity checks
130## GPG
131## Parsing changes (check for duplicates)
132## Parse dsc
133## file list checks
135# New check layout (TODO: Implement)
136## Permission checks
137### suite mappings
138### ACLs
139### version checks (suite)
140### override checks
142## Source checks
143### copy orig
144### unpack
145### BTS changelog
146### src contents
147### lintian
148### urgency log
150## Binary checks
151### timestamps
152### control checks
153### src relation check
154### contents
156## Database insertion (? copy from stuff)
157### BYHAND / NEW / Policy queues
158### Pool
160## Queue builds
162import datetime
163import errno
164import fcntl
165import functools
166import os
167import sys
168import traceback
169import apt_pkg
170import time
171from collections.abc import Iterable
172from typing import NoReturn
174from daklib import daklog
175from daklib.dbconn import *
176from daklib.urgencylog import UrgencyLog
177from daklib.summarystats import SummaryStats
178from daklib.config import Config
179import daklib.utils as utils
180from daklib.regexes import *
182import daklib.announce
183import daklib.archive
184import daklib.checks
185import daklib.upload
187###############################################################################
189Options = None
190Logger = None
192###############################################################################
195def usage(exit_code=0) -> NoReturn:
196 print("""Usage: dak process-upload [OPTION]... [CHANGES]...
197 -a, --automatic automatic run
198 -d, --directory <DIR> process uploads in <DIR>
199 -h, --help show this help and exit.
200 -n, --no-action don't do anything
201 -p, --no-lock don't check lockfile !! for cron.daily only !!
202 -s, --no-mail don't send any mail
203 -V, --version display the version number and exit""")
204 sys.exit(exit_code)
206###############################################################################
209def try_or_reject(function):
210 """Try to call function or reject the upload if that fails
211 """
212 @functools.wraps(function)
213 def wrapper(directory: str, upload: daklib.archive.ArchiveUpload, *args, **kwargs):
214 reason = 'No exception caught. This should not happen.'
216 try:
217 return function(directory, upload, *args, **kwargs)
218 except (daklib.archive.ArchiveException, daklib.checks.Reject) as e:
219 reason = str(e)
220 except Exception as e:
221 reason = "There was an uncaught exception when processing your upload:\n{0}\nAny original reject reason follows below.".format(traceback.format_exc())
223 try:
224 upload.rollback()
225 return real_reject(directory, upload, reason=reason)
226 except Exception as e:
227 reason = "In addition there was an exception when rejecting the package:\n{0}\nPrevious reasons:\n{1}".format(traceback.format_exc(), reason)
228 upload.rollback()
229 return real_reject(directory, upload, reason=reason, notify=False)
231 raise Exception('Rejecting upload failed after multiple tries. Giving up. Last reason:\n{0}'.format(reason))
233 return wrapper
236def get_processed_upload(upload: daklib.archive.ArchiveUpload) -> daklib.announce.ProcessedUpload:
237 changes = upload.changes
238 control = upload.changes.changes
240 pu = daklib.announce.ProcessedUpload()
242 pu.maintainer = control.get('Maintainer')
243 pu.changed_by = control.get('Changed-By')
244 pu.fingerprint = changes.primary_fingerprint
246 pu.suites = upload.final_suites or []
247 pu.from_policy_suites = []
249 with open(upload.changes.path, 'r') as fd:
250 pu.changes = fd.read()
251 pu.changes_filename = upload.changes.filename
252 pu.sourceful = upload.changes.sourceful
253 pu.source = control.get('Source')
254 pu.version = control.get('Version')
255 pu.architecture = control.get('Architecture')
256 pu.bugs = changes.closed_bugs
258 pu.program = "process-upload"
260 pu.warnings = upload.warnings
262 return pu
265@try_or_reject
266def accept(directory: str, upload: daklib.archive.ArchiveUpload) -> None:
267 cnf = Config()
269 Logger.log(['ACCEPT', upload.changes.filename])
270 print("ACCEPT")
272 upload.install()
273 utils.process_buildinfos(upload.directory, upload.changes.buildinfo_files,
274 upload.transaction.fs, Logger)
276 accepted_to_real_suite = any(suite.policy_queue is None for suite in upload.final_suites)
277 sourceful_upload = upload.changes.sourceful
279 control = upload.changes.changes
280 if sourceful_upload and not Options['No-Action']:
281 urgency = control.get('Urgency')
282 # As per policy 5.6.17, the urgency can be followed by a space and a
283 # comment. Extract only the urgency from the string.
284 if ' ' in urgency: 284 ↛ 285line 284 didn't jump to line 285, because the condition on line 284 was never true
285 urgency, comment = urgency.split(' ', 1)
286 if urgency not in cnf.value_list('Urgency::Valid'): 286 ↛ 287line 286 didn't jump to line 287, because the condition on line 286 was never true
287 urgency = cnf['Urgency::Default']
288 UrgencyLog().log(control['Source'], control['Version'], urgency)
290 pu = get_processed_upload(upload)
291 daklib.announce.announce_accept(pu)
293 # Move .changes to done, but only for uploads that were accepted to a
294 # real suite. process-policy will handle this for uploads to queues.
295 if accepted_to_real_suite:
296 src = os.path.join(upload.directory, upload.changes.filename)
298 now = datetime.datetime.now()
299 donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
300 dst = os.path.join(donedir, upload.changes.filename)
301 dst = utils.find_next_free(dst)
303 upload.transaction.fs.copy(src, dst, mode=0o644)
305 SummaryStats().accept_count += 1
306 SummaryStats().accept_bytes += upload.changes.bytes
309@try_or_reject
310def accept_to_new(directory: str, upload: daklib.archive.ArchiveUpload) -> None:
312 Logger.log(['ACCEPT-TO-NEW', upload.changes.filename])
313 print("ACCEPT-TO-NEW")
315 upload.install_to_new()
316 # TODO: tag bugs pending
318 pu = get_processed_upload(upload)
319 daklib.announce.announce_new(pu)
321 SummaryStats().accept_count += 1
322 SummaryStats().accept_bytes += upload.changes.bytes
325@try_or_reject
326def reject(directory: str, upload: daklib.archive.ArchiveUpload, reason=None, notify=True) -> None:
327 real_reject(directory, upload, reason, notify)
330def real_reject(directory: str, upload: daklib.archive.ArchiveUpload, reason=None, notify=True) -> None:
331 # XXX: rejection itself should go to daklib.archive.ArchiveUpload
332 cnf = Config()
334 Logger.log(['REJECT', upload.changes.filename])
335 print("REJECT")
337 fs = upload.transaction.fs
338 rejectdir = cnf['Dir::Reject']
340 files = [f.filename for f in upload.changes.files.values()]
341 files.append(upload.changes.filename)
343 for fn in files:
344 src = os.path.join(upload.directory, fn)
345 dst = utils.find_next_free(os.path.join(rejectdir, fn))
346 if not os.path.exists(src):
347 continue
348 fs.copy(src, dst)
350 if upload.reject_reasons is not None: 350 ↛ 355line 350 didn't jump to line 355, because the condition on line 350 was never false
351 if reason is None: 351 ↛ 353line 351 didn't jump to line 353, because the condition on line 351 was never false
352 reason = ''
353 reason = reason + '\n' + '\n'.join(upload.reject_reasons)
355 if reason is None: 355 ↛ 356line 355 didn't jump to line 356, because the condition on line 355 was never true
356 reason = '(Unknown reason. Please check logs.)'
358 dst = utils.find_next_free(os.path.join(rejectdir, '{0}.reason'.format(upload.changes.filename)))
359 fh = fs.create(dst)
360 fh.write(reason)
361 fh.close()
363 if notify: 363 ↛ 367line 363 didn't jump to line 367, because the condition on line 363 was never false
364 pu = get_processed_upload(upload)
365 daklib.announce.announce_reject(pu, reason)
367 SummaryStats().reject_count += 1
369###############################################################################
372def action(directory: str, upload: daklib.archive.ArchiveUpload) -> bool:
373 changes = upload.changes
374 processed = True
376 global Logger
378 cnf = Config()
380 okay = upload.check()
382 try:
383 summary = changes.changes.get('Changes', '')
384 except UnicodeDecodeError as e:
385 summary = "Reading changes failed: %s" % (e)
386 # the upload checks should have detected this, but make sure this
387 # upload gets rejected in any case
388 upload.reject_reasons.append(summary)
390 package_info = []
391 if okay:
392 if changes.source is not None:
393 package_info.append("source:{0}".format(changes.source.dsc['Source']))
394 for binary in changes.binaries:
395 package_info.append("binary:{0}".format(binary.control['Package']))
397 (prompt, answer) = ("", "XXX")
398 if Options["No-Action"] or Options["Automatic"]: 398 ↛ 401line 398 didn't jump to line 401, because the condition on line 398 was never false
399 answer = 'S'
401 print(summary)
402 print()
403 print("\n".join(package_info))
404 print()
405 if len(upload.warnings) > 0:
406 print("\n".join(upload.warnings))
407 print()
409 if len(upload.reject_reasons) > 0:
410 print("Reason:")
411 print("\n".join(upload.reject_reasons))
412 print()
414 path = os.path.join(directory, changes.filename)
415 created = os.stat(path).st_mtime
416 now = time.time()
417 too_new = (now - created < int(cnf['Dinstall::SkipTime']))
419 if too_new:
420 print("SKIP (too new)")
421 prompt = "[S]kip, Quit ?"
422 else:
423 prompt = "[R]eject, Skip, Quit ?"
424 if Options["Automatic"]: 424 ↛ 435line 424 didn't jump to line 435, because the condition on line 424 was never false
425 answer = 'R'
426 elif upload.new:
427 prompt = "[N]ew, Skip, Quit ?"
428 if Options['Automatic']: 428 ↛ 435line 428 didn't jump to line 435, because the condition on line 428 was never false
429 answer = 'N'
430 else:
431 prompt = "[A]ccept, Skip, Quit ?"
432 if Options['Automatic']: 432 ↛ 435line 432 didn't jump to line 435, because the condition on line 432 was never false
433 answer = 'A'
435 while prompt.find(answer) == -1: 435 ↛ 436line 435 didn't jump to line 436, because the condition on line 435 was never true
436 answer = utils.input_or_exit(prompt)
437 m = re_default_answer.match(prompt)
438 if answer == "":
439 answer = m.group(1)
440 answer = answer[:1].upper()
442 if answer == 'R':
443 reject(directory, upload)
444 elif answer == 'A':
445 # upload.try_autobyhand must not be run with No-Action.
446 if Options['No-Action']: 446 ↛ 447line 446 didn't jump to line 447, because the condition on line 446 was never true
447 accept(directory, upload)
448 elif upload.try_autobyhand(): 448 ↛ 451line 448 didn't jump to line 451, because the condition on line 448 was never false
449 accept(directory, upload)
450 else:
451 print("W: redirecting to BYHAND as automatic processing failed.")
452 accept_to_new(directory, upload)
453 elif answer == 'N':
454 accept_to_new(directory, upload)
455 elif answer == 'Q': 455 ↛ 456line 455 didn't jump to line 456, because the condition on line 455 was never true
456 sys.exit(0)
457 elif answer == 'S': 457 ↛ 460line 457 didn't jump to line 460, because the condition on line 457 was never false
458 processed = False
460 if not Options['No-Action']: 460 ↛ 463line 460 didn't jump to line 463, because the condition on line 460 was never false
461 upload.commit()
463 return processed
465###############################################################################
468def unlink_if_exists(path: str) -> None:
469 try:
470 os.unlink(path)
471 except OSError as e:
472 if e.errno != errno.ENOENT: 472 ↛ 473line 472 didn't jump to line 473, because the condition on line 472 was never true
473 raise
476def process_it(directory: str, changes: daklib.upload.Changes, keyrings: list[str]) -> None:
477 global Logger
479 print("\n{0}\n".format(changes.filename))
480 Logger.log(["Processing changes file", changes.filename])
482 with daklib.archive.ArchiveUpload(directory, changes, keyrings) as upload:
483 processed = action(directory, upload)
484 if processed and not Options['No-Action']:
485 session = DBConn().session()
486 history = SignatureHistory.from_signed_file(upload.changes)
487 if history.query(session) is None: 487 ↛ 490line 487 didn't jump to line 490, because the condition on line 487 was never false
488 session.add(history)
489 session.commit()
490 session.close()
492 unlink_if_exists(os.path.join(directory, changes.filename))
493 for fn in changes.files:
494 unlink_if_exists(os.path.join(directory, fn))
496###############################################################################
499def process_changes(changes_filenames: Iterable[str]):
500 session = DBConn().session()
501 keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority)
502 keyring_files = [k.keyring_name for k in keyrings]
503 session.close()
505 changes = []
506 for fn in changes_filenames:
507 try:
508 directory, filename = os.path.split(fn)
509 c = daklib.upload.Changes(directory, filename, keyring_files)
510 changes.append([directory, c])
511 except Exception as e:
512 try:
513 Logger.log([filename, "Error while loading changes file {0}: {1}".format(fn, e)])
514 except Exception as e:
515 Logger.log([filename, "Error while loading changes file {0}, with additional error while printing exception: {1}".format(fn, repr(e))])
517 changes.sort(key=lambda x: x[1])
519 for directory, c in changes:
520 process_it(directory, c, keyring_files)
522###############################################################################
525def main():
526 global Options, Logger
528 cnf = Config()
529 summarystats = SummaryStats()
531 Arguments = [('a', "automatic", "Dinstall::Options::Automatic"),
532 ('h', "help", "Dinstall::Options::Help"),
533 ('n', "no-action", "Dinstall::Options::No-Action"),
534 ('p', "no-lock", "Dinstall::Options::No-Lock"),
535 ('s', "no-mail", "Dinstall::Options::No-Mail"),
536 ('d', "directory", "Dinstall::Options::Directory", "HasArg")]
538 for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
539 "version", "directory"]:
540 key = "Dinstall::Options::%s" % i
541 if key not in cnf:
542 cnf[key] = ""
544 changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
545 Options = cnf.subtree("Dinstall::Options")
547 if Options["Help"]:
548 usage()
550 # -n/--dry-run invalidates some other options which would involve things happening
551 if Options["No-Action"]: 551 ↛ 552line 551 didn't jump to line 552, because the condition on line 551 was never true
552 Options["Automatic"] = ""
554 # Obtain lock if not in no-action mode and initialize the log
555 if not Options["No-Action"]: 555 ↛ 569line 555 didn't jump to line 569, because the condition on line 555 was never false
556 lock_fd = os.open(os.path.join(cnf["Dir::Lock"], 'process-upload.lock'), os.O_RDWR | os.O_CREAT)
557 try:
558 fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
559 except OSError as e:
560 if e.errno in (errno.EACCES, errno.EAGAIN):
561 utils.fubar("Couldn't obtain lock; assuming another 'dak process-upload' is already running.")
562 else:
563 raise
565 # Initialise UrgencyLog() - it will deal with the case where we don't
566 # want to log urgencies
567 urgencylog = UrgencyLog()
569 Logger = daklog.Logger("process-upload", Options["No-Action"])
571 # If we have a directory flag, use it to find our files
572 if cnf["Dinstall::Options::Directory"] != "": 572 ↛ 580line 572 didn't jump to line 580, because the condition on line 572 was never false
573 # Note that we clobber the list of files we were given in this case
574 # so warn if the user has done both
575 if len(changes_files) > 0: 575 ↛ 576line 575 didn't jump to line 576, because the condition on line 575 was never true
576 utils.warn("Directory provided so ignoring files given on command line")
578 changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
579 Logger.log(["Using changes files from directory", cnf["Dinstall::Options::Directory"], len(changes_files)])
580 elif not len(changes_files) > 0:
581 utils.fubar("No changes files given and no directory specified")
582 else:
583 Logger.log(["Using changes files from command-line", len(changes_files)])
585 process_changes(changes_files)
587 if summarystats.accept_count:
588 sets = "set"
589 if summarystats.accept_count > 1:
590 sets = "sets"
591 print("Installed %d package %s, %s." % (summarystats.accept_count, sets,
592 utils.size_type(int(summarystats.accept_bytes))))
593 Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
595 if summarystats.reject_count:
596 sets = "set"
597 if summarystats.reject_count > 1:
598 sets = "sets"
599 print("Rejected %d package %s." % (summarystats.reject_count, sets))
600 Logger.log(["rejected", summarystats.reject_count])
602 if not Options["No-Action"]: 602 ↛ 605line 602 didn't jump to line 605, because the condition on line 602 was never false
603 urgencylog.close()
605 Logger.close()
607###############################################################################
610if __name__ == '__main__':
611 main()