1#! /usr/bin/env python3
3"""
4Checks Debian packages from Incoming
5@contact: Debian FTP Master <ftpmaster@debian.org>
6@copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
7@copyright: 2009 Joerg Jaspert <joerg@debian.org>
8@copyright: 2009 Mark Hymers <mhy@debian.org>
9@copyright: 2009 Frank Lichtenheld <djpig@debian.org>
10@license: GNU General Public License version 2 or later
11"""
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27# based on process-unchecked and process-accepted
29## pu|pa: locking (daily.lock)
30## pu|pa: parse arguments -> list of changes files
31## pa: initialize urgency log
32## pu|pa: sort changes list
34## foreach changes:
35### pa: load dak file
36## pu: copy CHG to tempdir
37## pu: check CHG signature
38## pu: parse changes file
39## pu: checks:
40## pu: check distribution (mappings, rejects)
41## pu: copy FILES to tempdir
42## pu: check whether CHG already exists in CopyChanges
43## pu: check whether FILES already exist in one of the policy queues
44## for deb in FILES:
45## pu: extract control information
46## pu: various checks on control information
47## pu|pa: search for source (in CHG, projectb, policy queues)
48## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
49## pu|pa: check whether deb already exists in the pool
50## for src in FILES:
51## pu: various checks on filenames and CHG consistency
52## pu: if isdsc: check signature
53## for file in FILES:
54## pu: various checks
55## pu: NEW?
56## //pu: check whether file already exists in the pool
57## pu: store what "Component" the package is currently in
58## pu: check whether we found everything we were looking for in CHG
59## pu: check the DSC:
60## pu: check whether we need and have ONE DSC
61## pu: parse the DSC
62## pu: various checks //maybe drop some of the in favor of lintian
63## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
64## pu: check whether DSC_FILES is consistent with "Format"
65## for src in DSC_FILES:
66## pu|pa: check whether file already exists in the pool (with special handling for .orig.tar.gz)
67## pu: create new tempdir
68## pu: create symlink mirror of source
69## pu: unpack source
70## pu: extract changelog information for BTS
71## //pu: create missing .orig symlink
72## pu: check with lintian
73## for file in FILES:
74## pu: check checksums and sizes
75## for file in DSC_FILES:
76## pu: check checksums and sizes
77## pu: CHG: check urgency
78## for deb in FILES:
79## pu: extract contents list and check for dubious timestamps
80## pu: check that the uploader is actually allowed to upload the package
81### pa: install:
82### if stable_install:
83### pa: remove from p-u
84### pa: add to stable
85### pa: move CHG to morgue
86### pa: append data to ChangeLog
87### pa: send mail
88### pa: remove .dak file
89### else:
90### pa: add dsc to db:
91### for file in DSC_FILES:
92### pa: add file to file
93### pa: add file to dsc_files
94### pa: create source entry
95### pa: update source associations
96### pa: update src_uploaders
97### for deb in FILES:
98### pa: add deb to db:
99### pa: add file to file
100### pa: find source entry
101### pa: create binaries entry
102### pa: update binary associations
103### pa: .orig component move
104### pa: move files to pool
105### pa: save CHG
106### pa: move CHG to done/
107### pa: change entry in queue_build
108## pu: use dispatch table to choose target queue:
109## if NEW:
110## pu: write .dak file
111## pu: move to NEW
112## pu: send mail
113## elsif AUTOBYHAND:
114## pu: run autobyhand script
115## pu: if stuff left, do byhand or accept
116## elsif targetqueue in (oldstable, stable, embargo, unembargo):
117## pu: write .dak file
118## pu: check overrides
119## pu: move to queue
120## pu: send mail
121## else:
122## pu: write .dak file
123## pu: move to ACCEPTED
124## pu: send mails
125## pu: create files for BTS
126## pu: create entry in queue_build
127## pu: check overrides
129# Integrity checks
130## GPG
131## Parsing changes (check for duplicates)
132## Parse dsc
133## file list checks
135# New check layout (TODO: Implement)
136## Permission checks
137### suite mappings
138### ACLs
139### version checks (suite)
140### override checks
142## Source checks
143### copy orig
144### unpack
145### BTS changelog
146### src contents
147### lintian
148### urgency log
150## Binary checks
151### timestamps
152### control checks
153### src relation check
154### contents
156## Database insertion (? copy from stuff)
157### BYHAND / NEW / Policy queues
158### Pool
160## Queue builds
162import datetime
163import errno
164import fcntl
165import functools
166import os
167import sys
168import time
169import traceback
170from collections.abc import Iterable
171from typing import NoReturn
173import apt_pkg
175import daklib.announce
176import daklib.archive
177import daklib.checks
178import daklib.upload
179import daklib.utils as utils
180from daklib import daklog
181from daklib.config import Config
182from daklib.dbconn import DBConn, Keyring, SignatureHistory
183from daklib.regexes import re_default_answer
184from daklib.summarystats import SummaryStats
185from daklib.urgencylog import UrgencyLog
187###############################################################################
189Options = None
190Logger = None
192###############################################################################
195def usage(exit_code=0) -> NoReturn:
196 print(
197 """Usage: dak process-upload [OPTION]... [CHANGES]...
198 -a, --automatic automatic run
199 -d, --directory <DIR> process uploads in <DIR>
200 -h, --help show this help and exit.
201 -n, --no-action don't do anything
202 -p, --no-lock don't check lockfile !! for cron.daily only !!
203 -s, --no-mail don't send any mail
204 -V, --version display the version number and exit"""
205 )
206 sys.exit(exit_code)
209###############################################################################
212def try_or_reject(function):
213 """Try to call function or reject the upload if that fails"""
215 @functools.wraps(function)
216 def wrapper(directory: str, upload: daklib.archive.ArchiveUpload, *args, **kwargs):
217 reason = "No exception caught. This should not happen."
219 try:
220 return function(directory, upload, *args, **kwargs)
221 except (daklib.archive.ArchiveException, daklib.checks.Reject) as e:
222 reason = str(e)
223 except Exception:
224 reason = "There was an uncaught exception when processing your upload:\n{0}\nAny original reject reason follows below.".format(
225 traceback.format_exc()
226 )
228 try:
229 upload.rollback()
230 return real_reject(directory, upload, reason=reason)
231 except Exception:
232 reason = "In addition there was an exception when rejecting the package:\n{0}\nPrevious reasons:\n{1}".format(
233 traceback.format_exc(), reason
234 )
235 upload.rollback()
236 return real_reject(directory, upload, reason=reason, notify=False)
238 raise Exception(
239 "Rejecting upload failed after multiple tries. Giving up. Last reason:\n{0}".format(
240 reason
241 )
242 )
244 return wrapper
247def get_processed_upload(
248 upload: daklib.archive.ArchiveUpload,
249) -> daklib.announce.ProcessedUpload:
250 changes = upload.changes
251 control = upload.changes.changes
253 pu = daklib.announce.ProcessedUpload()
255 pu.maintainer = control.get("Maintainer")
256 pu.changed_by = control.get("Changed-By")
257 pu.fingerprint = changes.primary_fingerprint
259 pu.suites = upload.final_suites or []
260 pu.from_policy_suites = []
262 with open(upload.changes.path, "r") as fd:
263 pu.changes = fd.read()
264 pu.changes_filename = upload.changes.filename
265 pu.sourceful = upload.changes.sourceful
266 pu.source = control.get("Source")
267 pu.version = control.get("Version")
268 pu.architecture = control.get("Architecture")
269 pu.bugs = changes.closed_bugs
271 pu.program = "process-upload"
273 pu.warnings = upload.warnings
275 return pu
278@try_or_reject
279def accept(directory: str, upload: daklib.archive.ArchiveUpload) -> None:
280 cnf = Config()
282 Logger.log(["ACCEPT", upload.changes.filename])
283 print("ACCEPT")
285 upload.install()
286 utils.process_buildinfos(
287 upload.directory, upload.changes.buildinfo_files, upload.transaction.fs, Logger
288 )
290 accepted_to_real_suite = any(
291 suite.policy_queue is None for suite in upload.final_suites
292 )
293 sourceful_upload = upload.changes.sourceful
295 control = upload.changes.changes
296 if sourceful_upload and not Options["No-Action"]:
297 urgency = control.get("Urgency")
298 # As per policy 5.6.17, the urgency can be followed by a space and a
299 # comment. Extract only the urgency from the string.
300 if " " in urgency: 300 ↛ 301line 300 didn't jump to line 301, because the condition on line 300 was never true
301 urgency, comment = urgency.split(" ", 1)
302 if urgency not in cnf.value_list("Urgency::Valid"): 302 ↛ 303line 302 didn't jump to line 303, because the condition on line 302 was never true
303 urgency = cnf["Urgency::Default"]
304 UrgencyLog().log(control["Source"], control["Version"], urgency)
306 pu = get_processed_upload(upload)
307 daklib.announce.announce_accept(pu)
309 # Move .changes to done, but only for uploads that were accepted to a
310 # real suite. process-policy will handle this for uploads to queues.
311 if accepted_to_real_suite:
312 src = os.path.join(upload.directory, upload.changes.filename)
314 now = datetime.datetime.now()
315 donedir = os.path.join(cnf["Dir::Done"], now.strftime("%Y/%m/%d"))
316 dst = os.path.join(donedir, upload.changes.filename)
317 dst = utils.find_next_free(dst)
319 upload.transaction.fs.copy(src, dst, mode=0o644)
321 SummaryStats().accept_count += 1
322 SummaryStats().accept_bytes += upload.changes.bytes
325@try_or_reject
326def accept_to_new(directory: str, upload: daklib.archive.ArchiveUpload) -> None:
328 Logger.log(["ACCEPT-TO-NEW", upload.changes.filename])
329 print("ACCEPT-TO-NEW")
331 upload.install_to_new()
332 # TODO: tag bugs pending
334 pu = get_processed_upload(upload)
335 daklib.announce.announce_new(pu)
337 SummaryStats().accept_count += 1
338 SummaryStats().accept_bytes += upload.changes.bytes
341@try_or_reject
342def reject(
343 directory: str, upload: daklib.archive.ArchiveUpload, reason=None, notify=True
344) -> None:
345 real_reject(directory, upload, reason, notify)
348def real_reject(
349 directory: str, upload: daklib.archive.ArchiveUpload, reason=None, notify=True
350) -> None:
351 # XXX: rejection itself should go to daklib.archive.ArchiveUpload
352 cnf = Config()
354 Logger.log(["REJECT", upload.changes.filename])
355 print("REJECT")
357 fs = upload.transaction.fs
358 rejectdir = cnf["Dir::Reject"]
360 files = [f.filename for f in upload.changes.files.values()]
361 files.append(upload.changes.filename)
363 for fn in files:
364 src = os.path.join(upload.directory, fn)
365 dst = utils.find_next_free(os.path.join(rejectdir, fn))
366 if not os.path.exists(src):
367 continue
368 fs.copy(src, dst)
370 if upload.reject_reasons is not None: 370 ↛ 375line 370 didn't jump to line 375, because the condition on line 370 was never false
371 if reason is None: 371 ↛ 373line 371 didn't jump to line 373, because the condition on line 371 was never false
372 reason = ""
373 reason = reason + "\n" + "\n".join(upload.reject_reasons)
375 if reason is None: 375 ↛ 376line 375 didn't jump to line 376, because the condition on line 375 was never true
376 reason = "(Unknown reason. Please check logs.)"
378 dst = utils.find_next_free(
379 os.path.join(rejectdir, "{0}.reason".format(upload.changes.filename))
380 )
381 fh = fs.create(dst)
382 fh.write(reason)
383 fh.close()
385 if notify: 385 ↛ 389line 385 didn't jump to line 389, because the condition on line 385 was never false
386 pu = get_processed_upload(upload)
387 daklib.announce.announce_reject(pu, reason)
389 SummaryStats().reject_count += 1
392###############################################################################
395def action(directory: str, upload: daklib.archive.ArchiveUpload) -> bool:
396 changes = upload.changes
397 processed = True
399 global Logger
401 cnf = Config()
403 okay = upload.check()
405 try:
406 summary = changes.changes.get("Changes", "")
407 except UnicodeDecodeError as e:
408 summary = "Reading changes failed: %s" % (e)
409 # the upload checks should have detected this, but make sure this
410 # upload gets rejected in any case
411 upload.reject_reasons.append(summary)
413 package_info = []
414 if okay:
415 if changes.source is not None:
416 package_info.append("source:{0}".format(changes.source.dsc["Source"]))
417 for binary in changes.binaries:
418 package_info.append("binary:{0}".format(binary.control["Package"]))
420 (prompt, answer) = ("", "XXX")
421 if Options["No-Action"] or Options["Automatic"]: 421 ↛ 424line 421 didn't jump to line 424, because the condition on line 421 was never false
422 answer = "S"
424 print(summary)
425 print()
426 print("\n".join(package_info))
427 print()
428 if len(upload.warnings) > 0:
429 print("\n".join(upload.warnings))
430 print()
432 if len(upload.reject_reasons) > 0:
433 print("Reason:")
434 print("\n".join(upload.reject_reasons))
435 print()
437 path = os.path.join(directory, changes.filename)
438 created = os.stat(path).st_mtime
439 now = time.time()
440 too_new = now - created < int(cnf["Dinstall::SkipTime"])
442 if too_new:
443 print("SKIP (too new)")
444 prompt = "[S]kip, Quit ?"
445 else:
446 prompt = "[R]eject, Skip, Quit ?"
447 if Options["Automatic"]: 447 ↛ 458line 447 didn't jump to line 458, because the condition on line 447 was never false
448 answer = "R"
449 elif upload.new:
450 prompt = "[N]ew, Skip, Quit ?"
451 if Options["Automatic"]: 451 ↛ 458line 451 didn't jump to line 458, because the condition on line 451 was never false
452 answer = "N"
453 else:
454 prompt = "[A]ccept, Skip, Quit ?"
455 if Options["Automatic"]: 455 ↛ 458line 455 didn't jump to line 458, because the condition on line 455 was never false
456 answer = "A"
458 while prompt.find(answer) == -1: 458 ↛ 459line 458 didn't jump to line 459, because the condition on line 458 was never true
459 answer = utils.input_or_exit(prompt)
460 m = re_default_answer.match(prompt)
461 if answer == "":
462 answer = m.group(1)
463 answer = answer[:1].upper()
465 if answer == "R":
466 reject(directory, upload)
467 elif answer == "A":
468 # upload.try_autobyhand must not be run with No-Action.
469 if Options["No-Action"]: 469 ↛ 470line 469 didn't jump to line 470, because the condition on line 469 was never true
470 accept(directory, upload)
471 elif upload.try_autobyhand(): 471 ↛ 474line 471 didn't jump to line 474, because the condition on line 471 was never false
472 accept(directory, upload)
473 else:
474 print("W: redirecting to BYHAND as automatic processing failed.")
475 accept_to_new(directory, upload)
476 elif answer == "N":
477 accept_to_new(directory, upload)
478 elif answer == "Q": 478 ↛ 479line 478 didn't jump to line 479, because the condition on line 478 was never true
479 sys.exit(0)
480 elif answer == "S": 480 ↛ 483line 480 didn't jump to line 483, because the condition on line 480 was never false
481 processed = False
483 if not Options["No-Action"]: 483 ↛ 486line 483 didn't jump to line 486, because the condition on line 483 was never false
484 upload.commit()
486 return processed
489###############################################################################
492def unlink_if_exists(path: str) -> None:
493 try:
494 os.unlink(path)
495 except OSError as e:
496 if e.errno != errno.ENOENT: 496 ↛ 497line 496 didn't jump to line 497, because the condition on line 496 was never true
497 raise
500def process_it(
501 directory: str, changes: daklib.upload.Changes, keyrings: list[str]
502) -> None:
503 global Logger
505 print("\n{0}\n".format(changes.filename))
506 Logger.log(["Processing changes file", changes.filename])
508 with daklib.archive.ArchiveUpload(directory, changes, keyrings) as upload:
509 processed = action(directory, upload)
510 if processed and not Options["No-Action"]:
511 session = DBConn().session()
512 history = SignatureHistory.from_signed_file(upload.changes)
513 if history.query(session) is None: 513 ↛ 516line 513 didn't jump to line 516, because the condition on line 513 was never false
514 session.add(history)
515 session.commit()
516 session.close()
518 unlink_if_exists(os.path.join(directory, changes.filename))
519 for fn in changes.files:
520 unlink_if_exists(os.path.join(directory, fn))
523###############################################################################
526def process_changes(changes_filenames: Iterable[str]):
527 session = DBConn().session()
528 keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority)
529 keyring_files = [k.keyring_name for k in keyrings]
530 session.close()
532 changes = []
533 for fn in changes_filenames:
534 try:
535 directory, filename = os.path.split(fn)
536 c = daklib.upload.Changes(directory, filename, keyring_files)
537 changes.append([directory, c])
538 except Exception as e:
539 try:
540 Logger.log(
541 [
542 filename,
543 "Error while loading changes file {0}: {1}".format(fn, e),
544 ]
545 )
546 except Exception as e:
547 Logger.log(
548 [
549 filename,
550 "Error while loading changes file {0}, with additional error while printing exception: {1}".format(
551 fn, repr(e)
552 ),
553 ]
554 )
556 changes.sort(key=lambda x: x[1])
558 for directory, c in changes:
559 process_it(directory, c, keyring_files)
562###############################################################################
565def main():
566 global Options, Logger
568 cnf = Config()
569 summarystats = SummaryStats()
571 Arguments = [
572 ("a", "automatic", "Dinstall::Options::Automatic"),
573 ("h", "help", "Dinstall::Options::Help"),
574 ("n", "no-action", "Dinstall::Options::No-Action"),
575 ("p", "no-lock", "Dinstall::Options::No-Lock"),
576 ("s", "no-mail", "Dinstall::Options::No-Mail"),
577 ("d", "directory", "Dinstall::Options::Directory", "HasArg"),
578 ]
580 for i in [
581 "automatic",
582 "help",
583 "no-action",
584 "no-lock",
585 "no-mail",
586 "version",
587 "directory",
588 ]:
589 key = "Dinstall::Options::%s" % i
590 if key not in cnf:
591 cnf[key] = ""
593 changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
594 Options = cnf.subtree("Dinstall::Options")
596 if Options["Help"]:
597 usage()
599 # -n/--dry-run invalidates some other options which would involve things happening
600 if Options["No-Action"]: 600 ↛ 601line 600 didn't jump to line 601, because the condition on line 600 was never true
601 Options["Automatic"] = ""
603 # Obtain lock if not in no-action mode and initialize the log
604 if not Options["No-Action"]: 604 ↛ 623line 604 didn't jump to line 623, because the condition on line 604 was never false
605 lock_fd = os.open(
606 os.path.join(cnf["Dir::Lock"], "process-upload.lock"),
607 os.O_RDWR | os.O_CREAT,
608 )
609 try:
610 fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
611 except OSError as e:
612 if e.errno in (errno.EACCES, errno.EAGAIN):
613 utils.fubar(
614 "Couldn't obtain lock; assuming another 'dak process-upload' is already running."
615 )
616 else:
617 raise
619 # Initialise UrgencyLog() - it will deal with the case where we don't
620 # want to log urgencies
621 urgencylog = UrgencyLog()
623 Logger = daklog.Logger("process-upload", Options["No-Action"])
625 # If we have a directory flag, use it to find our files
626 if cnf["Dinstall::Options::Directory"] != "": 626 ↛ 646line 626 didn't jump to line 646, because the condition on line 626 was never false
627 # Note that we clobber the list of files we were given in this case
628 # so warn if the user has done both
629 if len(changes_files) > 0: 629 ↛ 630line 629 didn't jump to line 630, because the condition on line 629 was never true
630 utils.warn("Directory provided so ignoring files given on command line")
632 changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
633 # FIXME: quick hack to NOT have p-u run for ages, if some binnmu fun uploads thousands of changes
634 # might want to make the number configurable at some point
635 if len(changes_files) > 200: 635 ↛ 636line 635 didn't jump to line 636, because the condition on line 635 was never true
636 import random
638 changes_files = random.sample(changes_files, k=200)
639 Logger.log(
640 [
641 "Using changes files from directory",
642 cnf["Dinstall::Options::Directory"],
643 len(changes_files),
644 ]
645 )
646 elif not len(changes_files) > 0:
647 utils.fubar("No changes files given and no directory specified")
648 else:
649 Logger.log(["Using changes files from command-line", len(changes_files)])
651 process_changes(changes_files)
653 if summarystats.accept_count:
654 sets = "set"
655 if summarystats.accept_count > 1:
656 sets = "sets"
657 print(
658 "Installed %d package %s, %s."
659 % (
660 summarystats.accept_count,
661 sets,
662 utils.size_type(int(summarystats.accept_bytes)),
663 )
664 )
665 Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
667 if summarystats.reject_count:
668 sets = "set"
669 if summarystats.reject_count > 1:
670 sets = "sets"
671 print("Rejected %d package %s." % (summarystats.reject_count, sets))
672 Logger.log(["rejected", summarystats.reject_count])
674 if not Options["No-Action"]: 674 ↛ 677line 674 didn't jump to line 677, because the condition on line 674 was never false
675 urgencylog.close()
677 Logger.close()
680###############################################################################
683if __name__ == "__main__":
684 main()