Coverage for dak/process_upload.py: 82%
322 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-08-03 16:46 +0000
1"""
2Checks Debian packages from Incoming
3@contact: Debian FTP Master <ftpmaster@debian.org>
4@copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
5@copyright: 2009 Joerg Jaspert <joerg@debian.org>
6@copyright: 2009 Mark Hymers <mhy@debian.org>
7@copyright: 2009 Frank Lichtenheld <djpig@debian.org>
8@license: GNU General Public License version 2 or later
9"""
11# This program is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation; either version 2 of the License, or
14# (at your option) any later version.
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
21# You should have received a copy of the GNU General Public License
22# along with this program; if not, write to the Free Software
23# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25# based on process-unchecked and process-accepted
27## pu|pa: locking (daily.lock)
28## pu|pa: parse arguments -> list of changes files
29## pa: initialize urgency log
30## pu|pa: sort changes list
32## foreach changes:
33### pa: load dak file
34## pu: copy CHG to tempdir
35## pu: check CHG signature
36## pu: parse changes file
37## pu: checks:
38## pu: check distribution (mappings, rejects)
39## pu: copy FILES to tempdir
40## pu: check whether CHG already exists in CopyChanges
41## pu: check whether FILES already exist in one of the policy queues
42## for deb in FILES:
43## pu: extract control information
44## pu: various checks on control information
45## pu|pa: search for source (in CHG, projectb, policy queues)
46## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
47## pu|pa: check whether deb already exists in the pool
48## for src in FILES:
49## pu: various checks on filenames and CHG consistency
50## pu: if isdsc: check signature
51## for file in FILES:
52## pu: various checks
53## pu: NEW?
54## //pu: check whether file already exists in the pool
55## pu: store what "Component" the package is currently in
56## pu: check whether we found everything we were looking for in CHG
57## pu: check the DSC:
58## pu: check whether we need and have ONE DSC
59## pu: parse the DSC
60## pu: various checks //maybe drop some of the in favor of lintian
61## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
62## pu: check whether DSC_FILES is consistent with "Format"
63## for src in DSC_FILES:
64## pu|pa: check whether file already exists in the pool (with special handling for .orig.tar.gz)
65## pu: create new tempdir
66## pu: create symlink mirror of source
67## pu: unpack source
68## pu: extract changelog information for BTS
69## //pu: create missing .orig symlink
70## pu: check with lintian
71## for file in FILES:
72## pu: check checksums and sizes
73## for file in DSC_FILES:
74## pu: check checksums and sizes
75## pu: CHG: check urgency
76## for deb in FILES:
77## pu: extract contents list and check for dubious timestamps
78## pu: check that the uploader is actually allowed to upload the package
79### pa: install:
80### if stable_install:
81### pa: remove from p-u
82### pa: add to stable
83### pa: move CHG to morgue
84### pa: append data to ChangeLog
85### pa: send mail
86### pa: remove .dak file
87### else:
88### pa: add dsc to db:
89### for file in DSC_FILES:
90### pa: add file to file
91### pa: add file to dsc_files
92### pa: create source entry
93### pa: update source associations
94### pa: update src_uploaders
95### for deb in FILES:
96### pa: add deb to db:
97### pa: add file to file
98### pa: find source entry
99### pa: create binaries entry
100### pa: update binary associations
101### pa: .orig component move
102### pa: move files to pool
103### pa: save CHG
104### pa: move CHG to done/
105### pa: change entry in queue_build
106## pu: use dispatch table to choose target queue:
107## if NEW:
108## pu: write .dak file
109## pu: move to NEW
110## pu: send mail
111## elsif AUTOBYHAND:
112## pu: run autobyhand script
113## pu: if stuff left, do byhand or accept
114## elsif targetqueue in (oldstable, stable, embargo, unembargo):
115## pu: write .dak file
116## pu: check overrides
117## pu: move to queue
118## pu: send mail
119## else:
120## pu: write .dak file
121## pu: move to ACCEPTED
122## pu: send mails
123## pu: create files for BTS
124## pu: create entry in queue_build
125## pu: check overrides
127# Integrity checks
128## GPG
129## Parsing changes (check for duplicates)
130## Parse dsc
131## file list checks
133# New check layout (TODO: Implement)
134## Permission checks
135### suite mappings
136### ACLs
137### version checks (suite)
138### override checks
140## Source checks
141### copy orig
142### unpack
143### BTS changelog
144### src contents
145### lintian
146### urgency log
148## Binary checks
149### timestamps
150### control checks
151### src relation check
152### contents
154## Database insertion (? copy from stuff)
155### BYHAND / NEW / Policy queues
156### Pool
158## Queue builds
160import datetime
161import errno
162import fcntl
163import functools
164import os
165import random
166import sys
167import time
168import traceback
169from collections.abc import Callable, Iterable
170from typing import Concatenate, NoReturn
172import apt_pkg
174import daklib.announce
175import daklib.archive
176import daklib.checks
177import daklib.upload
178from daklib import daklog, utils
179from daklib.config import Config
180from daklib.dbconn import DBConn, SignatureHistory, get_active_keyring_files
181from daklib.regexes import re_default_answer
182from daklib.summarystats import SummaryStats
183from daklib.urgencylog import UrgencyLog
185###############################################################################
187Options: apt_pkg.Configuration
188Logger: daklog.Logger
190###############################################################################
193def usage(exit_code=0) -> NoReturn:
194 print(
195 """Usage: dak process-upload [OPTION]... [CHANGES]...
196 -a, --automatic automatic run
197 -d, --directory <DIR> process uploads in <DIR>
198 -h, --help show this help and exit.
199 --max-duration <D> stop processing after duration (e.g. 10m, 1h 5m)
200 -n, --no-action don't do anything
201 -p, --no-lock don't check lockfile !! for cron.daily only !!
202 -s, --no-mail don't send any mail
203 -V, --version display the version number and exit"""
204 )
205 sys.exit(exit_code)
208###############################################################################
210type Handler[**P, R] = Callable[Concatenate[str, daklib.archive.ArchiveUpload, P], R]
213def try_or_reject[**P, R](function: Handler[P, R]) -> Handler[P, R]:
214 """Try to call function or reject the upload if that fails"""
216 @functools.wraps(function)
217 def wrapper(directory: str, upload: daklib.archive.ArchiveUpload, *args, **kwargs):
218 reason = "No exception caught. This should not happen."
220 try:
221 return function(directory, upload, *args, **kwargs)
222 except (daklib.archive.ArchiveException, daklib.checks.Reject) as e:
223 reason = str(e)
224 except Exception:
225 reason = "There was an uncaught exception when processing your upload:\n{0}\nAny original reject reason follows below.".format(
226 traceback.format_exc()
227 )
229 try:
230 upload.rollback()
231 return real_reject(directory, upload, reason=reason)
232 except Exception:
233 reason = "In addition there was an exception when rejecting the package:\n{0}\nPrevious reasons:\n{1}".format(
234 traceback.format_exc(), reason
235 )
236 upload.rollback()
237 return real_reject(directory, upload, reason=reason, notify=False)
239 raise Exception(
240 "Rejecting upload failed after multiple tries. Giving up. Last reason:\n{0}".format(
241 reason
242 )
243 )
245 return wrapper
248def get_processed_upload(
249 upload: daklib.archive.ArchiveUpload,
250) -> daklib.announce.ProcessedUpload:
251 changes = upload.changes
252 control = upload.changes.changes
254 pu = daklib.announce.ProcessedUpload()
256 pu.maintainer = control.get("Maintainer")
257 pu.changed_by = control.get("Changed-By")
258 pu.fingerprint = changes.primary_fingerprint
259 pu.authorized_by_fingerprint = upload.authorized_by_fingerprint.fingerprint
261 pu.suites = upload.final_suites or []
262 pu.from_policy_suites = []
264 with open(upload.changes.path, "r") as fd:
265 pu.changes = fd.read()
266 pu.changes_filename = upload.changes.filename
267 pu.sourceful = upload.changes.sourceful
268 pu.source = control.get("Source")
269 pu.version = control.get("Version")
270 pu.architecture = control.get("Architecture")
271 pu.bugs = changes.closed_bugs
273 pu.program = "process-upload"
275 pu.warnings = upload.warnings
277 return pu
280@try_or_reject
281def accept(directory: str, upload: daklib.archive.ArchiveUpload) -> None:
282 cnf = Config()
284 Logger.log(["ACCEPT", upload.changes.filename])
285 print("ACCEPT")
287 upload.install()
288 utils.process_buildinfos(
289 upload.directory, upload.changes.buildinfo_files, upload.transaction.fs, Logger
290 )
292 assert upload.final_suites is not None
293 accepted_to_real_suite = any(
294 suite.policy_queue is None for suite in upload.final_suites
295 )
296 sourceful_upload = upload.changes.sourceful
298 control = upload.changes.changes
299 if sourceful_upload and not Options["No-Action"]:
300 urgency = control.get("Urgency")
301 # As per policy 5.6.17, the urgency can be followed by a space and a
302 # comment. Extract only the urgency from the string.
303 if " " in urgency: 303 ↛ 304line 303 didn't jump to line 304 because the condition on line 303 was never true
304 urgency, comment = urgency.split(" ", 1)
305 if urgency not in cnf.value_list("Urgency::Valid"): 305 ↛ 306line 305 didn't jump to line 306 because the condition on line 305 was never true
306 urgency = cnf["Urgency::Default"]
307 UrgencyLog().log(control["Source"], control["Version"], urgency)
309 pu = get_processed_upload(upload)
310 daklib.announce.announce_accept(pu)
312 # Move .changes to done, but only for uploads that were accepted to a
313 # real suite. process-policy will handle this for uploads to queues.
314 if accepted_to_real_suite:
315 src = os.path.join(upload.directory, upload.changes.filename)
317 now = datetime.datetime.now()
318 donedir = os.path.join(cnf["Dir::Done"], now.strftime("%Y/%m/%d"))
319 dst = os.path.join(donedir, upload.changes.filename)
320 dst = utils.find_next_free(dst)
322 upload.transaction.fs.copy(src, dst, mode=0o644)
324 SummaryStats().accept_count += 1
325 SummaryStats().accept_bytes += upload.changes.bytes
328@try_or_reject
329def accept_to_new(directory: str, upload: daklib.archive.ArchiveUpload) -> None:
331 Logger.log(["ACCEPT-TO-NEW", upload.changes.filename])
332 print("ACCEPT-TO-NEW")
334 upload.install_to_new()
335 # TODO: tag bugs pending
337 pu = get_processed_upload(upload)
338 daklib.announce.announce_new(pu)
340 SummaryStats().accept_count += 1
341 SummaryStats().accept_bytes += upload.changes.bytes
344@try_or_reject
345def reject(
346 directory: str,
347 upload: daklib.archive.ArchiveUpload,
348 reason: str | None = None,
349 notify=True,
350) -> None:
351 real_reject(directory, upload, reason, notify)
354def real_reject(
355 directory: str,
356 upload: daklib.archive.ArchiveUpload,
357 reason: str | None = None,
358 notify=True,
359) -> None:
360 # XXX: rejection itself should go to daklib.archive.ArchiveUpload
361 cnf = Config()
363 Logger.log(["REJECT", upload.changes.filename])
364 print("REJECT")
366 fs = upload.transaction.fs
367 rejectdir = cnf["Dir::Reject"]
369 files = [f.filename for f in upload.changes.files.values()]
370 files.append(upload.changes.filename)
372 for fn in files:
373 src = os.path.join(upload.directory, fn)
374 dst = utils.find_next_free(os.path.join(rejectdir, fn))
375 if not os.path.exists(src):
376 continue
377 fs.copy(src, dst)
379 if upload.reject_reasons is not None: 379 ↛ 384line 379 didn't jump to line 384 because the condition on line 379 was always true
380 if reason is None: 380 ↛ 382line 380 didn't jump to line 382 because the condition on line 380 was always true
381 reason = ""
382 reason = reason + "\n" + "\n".join(upload.reject_reasons)
384 if reason is None: 384 ↛ 385line 384 didn't jump to line 385 because the condition on line 384 was never true
385 reason = "(Unknown reason. Please check logs.)"
387 dst = utils.find_next_free(
388 os.path.join(rejectdir, "{0}.reason".format(upload.changes.filename))
389 )
390 fh = fs.create(dst)
391 fh.write(reason)
392 fh.close()
394 if notify: 394 ↛ 398line 394 didn't jump to line 398 because the condition on line 394 was always true
395 pu = get_processed_upload(upload)
396 daklib.announce.announce_reject(pu, reason)
398 SummaryStats().reject_count += 1
401###############################################################################
404def action(directory: str, upload: daklib.archive.ArchiveUpload) -> bool:
405 changes = upload.changes
406 processed = True
408 global Logger
410 cnf = Config()
412 okay = upload.check()
414 try:
415 summary = changes.changes.get("Changes", "")
416 except UnicodeDecodeError as e:
417 summary = "Reading changes failed: %s" % (e)
418 # the upload checks should have detected this, but make sure this
419 # upload gets rejected in any case
420 upload.reject_reasons.append(summary)
422 package_info = []
423 if okay:
424 if changes.source is not None:
425 package_info.append("source:{0}".format(changes.source.dsc["Source"]))
426 for binary in changes.binaries:
427 package_info.append("binary:{0}".format(binary.control["Package"]))
429 (prompt, answer) = ("", "XXX")
430 if Options["No-Action"] or Options["Automatic"]: 430 ↛ 433line 430 didn't jump to line 433 because the condition on line 430 was always true
431 answer = "S"
433 print(summary)
434 print()
435 print("\n".join(package_info))
436 print()
437 if len(upload.warnings) > 0:
438 print("\n".join(upload.warnings))
439 print()
441 if len(upload.reject_reasons) > 0:
442 print("Reason:")
443 print("\n".join(upload.reject_reasons))
444 print()
446 path = os.path.join(directory, changes.filename)
447 created = os.stat(path).st_mtime
448 now = time.time()
449 too_new = now - created < int(cnf["Dinstall::SkipTime"])
451 if too_new:
452 print("SKIP (too new)")
453 prompt = "[S]kip, Quit ?"
454 else:
455 prompt = "[R]eject, Skip, Quit ?"
456 if Options["Automatic"]: 456 ↛ 467line 456 didn't jump to line 467 because the condition on line 456 was always true
457 answer = "R"
458 elif upload.new:
459 prompt = "[N]ew, Skip, Quit ?"
460 if Options["Automatic"]: 460 ↛ 467line 460 didn't jump to line 467 because the condition on line 460 was always true
461 answer = "N"
462 else:
463 prompt = "[A]ccept, Skip, Quit ?"
464 if Options["Automatic"]: 464 ↛ 467line 464 didn't jump to line 467 because the condition on line 464 was always true
465 answer = "A"
467 while prompt.find(answer) == -1: 467 ↛ 468line 467 didn't jump to line 468 because the condition on line 467 was never true
468 answer = utils.input_or_exit(prompt)
469 m = re_default_answer.match(prompt)
470 if answer == "":
471 assert m is not None
472 answer = m.group(1)
473 answer = answer[:1].upper()
475 if answer == "R":
476 reject(directory, upload)
477 elif answer == "A":
478 # upload.try_autobyhand must not be run with No-Action.
479 if Options["No-Action"]: 479 ↛ 480line 479 didn't jump to line 480 because the condition on line 479 was never true
480 accept(directory, upload)
481 elif upload.try_autobyhand(): 481 ↛ 484line 481 didn't jump to line 484 because the condition on line 481 was always true
482 accept(directory, upload)
483 else:
484 print("W: redirecting to BYHAND as automatic processing failed.")
485 accept_to_new(directory, upload)
486 elif answer == "N":
487 accept_to_new(directory, upload)
488 elif answer == "Q": 488 ↛ 489line 488 didn't jump to line 489 because the condition on line 488 was never true
489 sys.exit(0)
490 elif answer == "S": 490 ↛ 493line 490 didn't jump to line 493 because the condition on line 490 was always true
491 processed = False
493 if not Options["No-Action"]: 493 ↛ 496line 493 didn't jump to line 496 because the condition on line 493 was always true
494 upload.commit()
496 return processed
499###############################################################################
502def unlink_if_exists(path: str) -> None:
503 try:
504 os.unlink(path)
505 except OSError as e:
506 if e.errno != errno.ENOENT: 506 ↛ 507line 506 didn't jump to line 507 because the condition on line 506 was never true
507 raise
510def process_it(
511 directory: str, changes: daklib.upload.Changes, keyrings: list[str]
512) -> None:
513 global Logger
515 print("\n{0}\n".format(changes.filename))
516 Logger.log(["Processing changes file", changes.filename])
518 with daklib.archive.ArchiveUpload(directory, changes, keyrings) as upload:
519 processed = action(directory, upload)
520 if processed and not Options["No-Action"]:
521 session = DBConn().session()
522 history = SignatureHistory.from_signed_file(upload.changes)
523 if history.query(session) is None: 523 ↛ 526line 523 didn't jump to line 526 because the condition on line 523 was always true
524 session.add(history)
525 session.commit()
526 session.close()
528 unlink_if_exists(os.path.join(directory, changes.filename))
529 for fn in changes.files:
530 unlink_if_exists(os.path.join(directory, fn))
533###############################################################################
536def _source_group(source: str) -> str:
537 """decide group for given source name
539 This is mostly for Secure Boot signing where "X" should be
540 processed in the same group as (and before) "X-signed-*".
542 As a further special case, "grub2" needs to be processed in the
543 same group as (and before) "grub-efi-*-signed".
544 """
546 group = source.split("-", 1)[0]
547 if group == "grub2": 547 ↛ 548line 547 didn't jump to line 548 because the condition on line 547 was never true
548 return "grub"
549 return group
552def _group_changes_by_source_and_shuffle(
553 changes: list[tuple[str, daklib.upload.Changes]],
554) -> list[tuple[str, daklib.upload.Changes]]:
555 """Group changes by Source, sort each group, and shuffle group order."""
556 grouped: dict[str, list[tuple[str, daklib.upload.Changes]]] = {}
557 for directory, change in changes:
558 source = _source_group(change.changes.get("Source", ""))
559 grouped.setdefault(source, []).append((directory, change))
561 for group in grouped.values():
562 group.sort(key=lambda item: item[1])
564 source_names = list(grouped)
565 random.shuffle(source_names)
566 return [item for source in source_names for item in grouped[source]]
569###############################################################################
572def process_changes(
573 changes_filenames: Iterable[str],
574 max_duration: datetime.timedelta | None = None,
575) -> None:
576 deadline: float | None = None
577 if max_duration is not None: 577 ↛ 578line 577 didn't jump to line 578 because the condition on line 577 was never true
578 deadline = time.monotonic() + max_duration.total_seconds()
580 session = DBConn().session()
581 keyring_files = get_active_keyring_files(session)
582 session.close()
584 changes = []
585 for fn in changes_filenames:
586 try:
587 directory, filename = os.path.split(fn)
588 c = daklib.upload.Changes(directory, filename, keyring_files)
589 changes.append((directory, c))
590 except Exception as e:
591 try:
592 Logger.log(
593 [
594 filename,
595 "Error while loading changes file {0}: {1}".format(fn, e),
596 ]
597 )
598 except Exception as e:
599 Logger.log(
600 [
601 filename,
602 "Error while loading changes file {0}, with additional error while printing exception: {1}".format(
603 fn, repr(e)
604 ),
605 ]
606 )
608 changes = _group_changes_by_source_and_shuffle(changes)
610 for directory, c in changes:
611 if deadline is not None and time.monotonic() >= deadline: 611 ↛ 612line 611 didn't jump to line 612 because the condition on line 611 was never true
612 Logger.log(["Max duration reached; stopping processing loop"])
613 break
614 process_it(directory, c, keyring_files)
617###############################################################################
620def main() -> None:
621 global Options, Logger
623 cnf = Config()
624 summarystats = SummaryStats()
626 Arguments = [
627 ("a", "automatic", "Dinstall::Options::Automatic"),
628 ("h", "help", "Dinstall::Options::Help"),
629 ("\0", "max-duration", "Dinstall::Options::Max-Duration", "HasArg"),
630 ("n", "no-action", "Dinstall::Options::No-Action"),
631 ("p", "no-lock", "Dinstall::Options::No-Lock"),
632 ("s", "no-mail", "Dinstall::Options::No-Mail"),
633 ("d", "directory", "Dinstall::Options::Directory", "HasArg"),
634 ]
636 for i in [
637 "automatic",
638 "help",
639 "max-duration",
640 "no-action",
641 "no-lock",
642 "no-mail",
643 "version",
644 "directory",
645 ]:
646 key = "Dinstall::Options::%s" % i
647 if key not in cnf:
648 cnf[key] = ""
650 changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) # type: ignore[attr-defined]
651 Options = cnf.subtree("Dinstall::Options")
653 if Options["Help"]:
654 usage()
656 # -n/--dry-run invalidates some other options which would involve things happening
657 if Options["No-Action"]: 657 ↛ 658line 657 didn't jump to line 658 because the condition on line 657 was never true
658 Options["Automatic"] = "" # type: ignore[index]
660 # Obtain lock if not in no-action mode and initialize the log
661 if not Options["No-Action"]: 661 ↛ 680line 661 didn't jump to line 680 because the condition on line 661 was always true
662 lock_fd = os.open(
663 os.path.join(cnf["Dir::Lock"], "process-upload.lock"),
664 os.O_RDWR | os.O_CREAT,
665 )
666 try:
667 fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
668 except OSError as e:
669 if e.errno in (errno.EACCES, errno.EAGAIN):
670 utils.fubar(
671 "Couldn't obtain lock; assuming another 'dak process-upload' is already running."
672 )
673 else:
674 raise
676 # Initialise UrgencyLog() - it will deal with the case where we don't
677 # want to log urgencies
678 urgencylog = UrgencyLog()
680 Logger = daklog.Logger("process-upload", Options["No-Action"])
682 # If we have a directory flag, use it to find our files
683 if cnf["Dinstall::Options::Directory"] != "": 683 ↛ 697line 683 didn't jump to line 697 because the condition on line 683 was always true
684 # Note that we clobber the list of files we were given in this case
685 # so warn if the user has done both
686 if len(changes_files) > 0: 686 ↛ 687line 686 didn't jump to line 687 because the condition on line 686 was never true
687 utils.warn("Directory provided so ignoring files given on command line")
689 changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
690 Logger.log(
691 [
692 "Using changes files from directory",
693 cnf["Dinstall::Options::Directory"],
694 len(changes_files),
695 ]
696 )
697 elif not len(changes_files) > 0:
698 utils.fubar("No changes files given and no directory specified")
699 else:
700 Logger.log(["Using changes files from command-line", len(changes_files)])
702 max_duration = None
703 if Options["Max-Duration"]: 703 ↛ 704line 703 didn't jump to line 704 because the condition on line 703 was never true
704 try:
705 max_duration = utils.parse_duration(Options["Max-Duration"])
706 except ValueError as e:
707 utils.fubar("Invalid --max-duration: %s" % e)
709 process_changes(changes_files, max_duration=max_duration)
711 if summarystats.accept_count:
712 sets = "set"
713 if summarystats.accept_count > 1:
714 sets = "sets"
715 print(
716 "Installed %d package %s, %s."
717 % (
718 summarystats.accept_count,
719 sets,
720 utils.size_type(int(summarystats.accept_bytes)),
721 )
722 )
723 Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
725 if summarystats.reject_count:
726 sets = "set"
727 if summarystats.reject_count > 1:
728 sets = "sets"
729 print("Rejected %d package %s." % (summarystats.reject_count, sets))
730 Logger.log(["rejected", summarystats.reject_count])
732 if not Options["No-Action"]: 732 ↛ 735line 732 didn't jump to line 735 because the condition on line 732 was always true
733 urgencylog.close()
735 Logger.close()