Coverage for dak/process_upload.py: 82%
319 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-02-10 22:10 +0000
1#! /usr/bin/env python3
3"""
4Checks Debian packages from Incoming
5@contact: Debian FTP Master <ftpmaster@debian.org>
6@copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
7@copyright: 2009 Joerg Jaspert <joerg@debian.org>
8@copyright: 2009 Mark Hymers <mhy@debian.org>
9@copyright: 2009 Frank Lichtenheld <djpig@debian.org>
10@license: GNU General Public License version 2 or later
11"""
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27# based on process-unchecked and process-accepted
29## pu|pa: locking (daily.lock)
30## pu|pa: parse arguments -> list of changes files
31## pa: initialize urgency log
32## pu|pa: sort changes list
34## foreach changes:
35### pa: load dak file
36## pu: copy CHG to tempdir
37## pu: check CHG signature
38## pu: parse changes file
39## pu: checks:
40## pu: check distribution (mappings, rejects)
41## pu: copy FILES to tempdir
42## pu: check whether CHG already exists in CopyChanges
43## pu: check whether FILES already exist in one of the policy queues
44## for deb in FILES:
45## pu: extract control information
46## pu: various checks on control information
47## pu|pa: search for source (in CHG, projectb, policy queues)
48## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
49## pu|pa: check whether deb already exists in the pool
50## for src in FILES:
51## pu: various checks on filenames and CHG consistency
52## pu: if isdsc: check signature
53## for file in FILES:
54## pu: various checks
55## pu: NEW?
56## //pu: check whether file already exists in the pool
57## pu: store what "Component" the package is currently in
58## pu: check whether we found everything we were looking for in CHG
59## pu: check the DSC:
60## pu: check whether we need and have ONE DSC
61## pu: parse the DSC
62## pu: various checks //maybe drop some of the in favor of lintian
63## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
64## pu: check whether DSC_FILES is consistent with "Format"
65## for src in DSC_FILES:
66## pu|pa: check whether file already exists in the pool (with special handling for .orig.tar.gz)
67## pu: create new tempdir
68## pu: create symlink mirror of source
69## pu: unpack source
70## pu: extract changelog information for BTS
71## //pu: create missing .orig symlink
72## pu: check with lintian
73## for file in FILES:
74## pu: check checksums and sizes
75## for file in DSC_FILES:
76## pu: check checksums and sizes
77## pu: CHG: check urgency
78## for deb in FILES:
79## pu: extract contents list and check for dubious timestamps
80## pu: check that the uploader is actually allowed to upload the package
81### pa: install:
82### if stable_install:
83### pa: remove from p-u
84### pa: add to stable
85### pa: move CHG to morgue
86### pa: append data to ChangeLog
87### pa: send mail
88### pa: remove .dak file
89### else:
90### pa: add dsc to db:
91### for file in DSC_FILES:
92### pa: add file to file
93### pa: add file to dsc_files
94### pa: create source entry
95### pa: update source associations
96### pa: update src_uploaders
97### for deb in FILES:
98### pa: add deb to db:
99### pa: add file to file
100### pa: find source entry
101### pa: create binaries entry
102### pa: update binary associations
103### pa: .orig component move
104### pa: move files to pool
105### pa: save CHG
106### pa: move CHG to done/
107### pa: change entry in queue_build
108## pu: use dispatch table to choose target queue:
109## if NEW:
110## pu: write .dak file
111## pu: move to NEW
112## pu: send mail
113## elsif AUTOBYHAND:
114## pu: run autobyhand script
115## pu: if stuff left, do byhand or accept
116## elsif targetqueue in (oldstable, stable, embargo, unembargo):
117## pu: write .dak file
118## pu: check overrides
119## pu: move to queue
120## pu: send mail
121## else:
122## pu: write .dak file
123## pu: move to ACCEPTED
124## pu: send mails
125## pu: create files for BTS
126## pu: create entry in queue_build
127## pu: check overrides
129# Integrity checks
130## GPG
131## Parsing changes (check for duplicates)
132## Parse dsc
133## file list checks
135# New check layout (TODO: Implement)
136## Permission checks
137### suite mappings
138### ACLs
139### version checks (suite)
140### override checks
142## Source checks
143### copy orig
144### unpack
145### BTS changelog
146### src contents
147### lintian
148### urgency log
150## Binary checks
151### timestamps
152### control checks
153### src relation check
154### contents
156## Database insertion (? copy from stuff)
157### BYHAND / NEW / Policy queues
158### Pool
160## Queue builds
162import datetime
163import errno
164import fcntl
165import functools
166import os
167import random
168import sys
169import time
170import traceback
171from collections.abc import Callable, Iterable
172from typing import Concatenate, NoReturn
174import apt_pkg
176import daklib.announce
177import daklib.archive
178import daklib.checks
179import daklib.upload
180import daklib.utils as utils
181from daklib import daklog
182from daklib.config import Config
183from daklib.dbconn import DBConn, Keyring, SignatureHistory
184from daklib.regexes import re_default_answer
185from daklib.summarystats import SummaryStats
186from daklib.urgencylog import UrgencyLog
188###############################################################################
190Options: apt_pkg.Configuration
191Logger: daklog.Logger
193###############################################################################
196def usage(exit_code=0) -> NoReturn:
197 print(
198 """Usage: dak process-upload [OPTION]... [CHANGES]...
199 -a, --automatic automatic run
200 -d, --directory <DIR> process uploads in <DIR>
201 -h, --help show this help and exit.
202 --max-duration <D> stop processing after duration (e.g. 10m, 1h 5m)
203 -n, --no-action don't do anything
204 -p, --no-lock don't check lockfile !! for cron.daily only !!
205 -s, --no-mail don't send any mail
206 -V, --version display the version number and exit"""
207 )
208 sys.exit(exit_code)
211###############################################################################
213type Handler[**P, R] = Callable[Concatenate[str, daklib.archive.ArchiveUpload, P], R]
216def try_or_reject[**P, R](function: Handler[P, R]) -> Handler[P, R]:
217 """Try to call function or reject the upload if that fails"""
219 @functools.wraps(function)
220 def wrapper(directory: str, upload: daklib.archive.ArchiveUpload, *args, **kwargs):
221 reason = "No exception caught. This should not happen."
223 try:
224 return function(directory, upload, *args, **kwargs)
225 except (daklib.archive.ArchiveException, daklib.checks.Reject) as e:
226 reason = str(e)
227 except Exception:
228 reason = "There was an uncaught exception when processing your upload:\n{0}\nAny original reject reason follows below.".format(
229 traceback.format_exc()
230 )
232 try:
233 upload.rollback()
234 return real_reject(directory, upload, reason=reason)
235 except Exception:
236 reason = "In addition there was an exception when rejecting the package:\n{0}\nPrevious reasons:\n{1}".format(
237 traceback.format_exc(), reason
238 )
239 upload.rollback()
240 return real_reject(directory, upload, reason=reason, notify=False)
242 raise Exception(
243 "Rejecting upload failed after multiple tries. Giving up. Last reason:\n{0}".format(
244 reason
245 )
246 )
248 return wrapper
251def get_processed_upload(
252 upload: daklib.archive.ArchiveUpload,
253) -> daklib.announce.ProcessedUpload:
254 changes = upload.changes
255 control = upload.changes.changes
257 pu = daklib.announce.ProcessedUpload()
259 pu.maintainer = control.get("Maintainer")
260 pu.changed_by = control.get("Changed-By")
261 pu.fingerprint = changes.primary_fingerprint
262 pu.authorized_by_fingerprint = upload.authorized_by_fingerprint.fingerprint
264 pu.suites = upload.final_suites or []
265 pu.from_policy_suites = []
267 with open(upload.changes.path, "r") as fd:
268 pu.changes = fd.read()
269 pu.changes_filename = upload.changes.filename
270 pu.sourceful = upload.changes.sourceful
271 pu.source = control.get("Source")
272 pu.version = control.get("Version")
273 pu.architecture = control.get("Architecture")
274 pu.bugs = changes.closed_bugs
276 pu.program = "process-upload"
278 pu.warnings = upload.warnings
280 return pu
283@try_or_reject
284def accept(directory: str, upload: daklib.archive.ArchiveUpload) -> None:
285 cnf = Config()
287 Logger.log(["ACCEPT", upload.changes.filename])
288 print("ACCEPT")
290 upload.install()
291 utils.process_buildinfos(
292 upload.directory, upload.changes.buildinfo_files, upload.transaction.fs, Logger
293 )
295 assert upload.final_suites is not None
296 accepted_to_real_suite = any(
297 suite.policy_queue is None for suite in upload.final_suites
298 )
299 sourceful_upload = upload.changes.sourceful
301 control = upload.changes.changes
302 if sourceful_upload and not Options["No-Action"]:
303 urgency = control.get("Urgency")
304 # As per policy 5.6.17, the urgency can be followed by a space and a
305 # comment. Extract only the urgency from the string.
306 if " " in urgency: 306 ↛ 307line 306 didn't jump to line 307 because the condition on line 306 was never true
307 urgency, comment = urgency.split(" ", 1)
308 if urgency not in cnf.value_list("Urgency::Valid"): 308 ↛ 309line 308 didn't jump to line 309 because the condition on line 308 was never true
309 urgency = cnf["Urgency::Default"]
310 UrgencyLog().log(control["Source"], control["Version"], urgency)
312 pu = get_processed_upload(upload)
313 daklib.announce.announce_accept(pu)
315 # Move .changes to done, but only for uploads that were accepted to a
316 # real suite. process-policy will handle this for uploads to queues.
317 if accepted_to_real_suite:
318 src = os.path.join(upload.directory, upload.changes.filename)
320 now = datetime.datetime.now()
321 donedir = os.path.join(cnf["Dir::Done"], now.strftime("%Y/%m/%d"))
322 dst = os.path.join(donedir, upload.changes.filename)
323 dst = utils.find_next_free(dst)
325 upload.transaction.fs.copy(src, dst, mode=0o644)
327 SummaryStats().accept_count += 1
328 SummaryStats().accept_bytes += upload.changes.bytes
331@try_or_reject
332def accept_to_new(directory: str, upload: daklib.archive.ArchiveUpload) -> None:
334 Logger.log(["ACCEPT-TO-NEW", upload.changes.filename])
335 print("ACCEPT-TO-NEW")
337 upload.install_to_new()
338 # TODO: tag bugs pending
340 pu = get_processed_upload(upload)
341 daklib.announce.announce_new(pu)
343 SummaryStats().accept_count += 1
344 SummaryStats().accept_bytes += upload.changes.bytes
347@try_or_reject
348def reject(
349 directory: str,
350 upload: daklib.archive.ArchiveUpload,
351 reason: str | None = None,
352 notify=True,
353) -> None:
354 real_reject(directory, upload, reason, notify)
357def real_reject(
358 directory: str,
359 upload: daklib.archive.ArchiveUpload,
360 reason: str | None = None,
361 notify=True,
362) -> None:
363 # XXX: rejection itself should go to daklib.archive.ArchiveUpload
364 cnf = Config()
366 Logger.log(["REJECT", upload.changes.filename])
367 print("REJECT")
369 fs = upload.transaction.fs
370 rejectdir = cnf["Dir::Reject"]
372 files = [f.filename for f in upload.changes.files.values()]
373 files.append(upload.changes.filename)
375 for fn in files:
376 src = os.path.join(upload.directory, fn)
377 dst = utils.find_next_free(os.path.join(rejectdir, fn))
378 if not os.path.exists(src):
379 continue
380 fs.copy(src, dst)
382 if upload.reject_reasons is not None: 382 ↛ 387line 382 didn't jump to line 387 because the condition on line 382 was always true
383 if reason is None: 383 ↛ 385line 383 didn't jump to line 385 because the condition on line 383 was always true
384 reason = ""
385 reason = reason + "\n" + "\n".join(upload.reject_reasons)
387 if reason is None: 387 ↛ 388line 387 didn't jump to line 388 because the condition on line 387 was never true
388 reason = "(Unknown reason. Please check logs.)"
390 dst = utils.find_next_free(
391 os.path.join(rejectdir, "{0}.reason".format(upload.changes.filename))
392 )
393 fh = fs.create(dst)
394 fh.write(reason)
395 fh.close()
397 if notify: 397 ↛ 401line 397 didn't jump to line 401 because the condition on line 397 was always true
398 pu = get_processed_upload(upload)
399 daklib.announce.announce_reject(pu, reason)
401 SummaryStats().reject_count += 1
404###############################################################################
407def action(directory: str, upload: daklib.archive.ArchiveUpload) -> bool:
408 changes = upload.changes
409 processed = True
411 global Logger
413 cnf = Config()
415 okay = upload.check()
417 try:
418 summary = changes.changes.get("Changes", "")
419 except UnicodeDecodeError as e:
420 summary = "Reading changes failed: %s" % (e)
421 # the upload checks should have detected this, but make sure this
422 # upload gets rejected in any case
423 upload.reject_reasons.append(summary)
425 package_info = []
426 if okay:
427 if changes.source is not None:
428 package_info.append("source:{0}".format(changes.source.dsc["Source"]))
429 for binary in changes.binaries:
430 package_info.append("binary:{0}".format(binary.control["Package"]))
432 (prompt, answer) = ("", "XXX")
433 if Options["No-Action"] or Options["Automatic"]: 433 ↛ 436line 433 didn't jump to line 436 because the condition on line 433 was always true
434 answer = "S"
436 print(summary)
437 print()
438 print("\n".join(package_info))
439 print()
440 if len(upload.warnings) > 0:
441 print("\n".join(upload.warnings))
442 print()
444 if len(upload.reject_reasons) > 0:
445 print("Reason:")
446 print("\n".join(upload.reject_reasons))
447 print()
449 path = os.path.join(directory, changes.filename)
450 created = os.stat(path).st_mtime
451 now = time.time()
452 too_new = now - created < int(cnf["Dinstall::SkipTime"])
454 if too_new:
455 print("SKIP (too new)")
456 prompt = "[S]kip, Quit ?"
457 else:
458 prompt = "[R]eject, Skip, Quit ?"
459 if Options["Automatic"]: 459 ↛ 470line 459 didn't jump to line 470 because the condition on line 459 was always true
460 answer = "R"
461 elif upload.new:
462 prompt = "[N]ew, Skip, Quit ?"
463 if Options["Automatic"]: 463 ↛ 470line 463 didn't jump to line 470 because the condition on line 463 was always true
464 answer = "N"
465 else:
466 prompt = "[A]ccept, Skip, Quit ?"
467 if Options["Automatic"]: 467 ↛ 470line 467 didn't jump to line 470 because the condition on line 467 was always true
468 answer = "A"
470 while prompt.find(answer) == -1: 470 ↛ 471line 470 didn't jump to line 471 because the condition on line 470 was never true
471 answer = utils.input_or_exit(prompt)
472 m = re_default_answer.match(prompt)
473 if answer == "":
474 assert m is not None
475 answer = m.group(1)
476 answer = answer[:1].upper()
478 if answer == "R":
479 reject(directory, upload)
480 elif answer == "A":
481 # upload.try_autobyhand must not be run with No-Action.
482 if Options["No-Action"]: 482 ↛ 483line 482 didn't jump to line 483 because the condition on line 482 was never true
483 accept(directory, upload)
484 elif upload.try_autobyhand(): 484 ↛ 487line 484 didn't jump to line 487 because the condition on line 484 was always true
485 accept(directory, upload)
486 else:
487 print("W: redirecting to BYHAND as automatic processing failed.")
488 accept_to_new(directory, upload)
489 elif answer == "N":
490 accept_to_new(directory, upload)
491 elif answer == "Q": 491 ↛ 492line 491 didn't jump to line 492 because the condition on line 491 was never true
492 sys.exit(0)
493 elif answer == "S": 493 ↛ 496line 493 didn't jump to line 496 because the condition on line 493 was always true
494 processed = False
496 if not Options["No-Action"]: 496 ↛ 499line 496 didn't jump to line 499 because the condition on line 496 was always true
497 upload.commit()
499 return processed
502###############################################################################
505def unlink_if_exists(path: str) -> None:
506 try:
507 os.unlink(path)
508 except OSError as e:
509 if e.errno != errno.ENOENT: 509 ↛ 510line 509 didn't jump to line 510 because the condition on line 509 was never true
510 raise
513def process_it(
514 directory: str, changes: daklib.upload.Changes, keyrings: list[str]
515) -> None:
516 global Logger
518 print("\n{0}\n".format(changes.filename))
519 Logger.log(["Processing changes file", changes.filename])
521 with daklib.archive.ArchiveUpload(directory, changes, keyrings) as upload:
522 processed = action(directory, upload)
523 if processed and not Options["No-Action"]:
524 session = DBConn().session()
525 history = SignatureHistory.from_signed_file(upload.changes)
526 if history.query(session) is None: 526 ↛ 529line 526 didn't jump to line 529 because the condition on line 526 was always true
527 session.add(history)
528 session.commit()
529 session.close()
531 unlink_if_exists(os.path.join(directory, changes.filename))
532 for fn in changes.files:
533 unlink_if_exists(os.path.join(directory, fn))
536###############################################################################
539def _group_changes_by_source_and_shuffle(
540 changes: list[tuple[str, daklib.upload.Changes]],
541) -> list[tuple[str, daklib.upload.Changes]]:
542 """Group changes by Source, sort each group, and shuffle group order."""
543 grouped: dict[str, list[tuple[str, daklib.upload.Changes]]] = {}
544 for directory, change in changes:
545 source = change.changes.get("Source", "")
546 grouped.setdefault(source, []).append((directory, change))
548 for group in grouped.values():
549 group.sort(key=lambda item: item[1])
551 source_names = list(grouped)
552 random.shuffle(source_names)
553 return [item for source in source_names for item in grouped[source]]
556###############################################################################
559def process_changes(
560 changes_filenames: Iterable[str],
561 max_duration: datetime.timedelta | None = None,
562) -> None:
563 deadline: float | None = None
564 if max_duration is not None: 564 ↛ 565line 564 didn't jump to line 565 because the condition on line 564 was never true
565 deadline = time.monotonic() + max_duration.total_seconds()
567 session = DBConn().session()
568 keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority)
569 keyring_files = [k.keyring_name for k in keyrings]
570 session.close()
572 changes = []
573 for fn in changes_filenames:
574 try:
575 directory, filename = os.path.split(fn)
576 c = daklib.upload.Changes(directory, filename, keyring_files)
577 changes.append((directory, c))
578 except Exception as e:
579 try:
580 Logger.log(
581 [
582 filename,
583 "Error while loading changes file {0}: {1}".format(fn, e),
584 ]
585 )
586 except Exception as e:
587 Logger.log(
588 [
589 filename,
590 "Error while loading changes file {0}, with additional error while printing exception: {1}".format(
591 fn, repr(e)
592 ),
593 ]
594 )
596 changes = _group_changes_by_source_and_shuffle(changes)
598 for directory, c in changes:
599 if deadline is not None and time.monotonic() >= deadline: 599 ↛ 600line 599 didn't jump to line 600 because the condition on line 599 was never true
600 Logger.log(["Max duration reached; stopping processing loop"])
601 break
602 process_it(directory, c, keyring_files)
605###############################################################################
608def main() -> None:
609 global Options, Logger
611 cnf = Config()
612 summarystats = SummaryStats()
614 Arguments = [
615 ("a", "automatic", "Dinstall::Options::Automatic"),
616 ("h", "help", "Dinstall::Options::Help"),
617 ("\0", "max-duration", "Dinstall::Options::Max-Duration", "HasArg"),
618 ("n", "no-action", "Dinstall::Options::No-Action"),
619 ("p", "no-lock", "Dinstall::Options::No-Lock"),
620 ("s", "no-mail", "Dinstall::Options::No-Mail"),
621 ("d", "directory", "Dinstall::Options::Directory", "HasArg"),
622 ]
624 for i in [
625 "automatic",
626 "help",
627 "max-duration",
628 "no-action",
629 "no-lock",
630 "no-mail",
631 "version",
632 "directory",
633 ]:
634 key = "Dinstall::Options::%s" % i
635 if key not in cnf:
636 cnf[key] = ""
638 changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) # type: ignore[attr-defined]
639 Options = cnf.subtree("Dinstall::Options")
641 if Options["Help"]:
642 usage()
644 # -n/--dry-run invalidates some other options which would involve things happening
645 if Options["No-Action"]: 645 ↛ 646line 645 didn't jump to line 646 because the condition on line 645 was never true
646 Options["Automatic"] = "" # type: ignore[index]
648 # Obtain lock if not in no-action mode and initialize the log
649 if not Options["No-Action"]: 649 ↛ 668line 649 didn't jump to line 668 because the condition on line 649 was always true
650 lock_fd = os.open(
651 os.path.join(cnf["Dir::Lock"], "process-upload.lock"),
652 os.O_RDWR | os.O_CREAT,
653 )
654 try:
655 fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
656 except OSError as e:
657 if e.errno in (errno.EACCES, errno.EAGAIN):
658 utils.fubar(
659 "Couldn't obtain lock; assuming another 'dak process-upload' is already running."
660 )
661 else:
662 raise
664 # Initialise UrgencyLog() - it will deal with the case where we don't
665 # want to log urgencies
666 urgencylog = UrgencyLog()
668 Logger = daklog.Logger("process-upload", Options["No-Action"])
670 # If we have a directory flag, use it to find our files
671 if cnf["Dinstall::Options::Directory"] != "": 671 ↛ 685line 671 didn't jump to line 685 because the condition on line 671 was always true
672 # Note that we clobber the list of files we were given in this case
673 # so warn if the user has done both
674 if len(changes_files) > 0: 674 ↛ 675line 674 didn't jump to line 675 because the condition on line 674 was never true
675 utils.warn("Directory provided so ignoring files given on command line")
677 changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
678 Logger.log(
679 [
680 "Using changes files from directory",
681 cnf["Dinstall::Options::Directory"],
682 len(changes_files),
683 ]
684 )
685 elif not len(changes_files) > 0:
686 utils.fubar("No changes files given and no directory specified")
687 else:
688 Logger.log(["Using changes files from command-line", len(changes_files)])
690 max_duration = None
691 if Options["Max-Duration"]: 691 ↛ 692line 691 didn't jump to line 692 because the condition on line 691 was never true
692 try:
693 max_duration = utils.parse_duration(Options["Max-Duration"])
694 except ValueError as e:
695 utils.fubar("Invalid --max-duration: %s" % e)
697 process_changes(changes_files, max_duration=max_duration)
699 if summarystats.accept_count:
700 sets = "set"
701 if summarystats.accept_count > 1:
702 sets = "sets"
703 print(
704 "Installed %d package %s, %s."
705 % (
706 summarystats.accept_count,
707 sets,
708 utils.size_type(int(summarystats.accept_bytes)),
709 )
710 )
711 Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
713 if summarystats.reject_count:
714 sets = "set"
715 if summarystats.reject_count > 1:
716 sets = "sets"
717 print("Rejected %d package %s." % (summarystats.reject_count, sets))
718 Logger.log(["rejected", summarystats.reject_count])
720 if not Options["No-Action"]: 720 ↛ 723line 720 didn't jump to line 723 because the condition on line 720 was always true
721 urgencylog.close()
723 Logger.close()
726###############################################################################
729if __name__ == "__main__":
730 main()