Coverage for dak/process_upload.py: 83%
297 statements
« prev ^ index » next coverage.py v6.5.0, created at 2025-08-26 22:11 +0000
1#! /usr/bin/env python3
3"""
4Checks Debian packages from Incoming
5@contact: Debian FTP Master <ftpmaster@debian.org>
6@copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
7@copyright: 2009 Joerg Jaspert <joerg@debian.org>
8@copyright: 2009 Mark Hymers <mhy@debian.org>
9@copyright: 2009 Frank Lichtenheld <djpig@debian.org>
10@license: GNU General Public License version 2 or later
11"""
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27# based on process-unchecked and process-accepted
29## pu|pa: locking (daily.lock)
30## pu|pa: parse arguments -> list of changes files
31## pa: initialize urgency log
32## pu|pa: sort changes list
34## foreach changes:
35### pa: load dak file
36## pu: copy CHG to tempdir
37## pu: check CHG signature
38## pu: parse changes file
39## pu: checks:
40## pu: check distribution (mappings, rejects)
41## pu: copy FILES to tempdir
42## pu: check whether CHG already exists in CopyChanges
43## pu: check whether FILES already exist in one of the policy queues
44## for deb in FILES:
45## pu: extract control information
46## pu: various checks on control information
47## pu|pa: search for source (in CHG, projectb, policy queues)
48## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
49## pu|pa: check whether deb already exists in the pool
50## for src in FILES:
51## pu: various checks on filenames and CHG consistency
52## pu: if isdsc: check signature
53## for file in FILES:
54## pu: various checks
55## pu: NEW?
56## //pu: check whether file already exists in the pool
57## pu: store what "Component" the package is currently in
58## pu: check whether we found everything we were looking for in CHG
59## pu: check the DSC:
60## pu: check whether we need and have ONE DSC
61## pu: parse the DSC
62## pu: various checks //maybe drop some of the in favor of lintian
63## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
64## pu: check whether DSC_FILES is consistent with "Format"
65## for src in DSC_FILES:
66## pu|pa: check whether file already exists in the pool (with special handling for .orig.tar.gz)
67## pu: create new tempdir
68## pu: create symlink mirror of source
69## pu: unpack source
70## pu: extract changelog information for BTS
71## //pu: create missing .orig symlink
72## pu: check with lintian
73## for file in FILES:
74## pu: check checksums and sizes
75## for file in DSC_FILES:
76## pu: check checksums and sizes
77## pu: CHG: check urgency
78## for deb in FILES:
79## pu: extract contents list and check for dubious timestamps
80## pu: check that the uploader is actually allowed to upload the package
81### pa: install:
82### if stable_install:
83### pa: remove from p-u
84### pa: add to stable
85### pa: move CHG to morgue
86### pa: append data to ChangeLog
87### pa: send mail
88### pa: remove .dak file
89### else:
90### pa: add dsc to db:
91### for file in DSC_FILES:
92### pa: add file to file
93### pa: add file to dsc_files
94### pa: create source entry
95### pa: update source associations
96### pa: update src_uploaders
97### for deb in FILES:
98### pa: add deb to db:
99### pa: add file to file
100### pa: find source entry
101### pa: create binaries entry
102### pa: update binary associations
103### pa: .orig component move
104### pa: move files to pool
105### pa: save CHG
106### pa: move CHG to done/
107### pa: change entry in queue_build
108## pu: use dispatch table to choose target queue:
109## if NEW:
110## pu: write .dak file
111## pu: move to NEW
112## pu: send mail
113## elsif AUTOBYHAND:
114## pu: run autobyhand script
115## pu: if stuff left, do byhand or accept
116## elsif targetqueue in (oldstable, stable, embargo, unembargo):
117## pu: write .dak file
118## pu: check overrides
119## pu: move to queue
120## pu: send mail
121## else:
122## pu: write .dak file
123## pu: move to ACCEPTED
124## pu: send mails
125## pu: create files for BTS
126## pu: create entry in queue_build
127## pu: check overrides
129# Integrity checks
130## GPG
131## Parsing changes (check for duplicates)
132## Parse dsc
133## file list checks
135# New check layout (TODO: Implement)
136## Permission checks
137### suite mappings
138### ACLs
139### version checks (suite)
140### override checks
142## Source checks
143### copy orig
144### unpack
145### BTS changelog
146### src contents
147### lintian
148### urgency log
150## Binary checks
151### timestamps
152### control checks
153### src relation check
154### contents
156## Database insertion (? copy from stuff)
157### BYHAND / NEW / Policy queues
158### Pool
160## Queue builds
162import datetime
163import errno
164import fcntl
165import functools
166import os
167import sys
168import time
169import traceback
170from collections.abc import Iterable
171from typing import NoReturn
173import apt_pkg
175import daklib.announce
176import daklib.archive
177import daklib.checks
178import daklib.upload
179import daklib.utils as utils
180from daklib import daklog
181from daklib.config import Config
182from daklib.dbconn import DBConn, Keyring, SignatureHistory
183from daklib.regexes import re_default_answer
184from daklib.summarystats import SummaryStats
185from daklib.urgencylog import UrgencyLog
187###############################################################################
189Options = None
190Logger = None
192###############################################################################
195def usage(exit_code=0) -> NoReturn:
196 print(
197 """Usage: dak process-upload [OPTION]... [CHANGES]...
198 -a, --automatic automatic run
199 -d, --directory <DIR> process uploads in <DIR>
200 -h, --help show this help and exit.
201 -n, --no-action don't do anything
202 -p, --no-lock don't check lockfile !! for cron.daily only !!
203 -s, --no-mail don't send any mail
204 -V, --version display the version number and exit"""
205 )
206 sys.exit(exit_code)
209###############################################################################
212def try_or_reject(function):
213 """Try to call function or reject the upload if that fails"""
215 @functools.wraps(function)
216 def wrapper(directory: str, upload: daklib.archive.ArchiveUpload, *args, **kwargs):
217 reason = "No exception caught. This should not happen."
219 try:
220 return function(directory, upload, *args, **kwargs)
221 except (daklib.archive.ArchiveException, daklib.checks.Reject) as e:
222 reason = str(e)
223 except Exception:
224 reason = "There was an uncaught exception when processing your upload:\n{0}\nAny original reject reason follows below.".format(
225 traceback.format_exc()
226 )
228 try:
229 upload.rollback()
230 return real_reject(directory, upload, reason=reason)
231 except Exception:
232 reason = "In addition there was an exception when rejecting the package:\n{0}\nPrevious reasons:\n{1}".format(
233 traceback.format_exc(), reason
234 )
235 upload.rollback()
236 return real_reject(directory, upload, reason=reason, notify=False)
238 raise Exception(
239 "Rejecting upload failed after multiple tries. Giving up. Last reason:\n{0}".format(
240 reason
241 )
242 )
244 return wrapper
247def get_processed_upload(
248 upload: daklib.archive.ArchiveUpload,
249) -> daklib.announce.ProcessedUpload:
250 changes = upload.changes
251 control = upload.changes.changes
253 pu = daklib.announce.ProcessedUpload()
255 pu.maintainer = control.get("Maintainer")
256 pu.changed_by = control.get("Changed-By")
257 pu.fingerprint = changes.primary_fingerprint
258 pu.authorized_by_fingerprint = upload.authorized_by_fingerprint.fingerprint
260 pu.suites = upload.final_suites or []
261 pu.from_policy_suites = []
263 with open(upload.changes.path, "r") as fd:
264 pu.changes = fd.read()
265 pu.changes_filename = upload.changes.filename
266 pu.sourceful = upload.changes.sourceful
267 pu.source = control.get("Source")
268 pu.version = control.get("Version")
269 pu.architecture = control.get("Architecture")
270 pu.bugs = changes.closed_bugs
272 pu.program = "process-upload"
274 pu.warnings = upload.warnings
276 return pu
279@try_or_reject
280def accept(directory: str, upload: daklib.archive.ArchiveUpload) -> None:
281 cnf = Config()
283 Logger.log(["ACCEPT", upload.changes.filename])
284 print("ACCEPT")
286 upload.install()
287 utils.process_buildinfos(
288 upload.directory, upload.changes.buildinfo_files, upload.transaction.fs, Logger
289 )
291 accepted_to_real_suite = any(
292 suite.policy_queue is None for suite in upload.final_suites
293 )
294 sourceful_upload = upload.changes.sourceful
296 control = upload.changes.changes
297 if sourceful_upload and not Options["No-Action"]:
298 urgency = control.get("Urgency")
299 # As per policy 5.6.17, the urgency can be followed by a space and a
300 # comment. Extract only the urgency from the string.
301 if " " in urgency: 301 ↛ 302line 301 didn't jump to line 302, because the condition on line 301 was never true
302 urgency, comment = urgency.split(" ", 1)
303 if urgency not in cnf.value_list("Urgency::Valid"): 303 ↛ 304line 303 didn't jump to line 304, because the condition on line 303 was never true
304 urgency = cnf["Urgency::Default"]
305 UrgencyLog().log(control["Source"], control["Version"], urgency)
307 pu = get_processed_upload(upload)
308 daklib.announce.announce_accept(pu)
310 # Move .changes to done, but only for uploads that were accepted to a
311 # real suite. process-policy will handle this for uploads to queues.
312 if accepted_to_real_suite:
313 src = os.path.join(upload.directory, upload.changes.filename)
315 now = datetime.datetime.now()
316 donedir = os.path.join(cnf["Dir::Done"], now.strftime("%Y/%m/%d"))
317 dst = os.path.join(donedir, upload.changes.filename)
318 dst = utils.find_next_free(dst)
320 upload.transaction.fs.copy(src, dst, mode=0o644)
322 SummaryStats().accept_count += 1
323 SummaryStats().accept_bytes += upload.changes.bytes
326@try_or_reject
327def accept_to_new(directory: str, upload: daklib.archive.ArchiveUpload) -> None:
329 Logger.log(["ACCEPT-TO-NEW", upload.changes.filename])
330 print("ACCEPT-TO-NEW")
332 upload.install_to_new()
333 # TODO: tag bugs pending
335 pu = get_processed_upload(upload)
336 daklib.announce.announce_new(pu)
338 SummaryStats().accept_count += 1
339 SummaryStats().accept_bytes += upload.changes.bytes
342@try_or_reject
343def reject(
344 directory: str, upload: daklib.archive.ArchiveUpload, reason=None, notify=True
345) -> None:
346 real_reject(directory, upload, reason, notify)
349def real_reject(
350 directory: str, upload: daklib.archive.ArchiveUpload, reason=None, notify=True
351) -> None:
352 # XXX: rejection itself should go to daklib.archive.ArchiveUpload
353 cnf = Config()
355 Logger.log(["REJECT", upload.changes.filename])
356 print("REJECT")
358 fs = upload.transaction.fs
359 rejectdir = cnf["Dir::Reject"]
361 files = [f.filename for f in upload.changes.files.values()]
362 files.append(upload.changes.filename)
364 for fn in files:
365 src = os.path.join(upload.directory, fn)
366 dst = utils.find_next_free(os.path.join(rejectdir, fn))
367 if not os.path.exists(src):
368 continue
369 fs.copy(src, dst)
371 if upload.reject_reasons is not None: 371 ↛ 376line 371 didn't jump to line 376, because the condition on line 371 was never false
372 if reason is None: 372 ↛ 374line 372 didn't jump to line 374, because the condition on line 372 was never false
373 reason = ""
374 reason = reason + "\n" + "\n".join(upload.reject_reasons)
376 if reason is None: 376 ↛ 377line 376 didn't jump to line 377, because the condition on line 376 was never true
377 reason = "(Unknown reason. Please check logs.)"
379 dst = utils.find_next_free(
380 os.path.join(rejectdir, "{0}.reason".format(upload.changes.filename))
381 )
382 fh = fs.create(dst)
383 fh.write(reason)
384 fh.close()
386 if notify: 386 ↛ 390line 386 didn't jump to line 390, because the condition on line 386 was never false
387 pu = get_processed_upload(upload)
388 daklib.announce.announce_reject(pu, reason)
390 SummaryStats().reject_count += 1
393###############################################################################
396def action(directory: str, upload: daklib.archive.ArchiveUpload) -> bool:
397 changes = upload.changes
398 processed = True
400 global Logger
402 cnf = Config()
404 okay = upload.check()
406 try:
407 summary = changes.changes.get("Changes", "")
408 except UnicodeDecodeError as e:
409 summary = "Reading changes failed: %s" % (e)
410 # the upload checks should have detected this, but make sure this
411 # upload gets rejected in any case
412 upload.reject_reasons.append(summary)
414 package_info = []
415 if okay:
416 if changes.source is not None:
417 package_info.append("source:{0}".format(changes.source.dsc["Source"]))
418 for binary in changes.binaries:
419 package_info.append("binary:{0}".format(binary.control["Package"]))
421 (prompt, answer) = ("", "XXX")
422 if Options["No-Action"] or Options["Automatic"]: 422 ↛ 425line 422 didn't jump to line 425, because the condition on line 422 was never false
423 answer = "S"
425 print(summary)
426 print()
427 print("\n".join(package_info))
428 print()
429 if len(upload.warnings) > 0:
430 print("\n".join(upload.warnings))
431 print()
433 if len(upload.reject_reasons) > 0:
434 print("Reason:")
435 print("\n".join(upload.reject_reasons))
436 print()
438 path = os.path.join(directory, changes.filename)
439 created = os.stat(path).st_mtime
440 now = time.time()
441 too_new = now - created < int(cnf["Dinstall::SkipTime"])
443 if too_new:
444 print("SKIP (too new)")
445 prompt = "[S]kip, Quit ?"
446 else:
447 prompt = "[R]eject, Skip, Quit ?"
448 if Options["Automatic"]: 448 ↛ 459line 448 didn't jump to line 459, because the condition on line 448 was never false
449 answer = "R"
450 elif upload.new:
451 prompt = "[N]ew, Skip, Quit ?"
452 if Options["Automatic"]: 452 ↛ 459line 452 didn't jump to line 459, because the condition on line 452 was never false
453 answer = "N"
454 else:
455 prompt = "[A]ccept, Skip, Quit ?"
456 if Options["Automatic"]: 456 ↛ 459line 456 didn't jump to line 459, because the condition on line 456 was never false
457 answer = "A"
459 while prompt.find(answer) == -1: 459 ↛ 460line 459 didn't jump to line 460, because the condition on line 459 was never true
460 answer = utils.input_or_exit(prompt)
461 m = re_default_answer.match(prompt)
462 if answer == "":
463 answer = m.group(1)
464 answer = answer[:1].upper()
466 if answer == "R":
467 reject(directory, upload)
468 elif answer == "A":
469 # upload.try_autobyhand must not be run with No-Action.
470 if Options["No-Action"]: 470 ↛ 471line 470 didn't jump to line 471, because the condition on line 470 was never true
471 accept(directory, upload)
472 elif upload.try_autobyhand(): 472 ↛ 475line 472 didn't jump to line 475, because the condition on line 472 was never false
473 accept(directory, upload)
474 else:
475 print("W: redirecting to BYHAND as automatic processing failed.")
476 accept_to_new(directory, upload)
477 elif answer == "N":
478 accept_to_new(directory, upload)
479 elif answer == "Q": 479 ↛ 480line 479 didn't jump to line 480, because the condition on line 479 was never true
480 sys.exit(0)
481 elif answer == "S": 481 ↛ 484line 481 didn't jump to line 484, because the condition on line 481 was never false
482 processed = False
484 if not Options["No-Action"]: 484 ↛ 487line 484 didn't jump to line 487, because the condition on line 484 was never false
485 upload.commit()
487 return processed
490###############################################################################
493def unlink_if_exists(path: str) -> None:
494 try:
495 os.unlink(path)
496 except OSError as e:
497 if e.errno != errno.ENOENT: 497 ↛ 498line 497 didn't jump to line 498, because the condition on line 497 was never true
498 raise
501def process_it(
502 directory: str, changes: daklib.upload.Changes, keyrings: list[str]
503) -> None:
504 global Logger
506 print("\n{0}\n".format(changes.filename))
507 Logger.log(["Processing changes file", changes.filename])
509 with daklib.archive.ArchiveUpload(directory, changes, keyrings) as upload:
510 processed = action(directory, upload)
511 if processed and not Options["No-Action"]:
512 session = DBConn().session()
513 history = SignatureHistory.from_signed_file(upload.changes)
514 if history.query(session) is None: 514 ↛ 517line 514 didn't jump to line 517, because the condition on line 514 was never false
515 session.add(history)
516 session.commit()
517 session.close()
519 unlink_if_exists(os.path.join(directory, changes.filename))
520 for fn in changes.files:
521 unlink_if_exists(os.path.join(directory, fn))
524###############################################################################
527def process_changes(changes_filenames: Iterable[str]):
528 session = DBConn().session()
529 keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority)
530 keyring_files = [k.keyring_name for k in keyrings]
531 session.close()
533 changes = []
534 for fn in changes_filenames:
535 try:
536 directory, filename = os.path.split(fn)
537 c = daklib.upload.Changes(directory, filename, keyring_files)
538 changes.append([directory, c])
539 except Exception as e:
540 try:
541 Logger.log(
542 [
543 filename,
544 "Error while loading changes file {0}: {1}".format(fn, e),
545 ]
546 )
547 except Exception as e:
548 Logger.log(
549 [
550 filename,
551 "Error while loading changes file {0}, with additional error while printing exception: {1}".format(
552 fn, repr(e)
553 ),
554 ]
555 )
557 changes.sort(key=lambda x: x[1])
559 for directory, c in changes:
560 process_it(directory, c, keyring_files)
563###############################################################################
566def main():
567 global Options, Logger
569 cnf = Config()
570 summarystats = SummaryStats()
572 Arguments = [
573 ("a", "automatic", "Dinstall::Options::Automatic"),
574 ("h", "help", "Dinstall::Options::Help"),
575 ("n", "no-action", "Dinstall::Options::No-Action"),
576 ("p", "no-lock", "Dinstall::Options::No-Lock"),
577 ("s", "no-mail", "Dinstall::Options::No-Mail"),
578 ("d", "directory", "Dinstall::Options::Directory", "HasArg"),
579 ]
581 for i in [
582 "automatic",
583 "help",
584 "no-action",
585 "no-lock",
586 "no-mail",
587 "version",
588 "directory",
589 ]:
590 key = "Dinstall::Options::%s" % i
591 if key not in cnf:
592 cnf[key] = ""
594 changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
595 Options = cnf.subtree("Dinstall::Options")
597 if Options["Help"]:
598 usage()
600 # -n/--dry-run invalidates some other options which would involve things happening
601 if Options["No-Action"]: 601 ↛ 602line 601 didn't jump to line 602, because the condition on line 601 was never true
602 Options["Automatic"] = ""
604 # Obtain lock if not in no-action mode and initialize the log
605 if not Options["No-Action"]: 605 ↛ 624line 605 didn't jump to line 624, because the condition on line 605 was never false
606 lock_fd = os.open(
607 os.path.join(cnf["Dir::Lock"], "process-upload.lock"),
608 os.O_RDWR | os.O_CREAT,
609 )
610 try:
611 fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
612 except OSError as e:
613 if e.errno in (errno.EACCES, errno.EAGAIN):
614 utils.fubar(
615 "Couldn't obtain lock; assuming another 'dak process-upload' is already running."
616 )
617 else:
618 raise
620 # Initialise UrgencyLog() - it will deal with the case where we don't
621 # want to log urgencies
622 urgencylog = UrgencyLog()
624 Logger = daklog.Logger("process-upload", Options["No-Action"])
626 # If we have a directory flag, use it to find our files
627 if cnf["Dinstall::Options::Directory"] != "": 627 ↛ 647line 627 didn't jump to line 647, because the condition on line 627 was never false
628 # Note that we clobber the list of files we were given in this case
629 # so warn if the user has done both
630 if len(changes_files) > 0: 630 ↛ 631line 630 didn't jump to line 631, because the condition on line 630 was never true
631 utils.warn("Directory provided so ignoring files given on command line")
633 changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
634 # FIXME: quick hack to NOT have p-u run for ages, if some binnmu fun uploads thousands of changes
635 # might want to make the number configurable at some point
636 if len(changes_files) > 200: 636 ↛ 637line 636 didn't jump to line 637, because the condition on line 636 was never true
637 import random
639 changes_files = random.sample(changes_files, k=200)
640 Logger.log(
641 [
642 "Using changes files from directory",
643 cnf["Dinstall::Options::Directory"],
644 len(changes_files),
645 ]
646 )
647 elif not len(changes_files) > 0:
648 utils.fubar("No changes files given and no directory specified")
649 else:
650 Logger.log(["Using changes files from command-line", len(changes_files)])
652 process_changes(changes_files)
654 if summarystats.accept_count:
655 sets = "set"
656 if summarystats.accept_count > 1:
657 sets = "sets"
658 print(
659 "Installed %d package %s, %s."
660 % (
661 summarystats.accept_count,
662 sets,
663 utils.size_type(int(summarystats.accept_bytes)),
664 )
665 )
666 Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
668 if summarystats.reject_count:
669 sets = "set"
670 if summarystats.reject_count > 1:
671 sets = "sets"
672 print("Rejected %d package %s." % (summarystats.reject_count, sets))
673 Logger.log(["rejected", summarystats.reject_count])
675 if not Options["No-Action"]: 675 ↛ 678line 675 didn't jump to line 678, because the condition on line 675 was never false
676 urgencylog.close()
678 Logger.close()
681###############################################################################
684if __name__ == "__main__":
685 main()