Coverage for dak/process_upload.py: 83%
299 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-01-04 16:18 +0000
1#! /usr/bin/env python3
3"""
4Checks Debian packages from Incoming
5@contact: Debian FTP Master <ftpmaster@debian.org>
6@copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
7@copyright: 2009 Joerg Jaspert <joerg@debian.org>
8@copyright: 2009 Mark Hymers <mhy@debian.org>
9@copyright: 2009 Frank Lichtenheld <djpig@debian.org>
10@license: GNU General Public License version 2 or later
11"""
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27# based on process-unchecked and process-accepted
29## pu|pa: locking (daily.lock)
30## pu|pa: parse arguments -> list of changes files
31## pa: initialize urgency log
32## pu|pa: sort changes list
34## foreach changes:
35### pa: load dak file
36## pu: copy CHG to tempdir
37## pu: check CHG signature
38## pu: parse changes file
39## pu: checks:
40## pu: check distribution (mappings, rejects)
41## pu: copy FILES to tempdir
42## pu: check whether CHG already exists in CopyChanges
43## pu: check whether FILES already exist in one of the policy queues
44## for deb in FILES:
45## pu: extract control information
46## pu: various checks on control information
47## pu|pa: search for source (in CHG, projectb, policy queues)
48## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
49## pu|pa: check whether deb already exists in the pool
50## for src in FILES:
51## pu: various checks on filenames and CHG consistency
52## pu: if isdsc: check signature
53## for file in FILES:
54## pu: various checks
55## pu: NEW?
56## //pu: check whether file already exists in the pool
57## pu: store what "Component" the package is currently in
58## pu: check whether we found everything we were looking for in CHG
59## pu: check the DSC:
60## pu: check whether we need and have ONE DSC
61## pu: parse the DSC
62## pu: various checks //maybe drop some of the in favor of lintian
63## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
64## pu: check whether DSC_FILES is consistent with "Format"
65## for src in DSC_FILES:
66## pu|pa: check whether file already exists in the pool (with special handling for .orig.tar.gz)
67## pu: create new tempdir
68## pu: create symlink mirror of source
69## pu: unpack source
70## pu: extract changelog information for BTS
71## //pu: create missing .orig symlink
72## pu: check with lintian
73## for file in FILES:
74## pu: check checksums and sizes
75## for file in DSC_FILES:
76## pu: check checksums and sizes
77## pu: CHG: check urgency
78## for deb in FILES:
79## pu: extract contents list and check for dubious timestamps
80## pu: check that the uploader is actually allowed to upload the package
81### pa: install:
82### if stable_install:
83### pa: remove from p-u
84### pa: add to stable
85### pa: move CHG to morgue
86### pa: append data to ChangeLog
87### pa: send mail
88### pa: remove .dak file
89### else:
90### pa: add dsc to db:
91### for file in DSC_FILES:
92### pa: add file to file
93### pa: add file to dsc_files
94### pa: create source entry
95### pa: update source associations
96### pa: update src_uploaders
97### for deb in FILES:
98### pa: add deb to db:
99### pa: add file to file
100### pa: find source entry
101### pa: create binaries entry
102### pa: update binary associations
103### pa: .orig component move
104### pa: move files to pool
105### pa: save CHG
106### pa: move CHG to done/
107### pa: change entry in queue_build
108## pu: use dispatch table to choose target queue:
109## if NEW:
110## pu: write .dak file
111## pu: move to NEW
112## pu: send mail
113## elsif AUTOBYHAND:
114## pu: run autobyhand script
115## pu: if stuff left, do byhand or accept
116## elsif targetqueue in (oldstable, stable, embargo, unembargo):
117## pu: write .dak file
118## pu: check overrides
119## pu: move to queue
120## pu: send mail
121## else:
122## pu: write .dak file
123## pu: move to ACCEPTED
124## pu: send mails
125## pu: create files for BTS
126## pu: create entry in queue_build
127## pu: check overrides
129# Integrity checks
130## GPG
131## Parsing changes (check for duplicates)
132## Parse dsc
133## file list checks
135# New check layout (TODO: Implement)
136## Permission checks
137### suite mappings
138### ACLs
139### version checks (suite)
140### override checks
142## Source checks
143### copy orig
144### unpack
145### BTS changelog
146### src contents
147### lintian
148### urgency log
150## Binary checks
151### timestamps
152### control checks
153### src relation check
154### contents
156## Database insertion (? copy from stuff)
157### BYHAND / NEW / Policy queues
158### Pool
160## Queue builds
162import datetime
163import errno
164import fcntl
165import functools
166import os
167import sys
168import time
169import traceback
170from collections.abc import Callable, Iterable
171from typing import Concatenate, NoReturn
173import apt_pkg
175import daklib.announce
176import daklib.archive
177import daklib.checks
178import daklib.upload
179import daklib.utils as utils
180from daklib import daklog
181from daklib.config import Config
182from daklib.dbconn import DBConn, Keyring, SignatureHistory
183from daklib.regexes import re_default_answer
184from daklib.summarystats import SummaryStats
185from daklib.urgencylog import UrgencyLog
187###############################################################################
189Options: apt_pkg.Configuration
190Logger: daklog.Logger
192###############################################################################
195def usage(exit_code=0) -> NoReturn:
196 print(
197 """Usage: dak process-upload [OPTION]... [CHANGES]...
198 -a, --automatic automatic run
199 -d, --directory <DIR> process uploads in <DIR>
200 -h, --help show this help and exit.
201 -n, --no-action don't do anything
202 -p, --no-lock don't check lockfile !! for cron.daily only !!
203 -s, --no-mail don't send any mail
204 -V, --version display the version number and exit"""
205 )
206 sys.exit(exit_code)
209###############################################################################
211type Handler[**P, R] = Callable[Concatenate[str, daklib.archive.ArchiveUpload, P], R]
214def try_or_reject[**P, R](function: Handler[P, R]) -> Handler[P, R]:
215 """Try to call function or reject the upload if that fails"""
217 @functools.wraps(function)
218 def wrapper(directory: str, upload: daklib.archive.ArchiveUpload, *args, **kwargs):
219 reason = "No exception caught. This should not happen."
221 try:
222 return function(directory, upload, *args, **kwargs)
223 except (daklib.archive.ArchiveException, daklib.checks.Reject) as e:
224 reason = str(e)
225 except Exception:
226 reason = "There was an uncaught exception when processing your upload:\n{0}\nAny original reject reason follows below.".format(
227 traceback.format_exc()
228 )
230 try:
231 upload.rollback()
232 return real_reject(directory, upload, reason=reason)
233 except Exception:
234 reason = "In addition there was an exception when rejecting the package:\n{0}\nPrevious reasons:\n{1}".format(
235 traceback.format_exc(), reason
236 )
237 upload.rollback()
238 return real_reject(directory, upload, reason=reason, notify=False)
240 raise Exception(
241 "Rejecting upload failed after multiple tries. Giving up. Last reason:\n{0}".format(
242 reason
243 )
244 )
246 return wrapper
249def get_processed_upload(
250 upload: daklib.archive.ArchiveUpload,
251) -> daklib.announce.ProcessedUpload:
252 changes = upload.changes
253 control = upload.changes.changes
255 pu = daklib.announce.ProcessedUpload()
257 pu.maintainer = control.get("Maintainer")
258 pu.changed_by = control.get("Changed-By")
259 pu.fingerprint = changes.primary_fingerprint
260 pu.authorized_by_fingerprint = upload.authorized_by_fingerprint.fingerprint
262 pu.suites = upload.final_suites or []
263 pu.from_policy_suites = []
265 with open(upload.changes.path, "r") as fd:
266 pu.changes = fd.read()
267 pu.changes_filename = upload.changes.filename
268 pu.sourceful = upload.changes.sourceful
269 pu.source = control.get("Source")
270 pu.version = control.get("Version")
271 pu.architecture = control.get("Architecture")
272 pu.bugs = changes.closed_bugs
274 pu.program = "process-upload"
276 pu.warnings = upload.warnings
278 return pu
281@try_or_reject
282def accept(directory: str, upload: daklib.archive.ArchiveUpload) -> None:
283 cnf = Config()
285 Logger.log(["ACCEPT", upload.changes.filename])
286 print("ACCEPT")
288 upload.install()
289 utils.process_buildinfos(
290 upload.directory, upload.changes.buildinfo_files, upload.transaction.fs, Logger
291 )
293 assert upload.final_suites is not None
294 accepted_to_real_suite = any(
295 suite.policy_queue is None for suite in upload.final_suites
296 )
297 sourceful_upload = upload.changes.sourceful
299 control = upload.changes.changes
300 if sourceful_upload and not Options["No-Action"]:
301 urgency = control.get("Urgency")
302 # As per policy 5.6.17, the urgency can be followed by a space and a
303 # comment. Extract only the urgency from the string.
304 if " " in urgency: 304 ↛ 305line 304 didn't jump to line 305 because the condition on line 304 was never true
305 urgency, comment = urgency.split(" ", 1)
306 if urgency not in cnf.value_list("Urgency::Valid"): 306 ↛ 307line 306 didn't jump to line 307 because the condition on line 306 was never true
307 urgency = cnf["Urgency::Default"]
308 UrgencyLog().log(control["Source"], control["Version"], urgency)
310 pu = get_processed_upload(upload)
311 daklib.announce.announce_accept(pu)
313 # Move .changes to done, but only for uploads that were accepted to a
314 # real suite. process-policy will handle this for uploads to queues.
315 if accepted_to_real_suite:
316 src = os.path.join(upload.directory, upload.changes.filename)
318 now = datetime.datetime.now()
319 donedir = os.path.join(cnf["Dir::Done"], now.strftime("%Y/%m/%d"))
320 dst = os.path.join(donedir, upload.changes.filename)
321 dst = utils.find_next_free(dst)
323 upload.transaction.fs.copy(src, dst, mode=0o644)
325 SummaryStats().accept_count += 1
326 SummaryStats().accept_bytes += upload.changes.bytes
329@try_or_reject
330def accept_to_new(directory: str, upload: daklib.archive.ArchiveUpload) -> None:
332 Logger.log(["ACCEPT-TO-NEW", upload.changes.filename])
333 print("ACCEPT-TO-NEW")
335 upload.install_to_new()
336 # TODO: tag bugs pending
338 pu = get_processed_upload(upload)
339 daklib.announce.announce_new(pu)
341 SummaryStats().accept_count += 1
342 SummaryStats().accept_bytes += upload.changes.bytes
345@try_or_reject
346def reject(
347 directory: str,
348 upload: daklib.archive.ArchiveUpload,
349 reason: str | None = None,
350 notify=True,
351) -> None:
352 real_reject(directory, upload, reason, notify)
355def real_reject(
356 directory: str,
357 upload: daklib.archive.ArchiveUpload,
358 reason: str | None = None,
359 notify=True,
360) -> None:
361 # XXX: rejection itself should go to daklib.archive.ArchiveUpload
362 cnf = Config()
364 Logger.log(["REJECT", upload.changes.filename])
365 print("REJECT")
367 fs = upload.transaction.fs
368 rejectdir = cnf["Dir::Reject"]
370 files = [f.filename for f in upload.changes.files.values()]
371 files.append(upload.changes.filename)
373 for fn in files:
374 src = os.path.join(upload.directory, fn)
375 dst = utils.find_next_free(os.path.join(rejectdir, fn))
376 if not os.path.exists(src):
377 continue
378 fs.copy(src, dst)
380 if upload.reject_reasons is not None: 380 ↛ 385line 380 didn't jump to line 385 because the condition on line 380 was always true
381 if reason is None: 381 ↛ 383line 381 didn't jump to line 383 because the condition on line 381 was always true
382 reason = ""
383 reason = reason + "\n" + "\n".join(upload.reject_reasons)
385 if reason is None: 385 ↛ 386line 385 didn't jump to line 386 because the condition on line 385 was never true
386 reason = "(Unknown reason. Please check logs.)"
388 dst = utils.find_next_free(
389 os.path.join(rejectdir, "{0}.reason".format(upload.changes.filename))
390 )
391 fh = fs.create(dst)
392 fh.write(reason)
393 fh.close()
395 if notify: 395 ↛ 399line 395 didn't jump to line 399 because the condition on line 395 was always true
396 pu = get_processed_upload(upload)
397 daklib.announce.announce_reject(pu, reason)
399 SummaryStats().reject_count += 1
402###############################################################################
405def action(directory: str, upload: daklib.archive.ArchiveUpload) -> bool:
406 changes = upload.changes
407 processed = True
409 global Logger
411 cnf = Config()
413 okay = upload.check()
415 try:
416 summary = changes.changes.get("Changes", "")
417 except UnicodeDecodeError as e:
418 summary = "Reading changes failed: %s" % (e)
419 # the upload checks should have detected this, but make sure this
420 # upload gets rejected in any case
421 upload.reject_reasons.append(summary)
423 package_info = []
424 if okay:
425 if changes.source is not None:
426 package_info.append("source:{0}".format(changes.source.dsc["Source"]))
427 for binary in changes.binaries:
428 package_info.append("binary:{0}".format(binary.control["Package"]))
430 (prompt, answer) = ("", "XXX")
431 if Options["No-Action"] or Options["Automatic"]: 431 ↛ 434line 431 didn't jump to line 434 because the condition on line 431 was always true
432 answer = "S"
434 print(summary)
435 print()
436 print("\n".join(package_info))
437 print()
438 if len(upload.warnings) > 0:
439 print("\n".join(upload.warnings))
440 print()
442 if len(upload.reject_reasons) > 0:
443 print("Reason:")
444 print("\n".join(upload.reject_reasons))
445 print()
447 path = os.path.join(directory, changes.filename)
448 created = os.stat(path).st_mtime
449 now = time.time()
450 too_new = now - created < int(cnf["Dinstall::SkipTime"])
452 if too_new:
453 print("SKIP (too new)")
454 prompt = "[S]kip, Quit ?"
455 else:
456 prompt = "[R]eject, Skip, Quit ?"
457 if Options["Automatic"]: 457 ↛ 468line 457 didn't jump to line 468 because the condition on line 457 was always true
458 answer = "R"
459 elif upload.new:
460 prompt = "[N]ew, Skip, Quit ?"
461 if Options["Automatic"]: 461 ↛ 468line 461 didn't jump to line 468 because the condition on line 461 was always true
462 answer = "N"
463 else:
464 prompt = "[A]ccept, Skip, Quit ?"
465 if Options["Automatic"]: 465 ↛ 468line 465 didn't jump to line 468 because the condition on line 465 was always true
466 answer = "A"
468 while prompt.find(answer) == -1: 468 ↛ 469line 468 didn't jump to line 469 because the condition on line 468 was never true
469 answer = utils.input_or_exit(prompt)
470 m = re_default_answer.match(prompt)
471 if answer == "":
472 assert m is not None
473 answer = m.group(1)
474 answer = answer[:1].upper()
476 if answer == "R":
477 reject(directory, upload)
478 elif answer == "A":
479 # upload.try_autobyhand must not be run with No-Action.
480 if Options["No-Action"]: 480 ↛ 481line 480 didn't jump to line 481 because the condition on line 480 was never true
481 accept(directory, upload)
482 elif upload.try_autobyhand(): 482 ↛ 485line 482 didn't jump to line 485 because the condition on line 482 was always true
483 accept(directory, upload)
484 else:
485 print("W: redirecting to BYHAND as automatic processing failed.")
486 accept_to_new(directory, upload)
487 elif answer == "N":
488 accept_to_new(directory, upload)
489 elif answer == "Q": 489 ↛ 490line 489 didn't jump to line 490 because the condition on line 489 was never true
490 sys.exit(0)
491 elif answer == "S": 491 ↛ 494line 491 didn't jump to line 494 because the condition on line 491 was always true
492 processed = False
494 if not Options["No-Action"]: 494 ↛ 497line 494 didn't jump to line 497 because the condition on line 494 was always true
495 upload.commit()
497 return processed
500###############################################################################
503def unlink_if_exists(path: str) -> None:
504 try:
505 os.unlink(path)
506 except OSError as e:
507 if e.errno != errno.ENOENT: 507 ↛ 508line 507 didn't jump to line 508 because the condition on line 507 was never true
508 raise
511def process_it(
512 directory: str, changes: daklib.upload.Changes, keyrings: list[str]
513) -> None:
514 global Logger
516 print("\n{0}\n".format(changes.filename))
517 Logger.log(["Processing changes file", changes.filename])
519 with daklib.archive.ArchiveUpload(directory, changes, keyrings) as upload:
520 processed = action(directory, upload)
521 if processed and not Options["No-Action"]:
522 session = DBConn().session()
523 history = SignatureHistory.from_signed_file(upload.changes)
524 if history.query(session) is None: 524 ↛ 527line 524 didn't jump to line 527 because the condition on line 524 was always true
525 session.add(history)
526 session.commit()
527 session.close()
529 unlink_if_exists(os.path.join(directory, changes.filename))
530 for fn in changes.files:
531 unlink_if_exists(os.path.join(directory, fn))
534###############################################################################
537def process_changes(changes_filenames: Iterable[str]) -> None:
538 session = DBConn().session()
539 keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority)
540 keyring_files = [k.keyring_name for k in keyrings]
541 session.close()
543 changes = []
544 for fn in changes_filenames:
545 try:
546 directory, filename = os.path.split(fn)
547 c = daklib.upload.Changes(directory, filename, keyring_files)
548 changes.append((directory, c))
549 except Exception as e:
550 try:
551 Logger.log(
552 [
553 filename,
554 "Error while loading changes file {0}: {1}".format(fn, e),
555 ]
556 )
557 except Exception as e:
558 Logger.log(
559 [
560 filename,
561 "Error while loading changes file {0}, with additional error while printing exception: {1}".format(
562 fn, repr(e)
563 ),
564 ]
565 )
567 changes.sort(key=lambda x: x[1])
569 for directory, c in changes:
570 process_it(directory, c, keyring_files)
573###############################################################################
576def main() -> None:
577 global Options, Logger
579 cnf = Config()
580 summarystats = SummaryStats()
582 Arguments = [
583 ("a", "automatic", "Dinstall::Options::Automatic"),
584 ("h", "help", "Dinstall::Options::Help"),
585 ("n", "no-action", "Dinstall::Options::No-Action"),
586 ("p", "no-lock", "Dinstall::Options::No-Lock"),
587 ("s", "no-mail", "Dinstall::Options::No-Mail"),
588 ("d", "directory", "Dinstall::Options::Directory", "HasArg"),
589 ]
591 for i in [
592 "automatic",
593 "help",
594 "no-action",
595 "no-lock",
596 "no-mail",
597 "version",
598 "directory",
599 ]:
600 key = "Dinstall::Options::%s" % i
601 if key not in cnf:
602 cnf[key] = ""
604 changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) # type: ignore[attr-defined]
605 Options = cnf.subtree("Dinstall::Options")
607 if Options["Help"]:
608 usage()
610 # -n/--dry-run invalidates some other options which would involve things happening
611 if Options["No-Action"]: 611 ↛ 612line 611 didn't jump to line 612 because the condition on line 611 was never true
612 Options["Automatic"] = "" # type: ignore[index]
614 # Obtain lock if not in no-action mode and initialize the log
615 if not Options["No-Action"]: 615 ↛ 634line 615 didn't jump to line 634 because the condition on line 615 was always true
616 lock_fd = os.open(
617 os.path.join(cnf["Dir::Lock"], "process-upload.lock"),
618 os.O_RDWR | os.O_CREAT,
619 )
620 try:
621 fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
622 except OSError as e:
623 if e.errno in (errno.EACCES, errno.EAGAIN):
624 utils.fubar(
625 "Couldn't obtain lock; assuming another 'dak process-upload' is already running."
626 )
627 else:
628 raise
630 # Initialise UrgencyLog() - it will deal with the case where we don't
631 # want to log urgencies
632 urgencylog = UrgencyLog()
634 Logger = daklog.Logger("process-upload", Options["No-Action"])
636 # If we have a directory flag, use it to find our files
637 if cnf["Dinstall::Options::Directory"] != "": 637 ↛ 657line 637 didn't jump to line 657 because the condition on line 637 was always true
638 # Note that we clobber the list of files we were given in this case
639 # so warn if the user has done both
640 if len(changes_files) > 0: 640 ↛ 641line 640 didn't jump to line 641 because the condition on line 640 was never true
641 utils.warn("Directory provided so ignoring files given on command line")
643 changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
644 # FIXME: quick hack to NOT have p-u run for ages, if some binnmu fun uploads thousands of changes
645 # might want to make the number configurable at some point
646 if len(changes_files) > 200: 646 ↛ 647line 646 didn't jump to line 647 because the condition on line 646 was never true
647 import random
649 changes_files = random.sample(changes_files, k=200)
650 Logger.log(
651 [
652 "Using changes files from directory",
653 cnf["Dinstall::Options::Directory"],
654 len(changes_files),
655 ]
656 )
657 elif not len(changes_files) > 0:
658 utils.fubar("No changes files given and no directory specified")
659 else:
660 Logger.log(["Using changes files from command-line", len(changes_files)])
662 process_changes(changes_files)
664 if summarystats.accept_count:
665 sets = "set"
666 if summarystats.accept_count > 1:
667 sets = "sets"
668 print(
669 "Installed %d package %s, %s."
670 % (
671 summarystats.accept_count,
672 sets,
673 utils.size_type(int(summarystats.accept_bytes)),
674 )
675 )
676 Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
678 if summarystats.reject_count:
679 sets = "set"
680 if summarystats.reject_count > 1:
681 sets = "sets"
682 print("Rejected %d package %s." % (summarystats.reject_count, sets))
683 Logger.log(["rejected", summarystats.reject_count])
685 if not Options["No-Action"]: 685 ↛ 688line 685 didn't jump to line 688 because the condition on line 685 was always true
686 urgencylog.close()
688 Logger.close()
691###############################################################################
694if __name__ == "__main__":
695 main()