Coverage for dak/clean_queues.py: 21%
121 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-08-03 16:46 +0000
1"""Clean incoming of old unused files"""
3# Copyright (C) 2000, 2001, 2002, 2006 James Troup <james@nocrew.org>
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License as published by
7# the Free Software Foundation; either version 2 of the License, or
8# (at your option) any later version.
10# This program is distributed in the hope that it will be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13# GNU General Public License for more details.
15# You should have received a copy of the GNU General Public License
16# along with this program; if not, write to the Free Software
17# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19################################################################################
21# <aj> Bdale, a ham-er, and the leader,
22# <aj> Willy, a GCC maintainer,
23# <aj> Lamont-work, 'cause he's the top uploader....
24# <aj> Penguin Puff' save the day!
25# <aj> Porting code, trying to build the world,
26# <aj> Here they come just in time...
27# <aj> The Penguin Puff' Guys!
28# <aj> [repeat]
29# <aj> Penguin Puff'!
30# <aj> willy: btw, if you don't maintain gcc you need to start, since
31# the lyrics fit really well that way
33################################################################################
35import os
36import os.path
37import stat
38import sys
39import time
40from datetime import datetime
41from typing import NoReturn
43import apt_pkg
45from daklib import daklog, utils
46from daklib.config import Config
48################################################################################
50Options: apt_pkg.Configuration
51Logger: daklog.Logger
52del_dir: str
53delete_date: int
55################################################################################
58def usage(exit_code=0) -> NoReturn:
59 print(
60 """Usage: dak clean-queues [OPTIONS]
61Clean out incoming directories.
63 -d, --days=DAYS remove anything older than DAYS old
64 -i, --incoming=INCOMING the incoming directory to clean
65 -n, --no-action don't do anything
66 -v, --verbose explain what is being done
67 -h, --help show this help and exit"""
68 )
70 sys.exit(exit_code)
73################################################################################
76def init(cnf) -> None:
77 global delete_date, del_dir
79 # Used for directory naming
80 now_date = datetime.now()
82 # Used for working out times
83 delete_date = int(time.time()) - (int(Options["Days"]) * 84600)
85 morguedir = cnf.get("Dir::Morgue", os.path.join("Dir::Pool", "morgue"))
86 morguesubdir = cnf.get("Clean-Queues::MorgueSubDir", "queue")
88 # Build directory as morguedir/morguesubdir/year/month/day
89 del_dir = os.path.join(
90 morguedir,
91 morguesubdir,
92 str(now_date.year),
93 "%.2d" % now_date.month,
94 "%.2d" % now_date.day,
95 )
97 # Ensure a directory exists to remove files to
98 if not Options["No-Action"]:
99 if not os.path.exists(del_dir):
100 os.makedirs(del_dir, 0o2775)
101 if not os.path.isdir(del_dir):
102 utils.fubar("%s must be a directory." % (del_dir))
104 # Move to the directory to clean
105 incoming = Options.get("Incoming")
106 if not incoming:
107 incoming = cnf.get("Dir::Unchecked")
108 if not incoming:
109 utils.fubar("Cannot find 'unchecked' directory")
111 try:
112 os.chdir(incoming)
113 except OSError as e:
114 utils.fubar(f"Cannot chdir to {incoming}: {e}")
117# Remove a file to the morgue
120def remove(from_dir: str, f: str) -> None:
121 fname = os.path.basename(f)
122 if os.access(f, os.R_OK):
123 Logger.log(["move file to morgue", from_dir, fname, del_dir])
124 if Options["Verbose"]:
125 print("Removing '%s' (to '%s')." % (fname, del_dir))
126 if Options["No-Action"]:
127 return
129 dest_filename = os.path.join(del_dir, fname)
130 # If the destination file exists; try to find another filename to use
131 if os.path.exists(dest_filename):
132 dest_filename = utils.find_next_free(dest_filename, 10)
133 Logger.log(
134 ["change destination file name", os.path.basename(dest_filename)]
135 )
136 utils.move(f, dest_filename, perms=0o660)
137 else:
138 Logger.log(["skipping file because of permission problem", fname])
139 utils.warn("skipping '%s', permission denied." % fname)
142# Removes any old files.
143# [Used for Incoming/REJECT]
144#
147def flush_old() -> None:
148 Logger.log(["check Incoming/REJECT for old files", os.getcwd()])
149 for f in os.listdir("."):
150 if os.path.isfile(f):
151 if os.stat(f)[stat.ST_MTIME] < delete_date:
152 remove("Incoming/REJECT", f)
153 else:
154 if Options["Verbose"]:
155 print("Skipping, too new, '%s'." % (os.path.basename(f)))
158# Removes any files which are old orphans (not associated with a valid .changes file).
159# [Used for Incoming]
160#
163def flush_orphans() -> None:
164 all_files = {}
165 changes_files = []
167 Logger.log(["check Incoming for old orphaned files", os.getcwd()])
168 # Build up the list of all files in the directory
169 for i in os.listdir("."):
170 if os.path.isfile(i):
171 all_files[i] = 1
172 if i.endswith(".changes"):
173 changes_files.append(i)
175 # Proces all .changes and .dsc files.
176 for changes_filename in changes_files:
177 try:
178 changes = utils.parse_changes(changes_filename)
179 files = utils.build_file_list(changes)
180 except:
181 utils.warn(
182 "error processing '%s'; skipping it. [Got %s]"
183 % (changes_filename, sys.exc_info()[0])
184 )
185 continue
187 dsc_files = {}
188 for f in files.keys():
189 if f.endswith(".dsc"):
190 try:
191 dsc = utils.parse_changes(f, dsc_file=True)
192 dsc_files = utils.build_file_list(dsc, is_a_dsc=True)
193 except:
194 utils.warn(
195 "error processing '%s'; skipping it. [Got %s]"
196 % (f, sys.exc_info()[0])
197 )
198 continue
200 # Ensure all the files we've seen aren't deleted
201 keys = [*files.keys(), *dsc_files.keys(), changes_filename]
202 for key in keys:
203 if key in all_files:
204 if Options["Verbose"]:
205 print("Skipping, has parents, '%s'." % (key))
206 del all_files[key]
208 # Anthing left at this stage is not referenced by a .changes (or
209 # a .dsc) and should be deleted if old enough.
210 for f in all_files.keys():
211 if os.stat(f)[stat.ST_MTIME] < delete_date:
212 remove("Incoming", f)
213 else:
214 if Options["Verbose"]:
215 print("Skipping, too new, '%s'." % (os.path.basename(f)))
218################################################################################
221def main() -> None:
222 global Options, Logger
224 cnf = Config()
226 for i in ["Help", "Incoming", "No-Action", "Verbose"]:
227 key = "Clean-Queues::Options::%s" % i
228 if key not in cnf: 228 ↛ 226line 228 didn't jump to line 226 because the condition on line 228 was always true
229 cnf[key] = ""
230 if "Clean-Queues::Options::Days" not in cnf: 230 ↛ 233line 230 didn't jump to line 233
231 cnf["Clean-Queues::Options::Days"] = "14"
233 Arguments = [
234 ("h", "help", "Clean-Queues::Options::Help"),
235 ("d", "days", "Clean-Queues::Options::Days", "IntLevel"),
236 ("i", "incoming", "Clean-Queues::Options::Incoming", "HasArg"),
237 ("n", "no-action", "Clean-Queues::Options::No-Action"),
238 ("v", "verbose", "Clean-Queues::Options::Verbose"),
239 ]
241 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) # type: ignore[attr-defined]
242 Options = cnf.subtree("Clean-Queues::Options")
244 if Options["Help"]: 244 ↛ 247line 244 didn't jump to line 247 because the condition on line 244 was always true
245 usage()
247 Logger = daklog.Logger("clean-queues", Options["No-Action"])
249 init(cnf)
251 if Options["Verbose"]:
252 print("Processing incoming...")
253 flush_orphans()
255 reject = cnf["Dir::Reject"]
256 if os.path.exists(reject) and os.path.isdir(reject):
257 if Options["Verbose"]:
258 print("Processing reject directory...")
259 os.chdir(reject)
260 flush_old()
262 Logger.close()