Coverage for dak/clean_queues.py: 21%
121 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-01-04 16:18 +0000
1#! /usr/bin/env python3
3"""Clean incoming of old unused files"""
4# Copyright (C) 2000, 2001, 2002, 2006 James Troup <james@nocrew.org>
6# This program is free software; you can redistribute it and/or modify
7# it under the terms of the GNU General Public License as published by
8# the Free Software Foundation; either version 2 of the License, or
9# (at your option) any later version.
11# This program is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14# GNU General Public License for more details.
16# You should have received a copy of the GNU General Public License
17# along with this program; if not, write to the Free Software
18# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20################################################################################
22# <aj> Bdale, a ham-er, and the leader,
23# <aj> Willy, a GCC maintainer,
24# <aj> Lamont-work, 'cause he's the top uploader....
25# <aj> Penguin Puff' save the day!
26# <aj> Porting code, trying to build the world,
27# <aj> Here they come just in time...
28# <aj> The Penguin Puff' Guys!
29# <aj> [repeat]
30# <aj> Penguin Puff'!
31# <aj> willy: btw, if you don't maintain gcc you need to start, since
32# the lyrics fit really well that way
34################################################################################
36import os
37import os.path
38import stat
39import sys
40import time
41from datetime import datetime
42from typing import NoReturn
44import apt_pkg
46from daklib import daklog, utils
47from daklib.config import Config
49################################################################################
51Options: apt_pkg.Configuration
52Logger: daklog.Logger
53del_dir: str
54delete_date: int
56################################################################################
59def usage(exit_code=0) -> NoReturn:
60 print(
61 """Usage: dak clean-queues [OPTIONS]
62Clean out incoming directories.
64 -d, --days=DAYS remove anything older than DAYS old
65 -i, --incoming=INCOMING the incoming directory to clean
66 -n, --no-action don't do anything
67 -v, --verbose explain what is being done
68 -h, --help show this help and exit"""
69 )
71 sys.exit(exit_code)
74################################################################################
77def init(cnf) -> None:
78 global delete_date, del_dir
80 # Used for directory naming
81 now_date = datetime.now()
83 # Used for working out times
84 delete_date = int(time.time()) - (int(Options["Days"]) * 84600)
86 morguedir = cnf.get("Dir::Morgue", os.path.join("Dir::Pool", "morgue"))
87 morguesubdir = cnf.get("Clean-Queues::MorgueSubDir", "queue")
89 # Build directory as morguedir/morguesubdir/year/month/day
90 del_dir = os.path.join(
91 morguedir,
92 morguesubdir,
93 str(now_date.year),
94 "%.2d" % now_date.month,
95 "%.2d" % now_date.day,
96 )
98 # Ensure a directory exists to remove files to
99 if not Options["No-Action"]:
100 if not os.path.exists(del_dir):
101 os.makedirs(del_dir, 0o2775)
102 if not os.path.isdir(del_dir):
103 utils.fubar("%s must be a directory." % (del_dir))
105 # Move to the directory to clean
106 incoming = Options.get("Incoming")
107 if not incoming:
108 incoming = cnf.get("Dir::Unchecked")
109 if not incoming:
110 utils.fubar("Cannot find 'unchecked' directory")
112 try:
113 os.chdir(incoming)
114 except OSError as e:
115 utils.fubar(f"Cannot chdir to {incoming}: {e}")
118# Remove a file to the morgue
121def remove(from_dir: str, f: str) -> None:
122 fname = os.path.basename(f)
123 if os.access(f, os.R_OK):
124 Logger.log(["move file to morgue", from_dir, fname, del_dir])
125 if Options["Verbose"]:
126 print("Removing '%s' (to '%s')." % (fname, del_dir))
127 if Options["No-Action"]:
128 return
130 dest_filename = os.path.join(del_dir, fname)
131 # If the destination file exists; try to find another filename to use
132 if os.path.exists(dest_filename):
133 dest_filename = utils.find_next_free(dest_filename, 10)
134 Logger.log(
135 ["change destination file name", os.path.basename(dest_filename)]
136 )
137 utils.move(f, dest_filename, perms=0o660)
138 else:
139 Logger.log(["skipping file because of permission problem", fname])
140 utils.warn("skipping '%s', permission denied." % fname)
143# Removes any old files.
144# [Used for Incoming/REJECT]
145#
148def flush_old() -> None:
149 Logger.log(["check Incoming/REJECT for old files", os.getcwd()])
150 for f in os.listdir("."):
151 if os.path.isfile(f):
152 if os.stat(f)[stat.ST_MTIME] < delete_date:
153 remove("Incoming/REJECT", f)
154 else:
155 if Options["Verbose"]:
156 print("Skipping, too new, '%s'." % (os.path.basename(f)))
159# Removes any files which are old orphans (not associated with a valid .changes file).
160# [Used for Incoming]
161#
164def flush_orphans() -> None:
165 all_files = {}
166 changes_files = []
168 Logger.log(["check Incoming for old orphaned files", os.getcwd()])
169 # Build up the list of all files in the directory
170 for i in os.listdir("."):
171 if os.path.isfile(i):
172 all_files[i] = 1
173 if i.endswith(".changes"):
174 changes_files.append(i)
176 # Proces all .changes and .dsc files.
177 for changes_filename in changes_files:
178 try:
179 changes = utils.parse_changes(changes_filename)
180 files = utils.build_file_list(changes)
181 except:
182 utils.warn(
183 "error processing '%s'; skipping it. [Got %s]"
184 % (changes_filename, sys.exc_info()[0])
185 )
186 continue
188 dsc_files = {}
189 for f in files.keys():
190 if f.endswith(".dsc"):
191 try:
192 dsc = utils.parse_changes(f, dsc_file=True)
193 dsc_files = utils.build_file_list(dsc, is_a_dsc=True)
194 except:
195 utils.warn(
196 "error processing '%s'; skipping it. [Got %s]"
197 % (f, sys.exc_info()[0])
198 )
199 continue
201 # Ensure all the files we've seen aren't deleted
202 keys = [*files.keys(), *dsc_files.keys(), changes_filename]
203 for key in keys:
204 if key in all_files:
205 if Options["Verbose"]:
206 print("Skipping, has parents, '%s'." % (key))
207 del all_files[key]
209 # Anthing left at this stage is not referenced by a .changes (or
210 # a .dsc) and should be deleted if old enough.
211 for f in all_files.keys():
212 if os.stat(f)[stat.ST_MTIME] < delete_date:
213 remove("Incoming", f)
214 else:
215 if Options["Verbose"]:
216 print("Skipping, too new, '%s'." % (os.path.basename(f)))
219################################################################################
222def main() -> None:
223 global Options, Logger
225 cnf = Config()
227 for i in ["Help", "Incoming", "No-Action", "Verbose"]:
228 key = "Clean-Queues::Options::%s" % i
229 if key not in cnf: 229 ↛ 227line 229 didn't jump to line 227 because the condition on line 229 was always true
230 cnf[key] = ""
231 if "Clean-Queues::Options::Days" not in cnf: 231 ↛ 234line 231 didn't jump to line 234
232 cnf["Clean-Queues::Options::Days"] = "14"
234 Arguments = [
235 ("h", "help", "Clean-Queues::Options::Help"),
236 ("d", "days", "Clean-Queues::Options::Days", "IntLevel"),
237 ("i", "incoming", "Clean-Queues::Options::Incoming", "HasArg"),
238 ("n", "no-action", "Clean-Queues::Options::No-Action"),
239 ("v", "verbose", "Clean-Queues::Options::Verbose"),
240 ]
242 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) # type: ignore[attr-defined]
243 Options = cnf.subtree("Clean-Queues::Options")
245 if Options["Help"]: 245 ↛ 248line 245 didn't jump to line 248 because the condition on line 245 was always true
246 usage()
248 Logger = daklog.Logger("clean-queues", Options["No-Action"])
250 init(cnf)
252 if Options["Verbose"]:
253 print("Processing incoming...")
254 flush_orphans()
256 reject = cnf["Dir::Reject"]
257 if os.path.exists(reject) and os.path.isdir(reject):
258 if Options["Verbose"]:
259 print("Processing reject directory...")
260 os.chdir(reject)
261 flush_old()
263 Logger.close()
266#######################################################################################
269if __name__ == "__main__":
270 main()