1#! /usr/bin/env python3
3""" Clean incoming of old unused files """
4# Copyright (C) 2000, 2001, 2002, 2006 James Troup <james@nocrew.org>
6# This program is free software; you can redistribute it and/or modify
7# it under the terms of the GNU General Public License as published by
8# the Free Software Foundation; either version 2 of the License, or
9# (at your option) any later version.
11# This program is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14# GNU General Public License for more details.
16# You should have received a copy of the GNU General Public License
17# along with this program; if not, write to the Free Software
18# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20################################################################################
22# <aj> Bdale, a ham-er, and the leader,
23# <aj> Willy, a GCC maintainer,
24# <aj> Lamont-work, 'cause he's the top uploader....
25# <aj> Penguin Puff' save the day!
26# <aj> Porting code, trying to build the world,
27# <aj> Here they come just in time...
28# <aj> The Penguin Puff' Guys!
29# <aj> [repeat]
30# <aj> Penguin Puff'!
31# <aj> willy: btw, if you don't maintain gcc you need to start, since
32# the lyrics fit really well that way
34################################################################################
36import os
37import os.path
38import stat
39import sys
40import time
41from datetime import datetime
42import apt_pkg
43from daklib import utils
44from daklib import daklog
45from daklib.config import Config
47################################################################################
49Options = None
50Logger = None
51del_dir = None
52delete_date = None
54################################################################################
57def usage(exit_code=0):
58 print("""Usage: dak clean-queues [OPTIONS]
59Clean out incoming directories.
61 -d, --days=DAYS remove anything older than DAYS old
62 -i, --incoming=INCOMING the incoming directory to clean
63 -n, --no-action don't do anything
64 -v, --verbose explain what is being done
65 -h, --help show this help and exit""")
67 sys.exit(exit_code)
69################################################################################
72def init(cnf):
73 global delete_date, del_dir
75 # Used for directory naming
76 now_date = datetime.now()
78 # Used for working out times
79 delete_date = int(time.time()) - (int(Options["Days"]) * 84600)
81 morguedir = cnf.get("Dir::Morgue", os.path.join("Dir::Pool", 'morgue'))
82 morguesubdir = cnf.get("Clean-Queues::MorgueSubDir", 'queue')
84 # Build directory as morguedir/morguesubdir/year/month/day
85 del_dir = os.path.join(morguedir,
86 morguesubdir,
87 str(now_date.year),
88 '%.2d' % now_date.month,
89 '%.2d' % now_date.day)
91 # Ensure a directory exists to remove files to
92 if not Options["No-Action"]:
93 if not os.path.exists(del_dir):
94 os.makedirs(del_dir, 0o2775)
95 if not os.path.isdir(del_dir):
96 utils.fubar("%s must be a directory." % (del_dir))
98 # Move to the directory to clean
99 incoming = Options.get("Incoming")
100 if not incoming:
101 incoming = cnf.get('Dir::Unchecked')
102 if not incoming:
103 utils.fubar("Cannot find 'unchecked' directory")
105 try:
106 os.chdir(incoming)
107 except OSError as e:
108 utils.fubar("Cannot chdir to %s" % incoming)
110# Remove a file to the morgue
113def remove(from_dir, f):
114 fname = os.path.basename(f)
115 if os.access(f, os.R_OK):
116 Logger.log(["move file to morgue", from_dir, fname, del_dir])
117 if Options["Verbose"]:
118 print("Removing '%s' (to '%s')." % (fname, del_dir))
119 if Options["No-Action"]:
120 return
122 dest_filename = os.path.join(del_dir, fname)
123 # If the destination file exists; try to find another filename to use
124 if os.path.exists(dest_filename):
125 dest_filename = utils.find_next_free(dest_filename, 10)
126 Logger.log(["change destination file name", os.path.basename(dest_filename)])
127 utils.move(f, dest_filename, 0o660)
128 else:
129 Logger.log(["skipping file because of permission problem", fname])
130 utils.warn("skipping '%s', permission denied." % fname)
132# Removes any old files.
133# [Used for Incoming/REJECT]
134#
137def flush_old():
138 Logger.log(["check Incoming/REJECT for old files", os.getcwd()])
139 for f in os.listdir('.'):
140 if os.path.isfile(f):
141 if os.stat(f)[stat.ST_MTIME] < delete_date:
142 remove('Incoming/REJECT', f)
143 else:
144 if Options["Verbose"]:
145 print("Skipping, too new, '%s'." % (os.path.basename(f)))
147# Removes any files which are old orphans (not associated with a valid .changes file).
148# [Used for Incoming]
149#
152def flush_orphans():
153 all_files = {}
154 changes_files = []
156 Logger.log(["check Incoming for old orphaned files", os.getcwd()])
157 # Build up the list of all files in the directory
158 for i in os.listdir('.'):
159 if os.path.isfile(i):
160 all_files[i] = 1
161 if i.endswith(".changes"):
162 changes_files.append(i)
164 # Proces all .changes and .dsc files.
165 for changes_filename in changes_files:
166 try:
167 changes = utils.parse_changes(changes_filename)
168 files = utils.build_file_list(changes)
169 except:
170 utils.warn("error processing '%s'; skipping it. [Got %s]" % (changes_filename, sys.exc_info()[0]))
171 continue
173 dsc_files = {}
174 for f in files.keys():
175 if f.endswith(".dsc"):
176 try:
177 dsc = utils.parse_changes(f, dsc_file=True)
178 dsc_files = utils.build_file_list(dsc, is_a_dsc=True)
179 except:
180 utils.warn("error processing '%s'; skipping it. [Got %s]" % (f, sys.exc_info()[0]))
181 continue
183 # Ensure all the files we've seen aren't deleted
184 keys = []
185 for i in (files.keys(), dsc_files.keys(), [changes_filename]):
186 keys.extend(i)
187 for key in keys:
188 if key in all_files:
189 if Options["Verbose"]:
190 print("Skipping, has parents, '%s'." % (key))
191 del all_files[key]
193 # Anthing left at this stage is not referenced by a .changes (or
194 # a .dsc) and should be deleted if old enough.
195 for f in all_files.keys():
196 if os.stat(f)[stat.ST_MTIME] < delete_date:
197 remove('Incoming', f)
198 else:
199 if Options["Verbose"]:
200 print("Skipping, too new, '%s'." % (os.path.basename(f)))
202################################################################################
205def main():
206 global Options, Logger
208 cnf = Config()
210 for i in ["Help", "Incoming", "No-Action", "Verbose"]:
211 key = "Clean-Queues::Options::%s" % i
212 if key not in cnf: 212 ↛ 210line 212 didn't jump to line 210, because the condition on line 212 was never false
213 cnf[key] = ""
214 if "Clean-Queues::Options::Days" not in cnf: 214 ↛ 217line 214 didn't jump to line 217, because the condition on line 214 was never false
215 cnf["Clean-Queues::Options::Days"] = "14"
217 Arguments = [('h', "help", "Clean-Queues::Options::Help"),
218 ('d', "days", "Clean-Queues::Options::Days", "IntLevel"),
219 ('i', "incoming", "Clean-Queues::Options::Incoming", "HasArg"),
220 ('n', "no-action", "Clean-Queues::Options::No-Action"),
221 ('v', "verbose", "Clean-Queues::Options::Verbose")]
223 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
224 Options = cnf.subtree("Clean-Queues::Options")
226 if Options["Help"]: 226 ↛ 229line 226 didn't jump to line 229, because the condition on line 226 was never false
227 usage()
229 Logger = daklog.Logger('clean-queues', Options['No-Action'])
231 init(cnf)
233 if Options["Verbose"]:
234 print("Processing incoming...")
235 flush_orphans()
237 reject = cnf["Dir::Reject"]
238 if os.path.exists(reject) and os.path.isdir(reject):
239 if Options["Verbose"]:
240 print("Processing reject directory...")
241 os.chdir(reject)
242 flush_old()
244 Logger.close()
246#######################################################################################
249if __name__ == '__main__':
250 main()