dak.clean_queues

1 #! /usr/bin/env python3 2 3 """Clean incoming of old unused files""" 4 # Copyright (C) 2000, 2001, 2002, 2006 James Troup <james@nocrew.org> 5 6 # This program is free software; you can redistribute it and/or modify 7 # it under the terms of the GNU General Public License as published by 8 # the Free Software Foundation; either version 2 of the License, or 9 # (at your option) any later version. 10 11 # This program is distributed in the hope that it will be useful, 12 # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 # GNU General Public License for more details. 15 16 # You should have received a copy of the GNU General Public License 17 # along with this program; if not, write to the Free Software 18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 19 20 ################################################################################ 21 22 # <aj> Bdale, a ham-er, and the leader, 23 # <aj> Willy, a GCC maintainer, 24 # <aj> Lamont-work, 'cause he's the top uploader.... 25 # <aj> Penguin Puff' save the day! 26 # <aj> Porting code, trying to build the world, 27 # <aj> Here they come just in time... 28 # <aj> The Penguin Puff' Guys! 29 # <aj> [repeat] 30 # <aj> Penguin Puff'! 31 # <aj> willy: btw, if you don't maintain gcc you need to start, since 32 # the lyrics fit really well that way 33 34 ################################################################################ 35 36 import os 37 import os.path 38 import stat 39 import sys 40 import time 41 from datetime import datetime 42 43 import apt_pkg 44 45 from daklib import daklog, utils 46 from daklib.config import Config 47 48 ################################################################################ 49 50 Options = None 51 Logger = None 52 del_dir = None 53 delete_date = None 54 55 ################################################################################ 56 57

58 -def usage(exit_code=0):

59 print( 60 """Usage: dak clean-queues [OPTIONS] 61 Clean out incoming directories. 62 63 -d, --days=DAYS remove anything older than DAYS old 64 -i, --incoming=INCOMING the incoming directory to clean 65 -n, --no-action don't do anything 66 -v, --verbose explain what is being done 67 -h, --help show this help and exit""" 68 ) 69 70 sys.exit(exit_code)

71 72 73 ################################################################################ 74 75

76 -def init(cnf):

77 global delete_date, del_dir 78 79 # Used for directory naming 80 now_date = datetime.now() 81 82 # Used for working out times 83 delete_date = int(time.time()) - (int(Options["Days"]) * 84600) 84 85 morguedir = cnf.get("Dir::Morgue", os.path.join("Dir::Pool", "morgue")) 86 morguesubdir = cnf.get("Clean-Queues::MorgueSubDir", "queue") 87 88 # Build directory as morguedir/morguesubdir/year/month/day 89 del_dir = os.path.join( 90 morguedir, 91 morguesubdir, 92 str(now_date.year), 93 "%.2d" % now_date.month, 94 "%.2d" % now_date.day, 95 ) 96 97 # Ensure a directory exists to remove files to 98 if not Options["No-Action"]: 99 if not os.path.exists(del_dir): 100 os.makedirs(del_dir, 0o2775) 101 if not os.path.isdir(del_dir): 102 utils.fubar("%s must be a directory." % (del_dir)) 103 104 # Move to the directory to clean 105 incoming = Options.get("Incoming") 106 if not incoming: 107 incoming = cnf.get("Dir::Unchecked") 108 if not incoming: 109 utils.fubar("Cannot find 'unchecked' directory") 110 111 try: 112 os.chdir(incoming) 113 except OSError as e: 114 utils.fubar(f"Cannot chdir to {incoming}: {e}")

115 116 117 # Remove a file to the morgue 118 119

120 -def remove(from_dir, f):

121 fname = os.path.basename(f) 122 if os.access(f, os.R_OK): 123 Logger.log(["move file to morgue", from_dir, fname, del_dir]) 124 if Options["Verbose"]: 125 print("Removing '%s' (to '%s')." % (fname, del_dir)) 126 if Options["No-Action"]: 127 return 128 129 dest_filename = os.path.join(del_dir, fname) 130 # If the destination file exists; try to find another filename to use 131 if os.path.exists(dest_filename): 132 dest_filename = utils.find_next_free(dest_filename, 10) 133 Logger.log( 134 ["change destination file name", os.path.basename(dest_filename)] 135 ) 136 utils.move(f, dest_filename, 0o660) 137 else: 138 Logger.log(["skipping file because of permission problem", fname]) 139 utils.warn("skipping '%s', permission denied." % fname)

140 141 142 # Removes any old files. 143 # [Used for Incoming/REJECT] 144 # 145 146

147 -def flush_old():

148 Logger.log(["check Incoming/REJECT for old files", os.getcwd()]) 149 for f in os.listdir("."): 150 if os.path.isfile(f): 151 if os.stat(f)[stat.ST_MTIME] < delete_date: 152 remove("Incoming/REJECT", f) 153 else: 154 if Options["Verbose"]: 155 print("Skipping, too new, '%s'." % (os.path.basename(f)))

156 157 158 # Removes any files which are old orphans (not associated with a valid .changes file). 159 # [Used for Incoming] 160 # 161 162

163 -def flush_orphans():

164 all_files = {} 165 changes_files = [] 166 167 Logger.log(["check Incoming for old orphaned files", os.getcwd()]) 168 # Build up the list of all files in the directory 169 for i in os.listdir("."): 170 if os.path.isfile(i): 171 all_files[i] = 1 172 if i.endswith(".changes"): 173 changes_files.append(i) 174 175 # Proces all .changes and .dsc files. 176 for changes_filename in changes_files: 177 try: 178 changes = utils.parse_changes(changes_filename) 179 files = utils.build_file_list(changes) 180 except: 181 utils.warn( 182 "error processing '%s'; skipping it. [Got %s]" 183 % (changes_filename, sys.exc_info()[0]) 184 ) 185 continue 186 187 dsc_files = {} 188 for f in files.keys(): 189 if f.endswith(".dsc"): 190 try: 191 dsc = utils.parse_changes(f, dsc_file=True) 192 dsc_files = utils.build_file_list(dsc, is_a_dsc=True) 193 except: 194 utils.warn( 195 "error processing '%s'; skipping it. [Got %s]" 196 % (f, sys.exc_info()[0]) 197 ) 198 continue 199 200 # Ensure all the files we've seen aren't deleted 201 keys = [] 202 for i in (files.keys(), dsc_files.keys(), [changes_filename]): 203 keys.extend(i) 204 for key in keys: 205 if key in all_files: 206 if Options["Verbose"]: 207 print("Skipping, has parents, '%s'." % (key)) 208 del all_files[key] 209 210 # Anthing left at this stage is not referenced by a .changes (or 211 # a .dsc) and should be deleted if old enough. 212 for f in all_files.keys(): 213 if os.stat(f)[stat.ST_MTIME] < delete_date: 214 remove("Incoming", f) 215 else: 216 if Options["Verbose"]: 217 print("Skipping, too new, '%s'." % (os.path.basename(f)))

218 219 220 ################################################################################ 221 222

223 -def main():

224 global Options, Logger 225 226 cnf = Config() 227 228 for i in ["Help", "Incoming", "No-Action", "Verbose"]: 229 key = "Clean-Queues::Options::%s" % i 230 if key not in cnf: 231 cnf[key] = "" 232 if "Clean-Queues::Options::Days" not in cnf: 233 cnf["Clean-Queues::Options::Days"] = "14" 234 235 Arguments = [ 236 ("h", "help", "Clean-Queues::Options::Help"), 237 ("d", "days", "Clean-Queues::Options::Days", "IntLevel"), 238 ("i", "incoming", "Clean-Queues::Options::Incoming", "HasArg"), 239 ("n", "no-action", "Clean-Queues::Options::No-Action"), 240 ("v", "verbose", "Clean-Queues::Options::Verbose"), 241 ] 242 243 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) 244 Options = cnf.subtree("Clean-Queues::Options") 245 246 if Options["Help"]: 247 usage() 248 249 Logger = daklog.Logger("clean-queues", Options["No-Action"]) 250 251 init(cnf) 252 253 if Options["Verbose"]: 254 print("Processing incoming...") 255 flush_orphans() 256 257 reject = cnf["Dir::Reject"] 258 if os.path.exists(reject) and os.path.isdir(reject): 259 if Options["Verbose"]: 260 print("Processing reject directory...") 261 os.chdir(reject) 262 flush_old() 263 264 Logger.close()

265 266 267 ####################################################################################### 268 269 270 if __name__ == "__main__": 271 main() 272

Source Code for Module dak.clean_queues