dak.clean_queues

1 #! /usr/bin/env python3 2 3 """ Clean incoming of old unused files """ 4 # Copyright (C) 2000, 2001, 2002, 2006 James Troup <james@nocrew.org> 5 6 # This program is free software; you can redistribute it and/or modify 7 # it under the terms of the GNU General Public License as published by 8 # the Free Software Foundation; either version 2 of the License, or 9 # (at your option) any later version. 10 11 # This program is distributed in the hope that it will be useful, 12 # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 # GNU General Public License for more details. 15 16 # You should have received a copy of the GNU General Public License 17 # along with this program; if not, write to the Free Software 18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 19 20 ################################################################################ 21 22 # <aj> Bdale, a ham-er, and the leader, 23 # <aj> Willy, a GCC maintainer, 24 # <aj> Lamont-work, 'cause he's the top uploader.... 25 # <aj> Penguin Puff' save the day! 26 # <aj> Porting code, trying to build the world, 27 # <aj> Here they come just in time... 28 # <aj> The Penguin Puff' Guys! 29 # <aj> [repeat] 30 # <aj> Penguin Puff'! 31 # <aj> willy: btw, if you don't maintain gcc you need to start, since 32 # the lyrics fit really well that way 33 34 ################################################################################ 35 36 import os 37 import os.path 38 import stat 39 import sys 40 import time 41 from datetime import datetime 42 import apt_pkg 43 from daklib import utils 44 from daklib import daklog 45 from daklib.config import Config 46 47 ################################################################################ 48 49 Options = None 50 Logger = None 51 del_dir = None 52 delete_date = None 53 54 ################################################################################ 55 56

57 -def usage(exit_code=0):

58 print("""Usage: dak clean-queues [OPTIONS] 59 Clean out incoming directories. 60 61 -d, --days=DAYS remove anything older than DAYS old 62 -i, --incoming=INCOMING the incoming directory to clean 63 -n, --no-action don't do anything 64 -v, --verbose explain what is being done 65 -h, --help show this help and exit""") 66 67 sys.exit(exit_code)

68 69 ################################################################################ 70 71

72 -def init(cnf):

73 global delete_date, del_dir 74 75 # Used for directory naming 76 now_date = datetime.now() 77 78 # Used for working out times 79 delete_date = int(time.time()) - (int(Options["Days"]) * 84600) 80 81 morguedir = cnf.get("Dir::Morgue", os.path.join("Dir::Pool", 'morgue')) 82 morguesubdir = cnf.get("Clean-Queues::MorgueSubDir", 'queue') 83 84 # Build directory as morguedir/morguesubdir/year/month/day 85 del_dir = os.path.join(morguedir, 86 morguesubdir, 87 str(now_date.year), 88 '%.2d' % now_date.month, 89 '%.2d' % now_date.day) 90 91 # Ensure a directory exists to remove files to 92 if not Options["No-Action"]: 93 if not os.path.exists(del_dir): 94 os.makedirs(del_dir, 0o2775) 95 if not os.path.isdir(del_dir): 96 utils.fubar("%s must be a directory." % (del_dir)) 97 98 # Move to the directory to clean 99 incoming = Options.get("Incoming") 100 if not incoming: 101 incoming = cnf.get('Dir::Unchecked') 102 if not incoming: 103 utils.fubar("Cannot find 'unchecked' directory") 104 105 try: 106 os.chdir(incoming) 107 except OSError as e: 108 utils.fubar("Cannot chdir to %s" % incoming)

109 110 # Remove a file to the morgue 111 112

113 -def remove(from_dir, f):

114 fname = os.path.basename(f) 115 if os.access(f, os.R_OK): 116 Logger.log(["move file to morgue", from_dir, fname, del_dir]) 117 if Options["Verbose"]: 118 print("Removing '%s' (to '%s')." % (fname, del_dir)) 119 if Options["No-Action"]: 120 return 121 122 dest_filename = os.path.join(del_dir, fname) 123 # If the destination file exists; try to find another filename to use 124 if os.path.exists(dest_filename): 125 dest_filename = utils.find_next_free(dest_filename, 10) 126 Logger.log(["change destination file name", os.path.basename(dest_filename)]) 127 utils.move(f, dest_filename, 0o660) 128 else: 129 Logger.log(["skipping file because of permission problem", fname]) 130 utils.warn("skipping '%s', permission denied." % fname)

131 132 # Removes any old files. 133 # [Used for Incoming/REJECT] 134 # 135 136

137 -def flush_old():

138 Logger.log(["check Incoming/REJECT for old files", os.getcwd()]) 139 for f in os.listdir('.'): 140 if os.path.isfile(f): 141 if os.stat(f)[stat.ST_MTIME] < delete_date: 142 remove('Incoming/REJECT', f) 143 else: 144 if Options["Verbose"]: 145 print("Skipping, too new, '%s'." % (os.path.basename(f)))

146 147 # Removes any files which are old orphans (not associated with a valid .changes file). 148 # [Used for Incoming] 149 # 150 151

152 -def flush_orphans():

153 all_files = {} 154 changes_files = [] 155 156 Logger.log(["check Incoming for old orphaned files", os.getcwd()]) 157 # Build up the list of all files in the directory 158 for i in os.listdir('.'): 159 if os.path.isfile(i): 160 all_files[i] = 1 161 if i.endswith(".changes"): 162 changes_files.append(i) 163 164 # Proces all .changes and .dsc files. 165 for changes_filename in changes_files: 166 try: 167 changes = utils.parse_changes(changes_filename) 168 files = utils.build_file_list(changes) 169 except: 170 utils.warn("error processing '%s'; skipping it. [Got %s]" % (changes_filename, sys.exc_info()[0])) 171 continue 172 173 dsc_files = {} 174 for f in files.keys(): 175 if f.endswith(".dsc"): 176 try: 177 dsc = utils.parse_changes(f, dsc_file=True) 178 dsc_files = utils.build_file_list(dsc, is_a_dsc=True) 179 except: 180 utils.warn("error processing '%s'; skipping it. [Got %s]" % (f, sys.exc_info()[0])) 181 continue 182 183 # Ensure all the files we've seen aren't deleted 184 keys = [] 185 for i in (files.keys(), dsc_files.keys(), [changes_filename]): 186 keys.extend(i) 187 for key in keys: 188 if key in all_files: 189 if Options["Verbose"]: 190 print("Skipping, has parents, '%s'." % (key)) 191 del all_files[key] 192 193 # Anthing left at this stage is not referenced by a .changes (or 194 # a .dsc) and should be deleted if old enough. 195 for f in all_files.keys(): 196 if os.stat(f)[stat.ST_MTIME] < delete_date: 197 remove('Incoming', f) 198 else: 199 if Options["Verbose"]: 200 print("Skipping, too new, '%s'." % (os.path.basename(f)))

201 202 ################################################################################ 203 204

205 -def main():

206 global Options, Logger 207 208 cnf = Config() 209 210 for i in ["Help", "Incoming", "No-Action", "Verbose"]: 211 key = "Clean-Queues::Options::%s" % i 212 if key not in cnf: 213 cnf[key] = "" 214 if "Clean-Queues::Options::Days" not in cnf: 215 cnf["Clean-Queues::Options::Days"] = "14" 216 217 Arguments = [('h', "help", "Clean-Queues::Options::Help"), 218 ('d', "days", "Clean-Queues::Options::Days", "IntLevel"), 219 ('i', "incoming", "Clean-Queues::Options::Incoming", "HasArg"), 220 ('n', "no-action", "Clean-Queues::Options::No-Action"), 221 ('v', "verbose", "Clean-Queues::Options::Verbose")] 222 223 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) 224 Options = cnf.subtree("Clean-Queues::Options") 225 226 if Options["Help"]: 227 usage() 228 229 Logger = daklog.Logger('clean-queues', Options['No-Action']) 230 231 init(cnf) 232 233 if Options["Verbose"]: 234 print("Processing incoming...") 235 flush_orphans() 236 237 reject = cnf["Dir::Reject"] 238 if os.path.exists(reject) and os.path.isdir(reject): 239 if Options["Verbose"]: 240 print("Processing reject directory...") 241 os.chdir(reject) 242 flush_old() 243 244 Logger.close()

245 246 ####################################################################################### 247 248 249 if __name__ == '__main__': 250 main() 251

Source Code for Module dak.clean_queues