Package dak :: Module clean_queues
[hide private]
[frames] | no frames]

Source Code for Module dak.clean_queues

  1  #! /usr/bin/env python3 
  2   
  3  """ Clean incoming of old unused files """ 
  4  # Copyright (C) 2000, 2001, 2002, 2006  James Troup <james@nocrew.org> 
  5   
  6  # This program is free software; you can redistribute it and/or modify 
  7  # it under the terms of the GNU General Public License as published by 
  8  # the Free Software Foundation; either version 2 of the License, or 
  9  # (at your option) any later version. 
 10   
 11  # This program is distributed in the hope that it will be useful, 
 12  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 13  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 14  # GNU General Public License for more details. 
 15   
 16  # You should have received a copy of the GNU General Public License 
 17  # along with this program; if not, write to the Free Software 
 18  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
 19   
 20  ################################################################################ 
 21   
 22  # <aj> Bdale, a ham-er, and the leader, 
 23  # <aj> Willy, a GCC maintainer, 
 24  # <aj> Lamont-work, 'cause he's the top uploader.... 
 25  # <aj>         Penguin Puff' save the day! 
 26  # <aj> Porting code, trying to build the world, 
 27  # <aj> Here they come just in time... 
 28  # <aj>         The Penguin Puff' Guys! 
 29  # <aj> [repeat] 
 30  # <aj> Penguin Puff'! 
 31  # <aj> willy: btw, if you don't maintain gcc you need to start, since 
 32  #      the lyrics fit really well that way 
 33   
 34  ################################################################################ 
 35   
 36  import os 
 37  import os.path 
 38  import stat 
 39  import sys 
 40  import time 
 41  from datetime import datetime 
 42  import apt_pkg 
 43  from daklib import utils 
 44  from daklib import daklog 
 45  from daklib.config import Config 
 46   
 47  ################################################################################ 
 48   
 49  Options = None 
 50  Logger = None 
 51  del_dir = None 
 52  delete_date = None 
 53   
 54  ################################################################################ 
 55   
 56   
57 -def usage(exit_code=0):
58 print("""Usage: dak clean-queues [OPTIONS] 59 Clean out incoming directories. 60 61 -d, --days=DAYS remove anything older than DAYS old 62 -i, --incoming=INCOMING the incoming directory to clean 63 -n, --no-action don't do anything 64 -v, --verbose explain what is being done 65 -h, --help show this help and exit""") 66 67 sys.exit(exit_code)
68 69 ################################################################################ 70 71
72 -def init(cnf):
73 global delete_date, del_dir 74 75 # Used for directory naming 76 now_date = datetime.now() 77 78 # Used for working out times 79 delete_date = int(time.time()) - (int(Options["Days"]) * 84600) 80 81 morguedir = cnf.get("Dir::Morgue", os.path.join("Dir::Pool", 'morgue')) 82 morguesubdir = cnf.get("Clean-Queues::MorgueSubDir", 'queue') 83 84 # Build directory as morguedir/morguesubdir/year/month/day 85 del_dir = os.path.join(morguedir, 86 morguesubdir, 87 str(now_date.year), 88 '%.2d' % now_date.month, 89 '%.2d' % now_date.day) 90 91 # Ensure a directory exists to remove files to 92 if not Options["No-Action"]: 93 if not os.path.exists(del_dir): 94 os.makedirs(del_dir, 0o2775) 95 if not os.path.isdir(del_dir): 96 utils.fubar("%s must be a directory." % (del_dir)) 97 98 # Move to the directory to clean 99 incoming = Options.get("Incoming") 100 if not incoming: 101 incoming = cnf.get('Dir::Unchecked') 102 if not incoming: 103 utils.fubar("Cannot find 'unchecked' directory") 104 105 try: 106 os.chdir(incoming) 107 except OSError as e: 108 utils.fubar("Cannot chdir to %s" % incoming)
109 110 # Remove a file to the morgue 111 112
113 -def remove(from_dir, f):
114 fname = os.path.basename(f) 115 if os.access(f, os.R_OK): 116 Logger.log(["move file to morgue", from_dir, fname, del_dir]) 117 if Options["Verbose"]: 118 print("Removing '%s' (to '%s')." % (fname, del_dir)) 119 if Options["No-Action"]: 120 return 121 122 dest_filename = os.path.join(del_dir, fname) 123 # If the destination file exists; try to find another filename to use 124 if os.path.exists(dest_filename): 125 dest_filename = utils.find_next_free(dest_filename, 10) 126 Logger.log(["change destination file name", os.path.basename(dest_filename)]) 127 utils.move(f, dest_filename, 0o660) 128 else: 129 Logger.log(["skipping file because of permission problem", fname]) 130 utils.warn("skipping '%s', permission denied." % fname)
131 132 # Removes any old files. 133 # [Used for Incoming/REJECT] 134 # 135 136
137 -def flush_old():
138 Logger.log(["check Incoming/REJECT for old files", os.getcwd()]) 139 for f in os.listdir('.'): 140 if os.path.isfile(f): 141 if os.stat(f)[stat.ST_MTIME] < delete_date: 142 remove('Incoming/REJECT', f) 143 else: 144 if Options["Verbose"]: 145 print("Skipping, too new, '%s'." % (os.path.basename(f)))
146 147 # Removes any files which are old orphans (not associated with a valid .changes file). 148 # [Used for Incoming] 149 # 150 151
152 -def flush_orphans():
153 all_files = {} 154 changes_files = [] 155 156 Logger.log(["check Incoming for old orphaned files", os.getcwd()]) 157 # Build up the list of all files in the directory 158 for i in os.listdir('.'): 159 if os.path.isfile(i): 160 all_files[i] = 1 161 if i.endswith(".changes"): 162 changes_files.append(i) 163 164 # Proces all .changes and .dsc files. 165 for changes_filename in changes_files: 166 try: 167 changes = utils.parse_changes(changes_filename) 168 files = utils.build_file_list(changes) 169 except: 170 utils.warn("error processing '%s'; skipping it. [Got %s]" % (changes_filename, sys.exc_info()[0])) 171 continue 172 173 dsc_files = {} 174 for f in files.keys(): 175 if f.endswith(".dsc"): 176 try: 177 dsc = utils.parse_changes(f, dsc_file=True) 178 dsc_files = utils.build_file_list(dsc, is_a_dsc=True) 179 except: 180 utils.warn("error processing '%s'; skipping it. [Got %s]" % (f, sys.exc_info()[0])) 181 continue 182 183 # Ensure all the files we've seen aren't deleted 184 keys = [] 185 for i in (files.keys(), dsc_files.keys(), [changes_filename]): 186 keys.extend(i) 187 for key in keys: 188 if key in all_files: 189 if Options["Verbose"]: 190 print("Skipping, has parents, '%s'." % (key)) 191 del all_files[key] 192 193 # Anthing left at this stage is not referenced by a .changes (or 194 # a .dsc) and should be deleted if old enough. 195 for f in all_files.keys(): 196 if os.stat(f)[stat.ST_MTIME] < delete_date: 197 remove('Incoming', f) 198 else: 199 if Options["Verbose"]: 200 print("Skipping, too new, '%s'." % (os.path.basename(f)))
201 202 ################################################################################ 203 204
205 -def main():
206 global Options, Logger 207 208 cnf = Config() 209 210 for i in ["Help", "Incoming", "No-Action", "Verbose"]: 211 key = "Clean-Queues::Options::%s" % i 212 if key not in cnf: 213 cnf[key] = "" 214 if "Clean-Queues::Options::Days" not in cnf: 215 cnf["Clean-Queues::Options::Days"] = "14" 216 217 Arguments = [('h', "help", "Clean-Queues::Options::Help"), 218 ('d', "days", "Clean-Queues::Options::Days", "IntLevel"), 219 ('i', "incoming", "Clean-Queues::Options::Incoming", "HasArg"), 220 ('n', "no-action", "Clean-Queues::Options::No-Action"), 221 ('v', "verbose", "Clean-Queues::Options::Verbose")] 222 223 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) 224 Options = cnf.subtree("Clean-Queues::Options") 225 226 if Options["Help"]: 227 usage() 228 229 Logger = daklog.Logger('clean-queues', Options['No-Action']) 230 231 init(cnf) 232 233 if Options["Verbose"]: 234 print("Processing incoming...") 235 flush_orphans() 236 237 reject = cnf["Dir::Reject"] 238 if os.path.exists(reject) and os.path.isdir(reject): 239 if Options["Verbose"]: 240 print("Processing reject directory...") 241 os.chdir(reject) 242 flush_old() 243 244 Logger.close()
245 246 ####################################################################################### 247 248 249 if __name__ == '__main__': 250 main() 251