Coverage for dak/archive_dedup_pool.py: 56%

1"""De-duplicates files in the pool directory 

2 

3@contact: Debian FTP Master <ftpmaster@debian.org> 

4@copyright: 2017 Bastian Blank <waldi@debian.org> 

5@license: GNU General Public License version 2 or later 

6""" 

7 

8# This program is free software; you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation; either version 2 of the License, or 

11# (at your option) any later version. 

12 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17 

18# You should have received a copy of the GNU General Public License 

19# along with this program; if not, write to the Free Software 

20# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 

21 

22################################################################################ 

23 

24import errno 

25import os 

26import sys 

27from typing import TYPE_CHECKING 

28 

29import apt_pkg 

30from sqlalchemy import sql 

31 

32from daklib import daklog 

33from daklib.config import Config 

34from daklib.dbconn import DBConn 

35 

36if TYPE_CHECKING: 

37 from sqlalchemy.orm import Session 

38 

39Options: apt_pkg.Configuration 

40Logger: daklog.Logger 

41 

42################################################################################ 

43################################################################################ 

44################################################################################ 

45 

46 

47def usage(exit_code=0): 

48 print( 

49 """Usage: dak archive-dedup-pool [OPTION]... 

50 -h, --help show this help and exit. 

51 -V, --version display the version number and exit 

52""" 

53 ) 

54 sys.exit(exit_code) 

55 

56 

57################################################################################ 

58 

59 

60def dedup_one(size: int, reference: str, *filenames: str) -> None: 

61 stat_reference = os.stat(reference) 

62 

63 # safety net 

64 if stat_reference.st_size != size: 

65 raise RuntimeError( 

66 "Size of {} does not match database: {} != {}".format( 

67 reference, size, stat_reference.st_size 

68 ) 

69 ) 

70 

71 for filename in filenames: 

72 stat_filename = os.stat(filename) 

73 

74 # if file is already a hard-linked, ignore 

75 if stat_reference == stat_filename: 

76 continue 

77 

78 # safety net 

79 if stat_filename.st_size != size: 

80 raise RuntimeError( 

81 "Size of {} does not match database: {} != {}".format( 

82 filename, size, stat_filename.st_size 

83 ) 

84 ) 

85 

86 tempfile = filename + ".new" 

87 os.link(reference, tempfile) 

88 try: 

89 Logger.log(["deduplicate", filename, reference]) 

90 os.rename(tempfile, filename) 

91 finally: 

92 try: 

93 os.unlink(tempfile) 

94 except OSError as e: 

95 if e.errno != errno.ENOENT: 

96 raise 

97 

98 

99################################################################################ 

100 

101 

102def dedup(session: "Session") -> None: 

103 results = session.execute( 

104 sql.text( 

105 """ 

106SELECT DISTINCT * 

107 FROM ( 

108 SELECT 

109 f.size, 

110 array_agg(a.path || '/pool/' || c.name || '/' || f.filename) OVER ( 

111 -- we aggregate all files with the same size, sha256sum and archive 

112 PARTITION BY f.size, f.sha256sum, a.id 

113 -- the oldest should be first 

114 ORDER by f.created 

115 -- we always want to see all rows 

116 ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING 

117 ) 

118 AS filenames 

119 FROM 

120 files AS f INNER JOIN 

121 files_archive_map AS fa ON f.id = fa.file_id INNER JOIN 

122 component c ON fa.component_id = c.id INNER JOIN 

123 archive a ON fa.archive_id = a.id 

124 ) AS f 

125 -- we only care about entries with more than one filename 

126 WHERE array_length(filenames, 1) > 1 

127 """ 

128 ) 

129 ).mappings() 

130 

131 for i in results: 131 ↛ 132line 131 didn't jump to line 132 because the loop on line 131 never started

132 dedup_one(i["size"], *i["filenames"]) 

133 

134 

135################################################################################ 

136 

137 

138def main(): 

139 global Options, Logger 

140 

141 cnf = Config() 

142 session = DBConn().session() 

143 

144 Arguments = [("h", "help", "Archive-Dedup-Pool::Options::Help")] 

145 

146 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) # type: ignore[attr-defined] 

147 

148 for i in ["help"]: 

149 key = "Archive-Dedup-Pool::Options::%s" % i 

150 if key not in cnf: 

151 cnf[key] = "" 

152 

153 Options = cnf.subtree("Archive-Dedup-Pool::Options") 

154 

155 if Options["Help"]: 

156 usage() 

157 

158 Logger = daklog.Logger("archive-dedup-pool") 

159 

160 dedup(session) 

161 

162 Logger.close()