Coverage for dak/generate

1# SPDX-License-Identifier: GPL-2.0-or-later

4"""

5Generate a md5sums index of an archive tree

7Walk an archive tree, print a md5sum(1)-compatible index of all

8regular files to stdout, and keep a cache keyed on path, mtime and

9size to avoid re-hashing unchanged files.

11All paths are handled as `bytes`: the index must list filenames exactly

12as they appear on disk.

13"""

15import contextlib

16import hashlib

17import os

18import sys

19from collections.abc import Collection, Iterator

20from dataclasses import dataclass

21from typing import BinaryIO

23import apt_pkg

25from daklib import utils

26from daklib.config import Config

29@dataclass(frozen=True, slots=True)

30class CacheEntry:

31 mtime_ns: int

32 size: int

33 md5: bytes

36type Cache = dict[bytes, CacheEntry]

39def read_cache(fh: BinaryIO) -> Cache:

40 """parse cache records as written by `generate`

42 Returns a mapping of path to `CacheEntry`. Unparsable lines are

43 ignored: the cache is only an optimization, and dropping a record

44 just means the file gets hashed again.

45 """

46 cache: Cache = {}

47 for line in fh:

48 try:

49 md5, mtime_ns, size, path = line.rstrip(b"\n").split(b" ", 3)

50 if len(md5) != 32:

51 continue

52 # discards the value: only checks that the md5 field is

53 # hexadecimal, a corrupt entry raises ValueError

54 int(md5, 16)

55 cache[path] = CacheEntry(int(mtime_ns), int(size), md5)

56 except ValueError:

57 continue

58 return cache

61def load_cache(cache_file: bytes) -> Cache:

62 """read the cache from `cache_file`; a missing file is an empty cache"""

63 try:

64 with open(cache_file, "rb") as fh:

65 return read_cache(fh)

66 except FileNotFoundError:

67 return {}

70def walk_tree(root: bytes) -> Iterator[tuple[bytes, os.stat_result]]:

71 """yield (path relative to `root`, lstat result) for all regular files

73 Directory entries are visited in sorted order. Symbolic links are

74 neither followed nor listed; other non-regular files are skipped.

75 """

77 def walk(dir: bytes, prefix: bytes) -> Iterator[tuple[bytes, os.stat_result]]:

78 with os.scandir(dir) as it:

79 entries = sorted(it, key=lambda entry: entry.name)

80 for entry in entries:

81 if entry.is_dir(follow_symlinks=False):

82 yield from walk(entry.path, prefix + entry.name + b"/")

83 elif entry.is_file(follow_symlinks=False):

84 yield prefix + entry.name, entry.stat(follow_symlinks=False)

86 yield from walk(root, b"")

89def hash_file(path: bytes) -> bytes:

90 with open(path, "rb") as fh:

91 return hashlib.file_digest(fh, "md5").hexdigest().encode("ascii")

94def generate(

95 root: bytes,

96 old_cache: Cache,

97 out: BinaryIO,

98 cache_out: BinaryIO,

99 exclude: Collection[bytes] = frozenset(),

100) -> tuple[int, int]:

101 """write a md5sums index of all regular files below `root` to `out`

102

103 Files whose relative path is listed in `exclude` are skipped. MD5

104 sums are taken from `old_cache` for files whose mtime and size are

105 unchanged and computed otherwise; the new cache is written to

106 `cache_out`. Returns the number of files (hashed, reused).

107 """

108 hashed = reused = 0

109 for path, st in walk_tree(root):

110 if path in exclude:

111 continue

112 if b"\n" in path:

113 raise ValueError(f"filename contains newline: {os.fsdecode(path)!r}")

114 cached = old_cache.get(path)

115 if (

116 cached is not None

117 and cached.mtime_ns == st.st_mtime_ns

118 and cached.size == st.st_size

119 ):

120 md5 = cached.md5

121 reused += 1

122 else:

123 md5 = hash_file(os.path.join(root, path))

124 hashed += 1

125 out.write(md5 + b" " + path + b"\n")

126 cache_out.write(md5 + b" %d %d " % (st.st_mtime_ns, st.st_size) + path + b"\n")

127 return hashed, reused

128

129

130def run(

131 root: bytes,

132 cache_file: bytes,

133 out: BinaryIO,

134 exclude: Collection[bytes] = frozenset(),

135) -> None:

136 """generate the index for `root`, updating `cache_file` atomically"""

137 old_cache = load_cache(cache_file)

138 cache_file_new = cache_file + b".new"

139 try:

140 with open(cache_file_new, "wb") as cache_out:

141 hashed, reused = generate(root, old_cache, out, cache_out, exclude)

142 cache_out.flush()

143 os.fdatasync(cache_out.fileno())

144 except BaseException:

145 with contextlib.suppress(FileNotFoundError):

146 os.unlink(cache_file_new)

147 raise

148 os.replace(cache_file_new, cache_file)

149 print(

150 f"{hashed + reused} files: {hashed} hashed, {reused} reused from cache",

151 file=sys.stderr,

152 )

153

154

155def usage() -> None:

156 print(

157 """Usage: dak generate-md5sums -c <cache-file> [<root>]

158

159Print a md5sum(1)-compatible index of all regular files below <root>

160(default: the current directory) to stdout in a deterministic order.

161MD5 sums are reused from <cache-file> for files whose path, mtime and

162size are unchanged; the cache file is rewritten atomically on success.

163

164 -c, --cache <file> cache file (required)

165 -e, --exclude <path> exclude a path (relative to <root>; may be repeated)

166"""

167 )

168

169

170def main(argv: list[str] | None = None) -> None:

171 if argv is None: 171 ↛ 174line 171 didn't jump to line 174

172 argv = sys.argv

173

174 arguments = [

175 ("h", "help", "Generate-Md5sums::Options::Help"),

176 ("c", "cache", "Generate-Md5sums::Options::Cache", "HasArg"),

177 ("e", "exclude", "Generate-Md5sums::Options::Exclude::", "HasArg"),

178 ]

179

180 cnf = Config()

181 args = apt_pkg.parse_commandline(cnf.Cnf, arguments, argv) # type: ignore[attr-defined]

182 options = cnf.subtree("Generate-Md5sums::Options")

183

184 if "Help" in options: 184 ↛ 185line 184 didn't jump to line 185 because the condition on line 184 was never true

185 usage()

186 sys.exit(0)

187 if "Cache" not in options or len(args) > 1: 187 ↛ 188line 187 didn't jump to line 188 because the condition on line 187 was never true

188 usage()

189 sys.exit(1)

190

191 root = args[0] if args else "."

192 exclude = frozenset(

193 os.fsencode(path)

194 for path in cnf.value_list("Generate-Md5sums::Options::Exclude")

195 )

196

197 try:

198 run(

199 os.fsencode(root), os.fsencode(options["Cache"]), sys.stdout.buffer, exclude

200 )

201 sys.stdout.buffer.flush()

202 except BrokenPipeError:

203 # stdout is gone (gzip died); exit quietly and let the shell's

204 # pipefail report the failure. Redirect stdout to /dev/null so

205 # the interpreter does not complain about the unflushable buffer

206 # at exit.

207 os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno())

208 sys.exit(1)

209 except OSError as e:

210 utils.fubar(str(e))

Coverage for dak/generate_md5sums.py: 85%

102 statements