Coverage for dak/stats.py: 12%
284 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-08-03 16:46 +0000
1"""Various statistical pr0nography fun and games"""
3# Copyright (C) 2000, 2001, 2002, 2003, 2006 James Troup <james@nocrew.org>
4# Copyright (C) 2013 Luca Falavigna <dktrkranz@debian.org>
6# This program is free software; you can redistribute it and/or modify
7# it under the terms of the GNU General Public License as published by
8# the Free Software Foundation; either version 2 of the License, or
9# (at your option) any later version.
11# This program is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14# GNU General Public License for more details.
16# You should have received a copy of the GNU General Public License
17# along with this program; if not, write to the Free Software
18# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20################################################################################
22# <aj> can we change the standards instead?
23# <neuro> standards?
24# <aj> whatever we're not conforming to
25# <aj> if there's no written standard, why don't we declare linux as
26# the defacto standard
27# <aj> go us!
29# [aj's attempt to avoid ABI changes for released architecture(s)]
31################################################################################
33import subprocess
34import sys
35import tempfile
36from datetime import datetime
37from email.utils import mktime_tz, parsedate_tz
38from mailbox import mbox
39from os import listdir
40from os.path import isfile, join, splitext
41from re import DOTALL, MULTILINE, findall
42from sys import stderr
43from typing import Any, NoReturn
45import apt_pkg
46from sqlalchemy import select, sql
47from yaml import safe_dump, safe_load
49from daklib import utils
50from daklib.dbconn import Architecture, DBConn, Suite, get_suite_architectures
52################################################################################
54Cnf: apt_pkg.Configuration
56stats: dict[str, Any] = {}
57users: dict[str, str] = {}
58buffer = 0
59FORMAT_SWITCH = "2009-08"
60blacklisted = ("dak", "katie")
62NEW = (
63 r"^(\d{14})\|(?:jennifer|process-unchecked|.*?\|dak)"
64 r"\|(Moving to new|ACCEPT-TO-NEW)"
65)
66new_ACTIONS = r"^(\d{14})\|[^\|]*\|(\S+)\|NEW (\S+)[:\|]"
67old_ACTIONS = (
68 r"(?:lisa|process-new)\|program start\|(.*?)\|" r"(?:lisa|process-new)\|program end"
69)
70old_ACTION = r"^(\d{14})\|(?:lisa|process-new)\|(Accepting changes|rejected)\|"
72################################################################################
75def usage(exit_code=0) -> NoReturn:
76 print(
77 """Usage: dak stats MODE
78Print various stats.
80 -h, --help show this help and exit.
82The following MODEs are available:
84 arch-space - displays space used by each architecture
85 pkg-nums - displays the number of packages by suite/architecture
86 daily-install - displays daily install stats suitable for graphing
87 new - stores stats about the NEW queue
88"""
89 )
90 sys.exit(exit_code)
93################################################################################
96def per_arch_space_use() -> None:
97 session = DBConn().session()
98 q = session.execute(
99 sql.text(
100 """
101SELECT a.arch_string as Architecture, sum(f.size) AS sum
102 FROM files f, binaries b, architecture a
103 WHERE a.id=b.architecture AND f.id=b.file
104 GROUP BY a.arch_string ORDER BY sum"""
105 )
106 ).fetchall()
107 for j in q:
108 print("%-15.15s %s" % (j[0], j[1]))
109 print()
110 q = session.execute(
111 sql.text(
112 "SELECT sum(size) FROM files WHERE filename ~ '.(diff.gz|tar.gz|dsc)$'"
113 )
114 ).fetchall()
115 print("%-15.15s %s" % ("Source", q[0][0]))
118################################################################################
121def daily_install_stats() -> None:
122 stats: dict[str, dict[str, float]] = {}
123 f = open("2001-11")
124 for line in f:
125 split = line.strip().split("|")
126 program = split[1]
127 if program != "katie" and program != "process-accepted":
128 continue
129 action = split[2]
130 if action != "installing changes" and action != "installed":
131 continue
132 date = split[0][:8]
133 if date not in stats:
134 stats[date] = {"packages": 0, "size": 0.0}
135 if action == "installing changes":
136 stats[date]["packages"] += 1
137 elif action == "installed":
138 stats[date]["size"] += float(split[5])
140 dates = sorted(stats)
141 for date in dates:
142 packages = stats[date]["packages"]
143 size = int(stats[date]["size"] / 1024.0 / 1024.0)
144 print("%s %s %s" % (date, packages, size))
147################################################################################
150def output_format(suite: str) -> str:
151 output_suite = []
152 for word in suite.split("-"):
153 output_suite.append(word[0])
154 return "-".join(output_suite)
157def number_of_packages() -> None:
158 arches: dict[int, str] = {}
159 arch_ids: dict[str, int] = {}
160 suites: dict[int, str] = {}
161 suite_ids: dict[str, int] = {}
162 session = DBConn().session()
163 # Build up suite mapping
164 for suite_id, suite_name in session.execute(
165 select(Suite.suite_id, Suite.suite_name)
166 ):
167 suites[suite_id] = suite_name
168 suite_ids[suite_name] = suite_id
169 # Build up architecture mapping
170 for arch_id, arch_string in session.execute(
171 select(Architecture.arch_id, Architecture.arch_string)
172 ):
173 arches[arch_id] = arch_string
174 arch_ids[arch_string] = arch_id
176 # Pre-create the dictionary
177 d: dict[int, dict[int, int]] = {
178 suite_id: {arch_id: 0 for arch_id in arches.keys()}
179 for suite_id in suites.keys()
180 }
182 # Get the raw data for binaries
183 # Simultate 'GROUP by suite, architecture' with a dictionary
184 # XXX: Why don't we just get the DB to do this?
185 for i in session.execute(
186 sql.text(
187 """SELECT suite, architecture, COUNT(suite)
188 FROM bin_associations
189 LEFT JOIN binaries ON bin = binaries.id
190 GROUP BY suite, architecture"""
191 )
192 ).fetchall():
193 d[i[0]][i[1]] = i[2]
194 # Get the raw data for source
195 arch_id = arch_ids["source"]
196 for i in session.execute(
197 sql.text("SELECT suite, COUNT(suite) FROM src_associations GROUP BY suite")
198 ).fetchall():
199 (suite_id, count) = i
200 d[suite_id][arch_id] = d[suite_id][arch_id] + count
201 ## Print the results
202 # Setup
203 suite_list = list(suites.values())
204 suite_id_list = []
205 suite_arches: dict[int, set[str]] = {}
206 for suite in suite_list:
207 suite_id = suite_ids[suite]
208 suite_arches[suite_id] = {a.arch_string for a in get_suite_architectures(suite)}
209 suite_id_list.append(suite_id)
210 output_list = [output_format(i) for i in suite_list]
211 longest_suite = max(len(suite) for suite in output_list)
212 arch_list = sorted(arches.values())
213 longest_arch = max(len(arch) for arch in arch_list)
214 # Header
215 output = (" " * longest_arch) + " |"
216 for suite in output_list:
217 output = output + suite.center(longest_suite) + " |"
218 output = output + "\n" + (len(output) * "-") + "\n"
219 # per-arch data
220 for arch in arch_list:
221 arch_id = arch_ids[arch]
222 output = output + arch.center(longest_arch) + " |"
223 for suite_id in suite_id_list:
224 if arch in suite_arches[suite_id]:
225 count = "%d" % d[suite_id][arch_id]
226 else:
227 count = "-"
228 output = output + count.rjust(longest_suite) + " |"
229 output = output + "\n"
230 print(output)
233################################################################################
236def parse_new_uploads(data: str) -> str:
237 global stats
238 latest_timestamp: str = stats["timestamp"]
239 for entry in findall(NEW, data, MULTILINE):
240 timestamp = entry[0]
241 if stats["timestamp"] >= timestamp:
242 continue
243 date = parse_timestamp(timestamp)
244 if date not in stats:
245 stats[date] = {
246 "stats": {"NEW": 0, "ACCEPT": 0, "REJECT": 0, "PROD": 0},
247 "members": {},
248 }
249 stats[date]["stats"]["NEW"] += 1
250 stats["history"]["stats"]["NEW"] += 1
251 latest_timestamp = timestamp
252 return latest_timestamp
255def parse_actions(data: str, logdate: str) -> str:
256 global stats
257 latest_timestamp: str = stats["timestamp"]
258 if logdate <= FORMAT_SWITCH:
259 for batch in findall(old_ACTIONS, data, DOTALL):
260 who = batch.split()[0]
261 if who in blacklisted:
262 continue
263 for entry in findall(old_ACTION, batch, MULTILINE):
264 action = entry[1]
265 if action.startswith("Accepting"):
266 action = "ACCEPT"
267 elif action.startswith("rejected"):
268 action = "REJECT"
269 timestamp = entry[0]
270 if stats["timestamp"] >= timestamp:
271 continue
272 date = parse_timestamp(entry[0])
273 if date not in stats:
274 stats[date] = {
275 "stats": {"NEW": 0, "ACCEPT": 0, "REJECT": 0, "PROD": 0},
276 "members": {},
277 }
278 stats[date]["stats"][action] += 1
279 stats["history"]["stats"][action] += 1
280 if who not in stats[date]["members"]:
281 stats[date]["members"][who] = {"ACCEPT": 0, "REJECT": 0, "PROD": 0}
282 stats[date]["members"][who][action] += 1
283 if who not in stats["history"]["members"]:
284 stats["history"]["members"][who] = {
285 "ACCEPT": 0,
286 "REJECT": 0,
287 "PROD": 0,
288 }
289 stats["history"]["members"][who][action] += 1
290 latest_timestamp = timestamp
291 parse_prod(logdate)
292 if logdate >= FORMAT_SWITCH:
293 for entry in findall(new_ACTIONS, data, MULTILINE):
294 action = entry[2]
295 timestamp = entry[0]
296 if stats["timestamp"] >= timestamp:
297 continue
298 date = parse_timestamp(timestamp)
299 if date not in stats:
300 stats[date] = {
301 "stats": {"NEW": 0, "ACCEPT": 0, "REJECT": 0, "PROD": 0},
302 "members": {},
303 }
304 member = entry[1]
305 if member in blacklisted:
306 continue
307 if date not in stats:
308 stats[date] = {
309 "stats": {"NEW": 0, "ACCEPT": 0, "REJECT": 0, "PROD": 0},
310 "members": {},
311 }
312 if member not in stats[date]["members"]:
313 stats[date]["members"][member] = {"ACCEPT": 0, "REJECT": 0, "PROD": 0}
314 if member not in stats["history"]["members"]:
315 stats["history"]["members"][member] = {
316 "ACCEPT": 0,
317 "REJECT": 0,
318 "PROD": 0,
319 }
320 stats[date]["stats"][action] += 1
321 stats[date]["members"][member][action] += 1
322 stats["history"]["stats"][action] += 1
323 stats["history"]["members"][member][action] += 1
324 latest_timestamp = timestamp
325 return latest_timestamp
328def parse_prod(logdate: str) -> None:
329 global stats
330 global users
331 maildate = "".join([x[-2:] for x in logdate.split("-")])
332 mailarchive = join(
333 utils.get_conf()["Dir::Base"], "mail/archive", "mail-%s.xz" % maildate
334 )
335 if not isfile(mailarchive):
336 return
337 with tempfile.NamedTemporaryFile(dir=utils.get_conf()["Dir::TempPath"]) as tmpfile:
338 with open(mailarchive, "rb") as fh:
339 subprocess.check_call(["xzcat"], stdin=fh, stdout=tmpfile)
340 for message in mbox(tmpfile.name):
341 if message["subject"] and message["subject"].startswith(
342 "Comments regarding"
343 ):
344 try:
345 member = users[" ".join(message["From"].split()[:-1])]
346 except KeyError:
347 continue
348 message_date = parsedate_tz(message["date"])
349 assert message_date is not None
350 ts = mktime_tz(message_date)
351 timestamp = datetime.fromtimestamp(ts).strftime("%Y%m%d%H%M%S")
352 date = parse_timestamp(timestamp)
353 if date not in stats:
354 stats[date] = {
355 "stats": {"NEW": 0, "ACCEPT": 0, "REJECT": 0, "PROD": 0},
356 "members": {},
357 }
358 if member not in stats[date]["members"]:
359 stats[date]["members"][member] = {
360 "ACCEPT": 0,
361 "REJECT": 0,
362 "PROD": 0,
363 }
364 if member not in stats["history"]["members"]:
365 stats["history"]["members"][member] = {
366 "ACCEPT": 0,
367 "REJECT": 0,
368 "PROD": 0,
369 }
370 stats[date]["stats"]["PROD"] += 1
371 stats[date]["members"][member]["PROD"] += 1
372 stats["history"]["stats"]["PROD"] += 1
373 stats["history"]["members"][member]["PROD"] += 1
376def parse_timestamp(timestamp: str) -> str:
377 y = int(timestamp[:4])
378 m = int(timestamp[4:6])
379 return "%d-%02d" % (y, m)
382def new_stats(logdir: str, yaml: str) -> None:
383 global Cnf
384 global stats
385 try:
386 with open(yaml, "r") as fd:
387 stats = safe_load(fd)
388 except OSError:
389 pass
390 if not stats:
391 stats = {
392 "history": {
393 "stats": {"NEW": 0, "ACCEPT": 0, "REJECT": 0, "PROD": 0},
394 "members": {},
395 },
396 "timestamp": "19700101000000",
397 }
398 latest_timestamp = stats["timestamp"]
399 for fn in sorted(listdir(logdir)):
400 if fn == "current":
401 continue
402 log = splitext(fn)[0]
403 if log < parse_timestamp(stats["timestamp"]):
404 continue
405 logfile = join(logdir, fn)
406 if isfile(logfile):
407 if fn.endswith(".bz2"):
408 # This hack is required becaue python2 does not support
409 # multi-stream files (http://bugs.python.org/issue1625)
410 with open(logfile, "rb") as fh:
411 data = subprocess.check_output(["bzcat"], stdin=fh)
412 elif fn.endswith(".xz"):
413 with open(logfile, "rb") as fh:
414 data = subprocess.check_output(["xzcat"], stdin=fh)
415 elif fn.endswith(".zst"):
416 with open(logfile, "rb") as fh:
417 data = subprocess.check_output(["zstdcat"], stdin=fh)
418 else:
419 with open(logfile, "rb") as fd:
420 data = fd.read()
421 try:
422 data_str = data.decode()
423 except UnicodeDecodeError:
424 data_str = data.decode("latin1")
425 ts = parse_new_uploads(data_str)
426 latest_timestamp = max(latest_timestamp, ts)
427 ts = parse_actions(data_str, log)
428 latest_timestamp = max(latest_timestamp, ts)
429 stderr.write(".")
430 stderr.flush()
431 stderr.write("\n")
432 stderr.flush()
433 stats["timestamp"] = latest_timestamp
434 with open(yaml, "w") as fd:
435 safe_dump(stats, fd)
438################################################################################
441def main() -> None:
442 global Cnf
443 global users
445 Cnf = utils.get_conf()
446 Arguments = [("h", "help", "Stats::Options::Help")]
447 for i in ["help"]:
448 key = "Stats::Options::%s" % i
449 if key not in Cnf: 449 ↛ 447line 449 didn't jump to line 447 because the condition on line 449 was always true
450 Cnf[key] = "" # type: ignore[index]
452 args = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) # type: ignore[attr-defined]
454 Options = Cnf.subtree("Stats::Options") # type: ignore[attr-defined]
455 if Options["Help"]: 455 ↛ 458line 455 didn't jump to line 458 because the condition on line 455 was always true
456 usage()
458 if len(args) < 1:
459 utils.warn("dak stats requires a MODE argument")
460 usage(1)
461 elif len(args) > 1:
462 if args[0].lower() != "new":
463 utils.warn("dak stats accepts only one MODE argument")
464 usage(1)
465 elif args[0].lower() == "new":
466 utils.warn("new MODE requires an output file")
467 usage(1)
468 mode = args[0].lower()
470 if mode == "arch-space":
471 per_arch_space_use()
472 elif mode == "pkg-nums":
473 number_of_packages()
474 elif mode == "daily-install":
475 daily_install_stats()
476 elif mode == "new":
477 users = utils.get_users_from_ldap()
478 new_stats(Cnf["Dir::Log"], args[1])
479 else:
480 utils.warn("unknown mode '%s'" % (mode))
481 usage(1)