Coverage for dak/clean_suites.py: 72%

1"""Cleans up unassociated binary and source packages

3@contact: Debian FTPMaster <ftpmaster@debian.org>

4@copyright: 2000, 2001, 2002, 2003, 2006 James Troup <james@nocrew.org>

5@copyright: 2009 Mark Hymers <mhy@debian.org>

6@copyright: 2010 Joerg Jaspert <joerg@debian.org>

7@license: GNU General Public License version 2 or later

8"""

10# This program is free software; you can redistribute it and/or modify

11# it under the terms of the GNU General Public License as published by

12# the Free Software Foundation; either version 2 of the License, or

13# (at your option) any later version.

15# This program is distributed in the hope that it will be useful,

16# but WITHOUT ANY WARRANTY; without even the implied warranty of

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

18# GNU General Public License for more details.

20# You should have received a copy of the GNU General Public License

21# along with this program; if not, write to the Free Software

22# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

24################################################################################

26# 07:05|<elmo> well.. *shrug*.. no, probably not.. but to fix it,

27# | we're going to have to implement reference counting

28# | through dependencies.. do we really want to go down

29# | that road?

30#

31# 07:05|<Culus> elmo: Augh! <brain jumps out of skull>

33################################################################################

35import errno

36import os

37import stat

38import sys

39from datetime import datetime

41import apt_pkg

42from sqlalchemy import select, sql

43from sqlalchemy.orm import contains_eager

45from daklib import daklog, utils

46from daklib.config import Config

47from daklib.dbconn import Archive, ArchiveFile, DBConn

49################################################################################

51Options: apt_pkg.Configuration

52Logger: daklog.Logger

54################################################################################

57def usage(exit_code=0):

58 print(

59 """Usage: dak clean-suites [OPTIONS]

60Clean old packages from suites.

62 -n, --no-action don't do anything

63 -h, --help show this help and exit

64 -m, --maximum maximum number of files to remove"""

65 )

66 sys.exit(exit_code)

69################################################################################

72def check_binaries(now_date, session):

73 Logger.log(["Checking for orphaned binary packages..."])

75 # Get the list of binary packages not in a suite and mark them for

76 # deletion.

77 # Check for any binaries which are marked for eventual deletion

78 # but are now used again.

80 query = """

81 WITH usage AS (

82 SELECT

83 af.archive_id AS archive_id,

84 af.file_id AS file_id,

85 af.component_id AS component_id,

86 BOOL_OR(EXISTS (SELECT 1 FROM bin_associations ba

87 JOIN suite s ON ba.suite = s.id

88 WHERE ba.bin = b.id

89 AND s.archive_id = af.archive_id))

90 AS in_use

91 FROM files_archive_map af

92 JOIN binaries b ON af.file_id = b.file

93 GROUP BY af.archive_id, af.file_id, af.component_id

94 )

96 UPDATE files_archive_map af

97 SET last_used = CASE WHEN usage.in_use THEN NULL ELSE :last_used END

98 FROM usage, files f, archive

99 WHERE af.archive_id = usage.archive_id AND af.file_id = usage.file_id AND af.component_id = usage.component_id

100 AND ((af.last_used IS NULL AND NOT usage.in_use) OR (af.last_used IS NOT NULL AND usage.in_use))

101 AND af.file_id = f.id

102 AND af.archive_id = archive.id

103 RETURNING archive.name, f.filename, af.last_used IS NULL"""

104

105 res = session.execute(sql.text(query), {"last_used": now_date})

106 for i in res:

107 op = "set lastused"

108 if i[2]: 108 ↛ 109line 108 didn't jump to line 109 because the condition on line 108 was never true

109 op = "unset lastused"

110 Logger.log([op, i[0], i[1]])

111

112

113########################################

114

115

116def check_sources(now_date, session):

117 Logger.log(["Checking for orphaned source packages..."])

118

119 # Get the list of source packages not in a suite and not used by

120 # any binaries.

121

122 # Check for any sources which are marked for deletion but which

123 # are now used again.

124

125 # TODO: the UPDATE part is the same as in check_binaries. Merge?

126

127 query = """

128 WITH usage AS (

129 SELECT

130 af.archive_id AS archive_id,

131 af.file_id AS file_id,

132 af.component_id AS component_id,

133 BOOL_OR(EXISTS (SELECT 1 FROM src_associations sa

134 JOIN suite s ON sa.suite = s.id

135 WHERE sa.source = df.source

136 AND s.archive_id = af.archive_id)

137 OR EXISTS (SELECT 1 FROM files_archive_map af_bin

138 JOIN binaries b ON af_bin.file_id = b.file

139 WHERE b.source = df.source

140 AND af_bin.archive_id = af.archive_id

141 AND (af_bin.last_used IS NULL OR af_bin.last_used > ad.delete_date))

142 OR EXISTS (SELECT 1 FROM extra_src_references esr

143 JOIN bin_associations ba ON esr.bin_id = ba.bin

144 JOIN binaries b ON ba.bin = b.id

145 JOIN suite s ON ba.suite = s.id

146 WHERE esr.src_id = df.source

147 AND s.archive_id = af.archive_id))

148 AS in_use

149 FROM files_archive_map af

150 JOIN dsc_files df ON af.file_id = df.file

151 JOIN archive_delete_date ad ON af.archive_id = ad.archive_id

152 GROUP BY af.archive_id, af.file_id, af.component_id

153 )

154

155 UPDATE files_archive_map af

156 SET last_used = CASE WHEN usage.in_use THEN NULL ELSE :last_used END

157 FROM usage, files f, archive

158 WHERE af.archive_id = usage.archive_id AND af.file_id = usage.file_id AND af.component_id = usage.component_id

159 AND ((af.last_used IS NULL AND NOT usage.in_use) OR (af.last_used IS NOT NULL AND usage.in_use))

160 AND af.file_id = f.id

161 AND af.archive_id = archive.id

162

163 RETURNING archive.name, f.filename, af.last_used IS NULL

164 """

165

166 res = session.execute(sql.text(query), {"last_used": now_date})

167 for i in res:

168 op = "set lastused"

169 if i[2]: 169 ↛ 170line 169 didn't jump to line 170 because the condition on line 169 was never true

170 op = "unset lastused"

171 Logger.log([op, i[0], i[1]])

172

173

174########################################

175

176

177def check_files(now_date, session):

178 # FIXME: this is evil; nothing should ever be in this state. if

179 # they are, it's a bug.

180

181 # However, we've discovered it happens sometimes so we print a huge warning

182 # and then mark the file for deletion. This probably masks a bug somwhere

183 # else but is better than collecting cruft forever

184

185 Logger.log(["Checking for unused files..."])

186 q = session.execute(

187 sql.text(

188 """

189 UPDATE files_archive_map af

190 SET last_used = :last_used

191 FROM files f, archive

192 WHERE af.file_id = f.id

193 AND af.archive_id = archive.id

194 AND NOT EXISTS (SELECT 1 FROM binaries b WHERE b.file = af.file_id)

195 AND NOT EXISTS (SELECT 1 FROM dsc_files df WHERE df.file = af.file_id)

196 AND af.last_used IS NULL

197 RETURNING archive.name, f.filename"""

198 ),

199 {"last_used": now_date},

200 )

201

202 for x in q: 202 ↛ 203line 202 didn't jump to line 203 because the loop on line 202 never started

203 utils.warn("orphaned file: {0}".format(x))

204 Logger.log(["set lastused", x[0], x[1], "ORPHANED FILE"])

205

206 if not Options["No-Action"]: 206 ↛ exitline 206 didn't return from function 'check_files' because the condition on line 206 was always true

207 session.commit()

208

209

210def clean_binaries(now_date, session):

211 # We do this here so that the binaries we remove will have their

212 # source also removed (if possible).

213

214 # XXX: why doesn't this remove the files here as well? I don't think it

215 # buys anything keeping this separate

216

217 Logger.log(["Deleting from binaries table... "])

218 q = session.execute(

219 sql.text(

220 """

221 DELETE FROM binaries b

222 USING files f

223 WHERE f.id = b.file

224 AND NOT EXISTS (SELECT 1 FROM files_archive_map af

225 JOIN archive_delete_date ad ON af.archive_id = ad.archive_id

226 WHERE af.file_id = b.file

227 AND (af.last_used IS NULL OR af.last_used > ad.delete_date))

228 RETURNING f.filename

229 """

230 )

231 )

232 for b in q:

233 Logger.log(["delete binary", b[0]])

234

235

236########################################

237

238

239def clean(now_date, archives, max_delete, session):

240 cnf = Config()

241

242 count = 0

243 size = 0

244

245 Logger.log(["Cleaning out packages..."])

246

247 morguedir = cnf.get("Dir::Morgue", os.path.join("Dir::Pool", "morgue"))

248 morguesubdir = cnf.get("Clean-Suites::MorgueSubDir", "pool")

249

250 # Build directory as morguedir/morguesubdir/year/month/day

251 dest = os.path.join(

252 morguedir,

253 morguesubdir,

254 str(now_date.year),

255 "%.2d" % now_date.month,

256 "%.2d" % now_date.day,

257 )

258

259 if not Options["No-Action"] and not os.path.exists(dest):

260 os.makedirs(dest)

261

262 # Delete from source

263 Logger.log(["Deleting from source table..."])

264 q = session.execute(

265 sql.text(

266 """

267 WITH

268 deleted_sources AS (

269 DELETE FROM source

270 USING files f

271 WHERE source.file = f.id

272 AND NOT EXISTS (SELECT 1 FROM files_archive_map af

273 JOIN archive_delete_date ad ON af.archive_id = ad.archive_id

274 WHERE af.file_id = source.file

275 AND (af.last_used IS NULL OR af.last_used > ad.delete_date))

276 RETURNING source.id AS id, f.filename AS filename

277 ),

278 deleted_dsc_files AS (

279 DELETE FROM dsc_files df WHERE df.source IN (SELECT id FROM deleted_sources)

280 RETURNING df.file AS file_id

281 ),

282 now_unused_source_files AS (

283 UPDATE files_archive_map af

284 SET last_used = '1977-03-13 13:37:42' -- Kill it now. We waited long enough before removing the .dsc.

285 WHERE af.file_id IN (SELECT file_id FROM deleted_dsc_files)

286 AND NOT EXISTS (SELECT 1 FROM dsc_files df WHERE df.file = af.file_id)

287 )

288 SELECT filename FROM deleted_sources"""

289 )

290 )

291 for s in q: 291 ↛ 292line 291 didn't jump to line 292 because the loop on line 291 never started

292 Logger.log(["delete source", s[0]])

293

294 if not Options["No-Action"]: 294 ↛ 302line 294 didn't jump to line 302

295 session.commit()

296

297 # Delete files from the pool

298 # Lock the mapping rows we are about to process. Otherwise two

299 # clean-suites runs can select the same rows; one run can then delete the

300 # files and files_archive_map rows while the other still has stale ORM

301 # objects whose lazy-loaded relationships no longer exist.

302 old_files = (

303 select(ArchiveFile)

304 .join(ArchiveFile.archive)

305 .join(ArchiveFile.component)

306 .join(ArchiveFile.file)

307 .options(

308 contains_eager(ArchiveFile.archive),

309 contains_eager(ArchiveFile.component),

310 contains_eager(ArchiveFile.file),

311 )

312 .where(

313 sql.text(

314 "files_archive_map.last_used <= (SELECT delete_date FROM archive_delete_date ad WHERE ad.archive_id = files_archive_map.archive_id)"

315 )

316 )

317 )

318

319 if archives is not None:

320 archive_ids = [a.archive_id for a in archives]

321 old_files = old_files.where(ArchiveFile.archive_id.in_(archive_ids))

322

323 if max_delete is not None: 323 ↛ 324line 323 didn't jump to line 324 because the condition on line 323 was never true

324 old_files = old_files.limit(max_delete)

325 Logger.log(["Limiting removals to %d" % max_delete])

326

327 old_files = old_files.with_for_update(of=ArchiveFile, skip_locked=True)

328

329 for af in session.scalars(old_files):

330 filename = af.path

331 try:

332 st = os.lstat(filename)

333 except FileNotFoundError:

334 Logger.log(["database referred to non-existing file", filename])

335 session.delete(af)

336 continue

337 Logger.log(["delete archive file", filename])

338 if stat.S_ISLNK(st.st_mode): 338 ↛ 339line 338 didn't jump to line 339 because the condition on line 338 was never true

339 count += 1

340 Logger.log(["delete symlink", filename])

341 if not Options["No-Action"]:

342 os.unlink(filename)

343 session.delete(af)

344 elif stat.S_ISREG(st.st_mode): 344 ↛ 363line 344 didn't jump to line 363 because the condition on line 344 was always true

345 size += st.st_size

346 count += 1

347

348 dest_filename = dest + "/" + os.path.basename(filename)

349 # If the destination file exists; try to find another filename to use

350 if os.path.lexists(dest_filename): 350 ↛ 351line 350 didn't jump to line 351 because the condition on line 350 was never true

351 dest_filename = utils.find_next_free(dest_filename)

352

353 if not Options["No-Action"]: 353 ↛ 329line 353 didn't jump to line 329 because the condition on line 353 was always true

354 if af.archive.use_morgue: 354 ↛ 355line 354 didn't jump to line 355 because the condition on line 354 was never true

355 Logger.log(["move to morgue", filename, dest_filename])

356 utils.move(filename, dest_filename)

357 else:

358 Logger.log(["removed file", filename])

359 os.unlink(filename)

360 session.delete(af)

361

362 else:

363 utils.fubar("%s is neither symlink nor file?!" % (filename))

364

365 session.flush()

366

367 if count > 0: 367 ↛ 371line 367 didn't jump to line 371 because the condition on line 367 was always true

368 Logger.log(["total", count, utils.size_type(size)])

369

370 # Delete entries in files no longer referenced by any archive

371 query = """

372 DELETE FROM files f

373 WHERE NOT EXISTS (SELECT 1 FROM files_archive_map af WHERE af.file_id = f.id)

374 """

375 session.execute(sql.text(query))

376

377 if not Options["No-Action"]: 377 ↛ exitline 377 didn't return from function 'clean' because the condition on line 377 was always true

378 session.commit()

379

380

381################################################################################

382

383

384def clean_maintainers(now_date, session):

385 Logger.log(["Cleaning out unused Maintainer entries..."])

386

387 # TODO Replace this whole thing with one SQL statement

388 q = session.execute(

389 sql.text(

390 """

391SELECT m.id, m.name FROM maintainer m

392 WHERE NOT EXISTS (SELECT 1 FROM binaries b WHERE b.maintainer = m.id)

393 AND NOT EXISTS (SELECT 1 FROM source s WHERE s.maintainer = m.id OR s.changedby = m.id)

394 AND NOT EXISTS (SELECT 1 FROM src_uploaders u WHERE u.maintainer = m.id)"""

395 )

396 )

397

398 count = 0

399

400 for i in q.fetchall(): 400 ↛ 401line 400 didn't jump to line 401 because the loop on line 400 never started

401 maintainer_id = i[0]

402 Logger.log(["delete maintainer", i[1]])

403 if not Options["No-Action"]:

404 session.execute(

405 sql.text("DELETE FROM maintainer WHERE id = :maint"),

406 {"maint": maintainer_id},

407 )

408 count += 1

409

410 if not Options["No-Action"]: 410 ↛ 413line 410 didn't jump to line 413 because the condition on line 410 was always true

411 session.commit()

412

413 if count > 0: 413 ↛ 414line 413 didn't jump to line 414 because the condition on line 413 was never true

414 Logger.log(["total", count])

415

416

417################################################################################

418

419

420def clean_fingerprints(now_date, session):

421 Logger.log(["Cleaning out unused fingerprint entries..."])

422

423 # TODO Replace this whole thing with one SQL statement

424 q = session.execute(

425 sql.text(

426 """

427SELECT f.id, f.fingerprint FROM fingerprint f

428 WHERE f.keyring IS NULL

429 AND NOT EXISTS (SELECT 1 FROM binaries b WHERE b.sig_fpr = f.id)

430 AND NOT EXISTS (SELECT 1 FROM source s WHERE s.sig_fpr = f.id OR s.authorized_by_fingerprint_id = f.id)

431 AND NOT EXISTS (SELECT 1 FROM acl_per_source aps WHERE aps.created_by_id = f.id)"""

432 )

433 )

434

435 count = 0

436

437 for i in q.fetchall(): 437 ↛ 438line 437 didn't jump to line 438 because the loop on line 437 never started

438 fingerprint_id = i[0]

439 Logger.log(["delete fingerprint", i[1]])

440 if not Options["No-Action"]:

441 session.execute(

442 sql.text("DELETE FROM fingerprint WHERE id = :fpr"),

443 {"fpr": fingerprint_id},

444 )

445 count += 1

446

447 if not Options["No-Action"]: 447 ↛ 450line 447 didn't jump to line 450 because the condition on line 447 was always true

448 session.commit()

449

450 if count > 0: 450 ↛ 451line 450 didn't jump to line 451 because the condition on line 450 was never true

451 Logger.log(["total", count])

452

453

454################################################################################

455

456

457def clean_byhash(now_date, session):

458 Logger.log(["Cleaning out unused by-hash files..."])

459

460 q = session.execute(

461 sql.text(

462 """

463 DELETE FROM hashfile h

464 USING suite s, archive a

465 WHERE s.id = h.suite_id

466 AND a.id = s.archive_id

467 AND h.unreferenced + a.stayofexecution < CURRENT_TIMESTAMP

468 RETURNING a.path, s.suite_name, h.path"""

469 )

470 )

471 count = q.rowcount

472

473 if not Options["No-Action"]: 473 ↛ 487line 473 didn't jump to line 487 because the condition on line 473 was always true

474 for base, suite, path in q: 474 ↛ 475line 474 didn't jump to line 475 because the loop on line 474 never started

475 suite_suffix = utils.suite_suffix(suite)

476 filename = os.path.join(base, "dists", suite, suite_suffix, path)

477 try:

478 os.unlink(filename)

479 except OSError as exc:

480 if exc.errno != errno.ENOENT:

481 raise

482 Logger.log(["database referred to non-existing file", filename])

483 else:

484 Logger.log(["delete hashfile", suite, path])

485 session.commit()

486

487 if count > 0: 487 ↛ 488line 487 didn't jump to line 488 because the condition on line 487 was never true

488 Logger.log(["total", count])

489

490

491################################################################################

492

493

494def clean_empty_directories(session):

495 """

496 Removes empty directories from pool directories.

497 """

498

499 Logger.log(["Cleaning out empty directories..."])

500

501 count = 0

502

503 cursor = session.execute(sql.text("""SELECT DISTINCT(path) FROM archive"""))

504 bases = [x[0] for x in cursor.fetchall()]

505

506 for base in bases:

507 for dirpath, dirnames, filenames in os.walk(base, topdown=False):

508 if not filenames and not dirnames:

509 to_remove = os.path.join(base, dirpath)

510 if not Options["No-Action"]: 510 ↛ 513line 510 didn't jump to line 513 because the condition on line 510 was always true

511 Logger.log(["removing directory", to_remove])

512 os.removedirs(to_remove)

513 count += 1

514

515 if count: 515 ↛ exitline 515 didn't return from function 'clean_empty_directories' because the condition on line 515 was always true

516 Logger.log(["total removed directories", count])

517

518

519################################################################################

520

521

522def set_archive_delete_dates(now_date, session):

523 session.execute(

524 sql.text(

525 """

526 CREATE TEMPORARY TABLE archive_delete_date (

527 archive_id INT NOT NULL,

528 delete_date TIMESTAMP NOT NULL

529 )"""

530 )

531 )

532

533 session.execute(

534 sql.text(

535 """

536 INSERT INTO archive_delete_date

537 (archive_id, delete_date)

538 SELECT

539 archive.id, :now_date - archive.stayofexecution

540 FROM archive"""

541 ),

542 {"now_date": now_date},

543 )

544

545 session.flush()

546

547

548################################################################################

549

550

551def main():

552 global Options, Logger

553

554 cnf = Config()

555

556 for i in ["Help", "No-Action", "Maximum"]:

557 key = "Clean-Suites::Options::%s" % i

558 if key not in cnf: 558 ↛ 556line 558 didn't jump to line 556 because the condition on line 558 was always true

559 cnf[key] = ""

560

561 Arguments = [

562 ("h", "help", "Clean-Suites::Options::Help"),

563 ("a", "archive", "Clean-Suites::Options::Archive", "HasArg"),

564 ("n", "no-action", "Clean-Suites::Options::No-Action"),

565 ("m", "maximum", "Clean-Suites::Options::Maximum", "HasArg"),

566 ]

567

568 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) # type: ignore[attr-defined]

569 Options = cnf.subtree("Clean-Suites::Options")

570

571 if cnf["Clean-Suites::Options::Maximum"] != "": 571 ↛ 572line 571 didn't jump to line 572 because the condition on line 571 was never true

572 try:

573 # Only use Maximum if it's an integer

574 max_delete = int(cnf["Clean-Suites::Options::Maximum"])

575 if max_delete < 1:

576 utils.fubar("If given, Maximum must be at least 1")

577 except ValueError:

578 utils.fubar("If given, Maximum must be an integer")

579 else:

580 max_delete = None

581

582 if Options["Help"]:

583 usage()

584

585 program = "clean-suites"

586 if Options["No-Action"]: 586 ↛ 587line 586 didn't jump to line 587 because the condition on line 586 was never true

587 program = "clean-suites (no action)"

588 Logger = daklog.Logger(program, debug=Options["No-Action"])

589

590 session = DBConn().session()

591

592 archives = None

593 if "Archive" in Options:

594 archive_names = Options["Archive"].split(",")

595 archives = list(

596 session.scalars(

597 select(Archive).where(Archive.archive_name.in_(archive_names))

598 )

599 )

600 if len(archives) == 0: 600 ↛ 601line 600 didn't jump to line 601 because the condition on line 600 was never true

601 utils.fubar("Unknown archive.")

602

603 now_date = datetime.now()

604

605 set_archive_delete_dates(now_date, session)

606

607 check_binaries(now_date, session)

608 clean_binaries(now_date, session)

609 check_sources(now_date, session)

610 check_files(now_date, session)

611 clean(now_date, archives, max_delete, session)

612 clean_maintainers(now_date, session)

613 clean_fingerprints(now_date, session)

614 clean_byhash(now_date, session)

615 clean_empty_directories(session)

616

617 session.rollback()

618

619 Logger.close()