1#! /usr/bin/env python3
3"""Manipulate suite tags"""
4# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
6# This program is free software; you can redistribute it and/or modify
7# it under the terms of the GNU General Public License as published by
8# the Free Software Foundation; either version 2 of the License, or
9# (at your option) any later version.
11# This program is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14# GNU General Public License for more details.
16# You should have received a copy of the GNU General Public License
17# along with this program; if not, write to the Free Software
18# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20#######################################################################################
22# 8to6Guy: "Wow, Bob, You look rough!"
23# BTAF: "Mbblpmn..."
24# BTAF <.oO>: "You moron! This is what you get for staying up all night drinking vodka and salad dressing!"
25# BTAF <.oO>: "This coffee I.V. drip is barely even keeping me awake! I need something with more kick! But what?"
26# BTAF: "OMIGOD! I OVERDOSED ON HEROIN"
27# CoWorker#n: "Give him air!!"
28# CoWorker#n+1: "We need a syringe full of adrenaline!"
29# CoWorker#n+2: "Stab him in the heart!"
30# BTAF: "*YES!*"
31# CoWorker#n+3: "Bob's been overdosing quite a bit lately..."
32# CoWorker#n+4: "Third time this week."
34# -- http://www.angryflower.com/8to6.gif
36#######################################################################################
38# Adds or removes packages from a suite. Takes the list of files
39# either from stdin or as a command line argument. Special action
40# "set", will reset the suite (!) and add all packages from scratch.
42#######################################################################################
44import functools
45import os
46import sys
48import apt_pkg
50from daklib import daklog, utils
51from daklib.archive import ArchiveTransaction
52from daklib.config import Config
53from daklib.dbconn import (
54 Architecture,
55 DBBinary,
56 DBConn,
57 DBSource,
58 Suite,
59 get_suite,
60 get_version_checks,
61)
62from daklib.queue import get_suite_version_by_package, get_suite_version_by_source
64#######################################################################################
66Logger = None
68################################################################################
71def usage(exit_code=0):
72 print(
73 """Usage: dak control-suite [OPTIONS] [FILE]
74Display or alter the contents of a suite using FILE(s), or stdin.
76 -a, --add=SUITE add to SUITE
77 -h, --help show this help and exit
78 -l, --list=SUITE list the contents of SUITE
79 -r, --remove=SUITE remove from SUITE
80 -s, --set=SUITE set SUITE
81 -b, --britney generate changelog entry for britney runs"""
82 )
84 sys.exit(exit_code)
87#######################################################################################
90def get_pkg(package, version, architecture, session):
91 if architecture == "source":
92 q = (
93 session.query(DBSource)
94 .filter_by(source=package, version=version)
95 .join(DBSource.poolfile)
96 )
97 else:
98 q = (
99 session.query(DBBinary)
100 .filter_by(package=package, version=version)
101 .join(DBBinary.architecture)
102 .filter(Architecture.arch_string.in_([architecture, "all"]))
103 .join(DBBinary.poolfile)
104 )
106 pkg = q.first()
107 if pkg is None: 107 ↛ 108line 107 didn't jump to line 108, because the condition on line 107 was never true
108 utils.warn("Could not find {0}_{1}_{2}.".format(package, version, architecture))
109 return pkg
112#######################################################################################
115def britney_changelog(packages, suite, session):
117 old = {}
118 current = {}
119 Cnf = utils.get_conf()
121 try:
122 q = session.execute(
123 "SELECT changelog FROM suite WHERE id = :suiteid",
124 {"suiteid": suite.suite_id},
125 )
126 brit_file = q.fetchone()[0]
127 except:
128 brit_file = None
130 if brit_file: 130 ↛ 133line 130 didn't jump to line 133, because the condition on line 130 was never false
131 brit_file = os.path.join(Cnf["Dir::Root"], brit_file)
132 else:
133 return
135 q = session.execute(
136 """SELECT s.source, s.version, sa.id
137 FROM source s, src_associations sa
138 WHERE sa.suite = :suiteid
139 AND sa.source = s.id""",
140 {"suiteid": suite.suite_id},
141 )
143 for p in q.fetchall():
144 current[p[0]] = p[1]
145 for p in packages.keys(): 145 ↛ 146line 145 didn't jump to line 146, because the loop on line 145 never started
146 if p[2] == "source":
147 old[p[0]] = p[1]
149 new = {}
150 for p in current.keys():
151 if p in old: 151 ↛ 152line 151 didn't jump to line 152, because the condition on line 151 was never true
152 if apt_pkg.version_compare(current[p], old[p]) > 0:
153 new[p] = [current[p], old[p]]
154 else:
155 new[p] = [current[p], None]
157 params = {}
158 query = "SELECT source, changelog FROM changelogs WHERE"
159 for n, p in enumerate(new.keys()):
160 query += f" source = :source_{n} AND (:version1_{n} IS NULL OR version > :version1_{n}) AND version <= :version2_{n}"
161 query += " AND architecture LIKE '%source%' AND distribution in \
162 ('unstable', 'experimental', 'testing-proposed-updates') OR"
163 params[f"source_{n}"] = p
164 params[f"version1_{n}"] = new[p][1]
165 params[f"version2_{n}"] = new[p][0]
166 query += " False ORDER BY source, version DESC"
167 q = session.execute(query, params)
169 pu = None
170 with open(brit_file, "w") as brit:
172 for u in q:
173 if pu and pu != u[0]: 173 ↛ 174line 173 didn't jump to line 174, because the condition on line 173 was never true
174 brit.write("\n")
175 brit.write("%s\n" % u[1])
176 pu = u[0]
177 if q.rowcount: 177 ↛ 180line 177 didn't jump to line 180, because the condition on line 177 was never false
178 brit.write("\n\n\n")
180 for p in list(set(old.keys()).difference(current.keys())): 180 ↛ 181line 180 didn't jump to line 181, because the loop on line 180 never started
181 brit.write("REMOVED: %s %s\n" % (p, old[p]))
183 brit.flush()
186#######################################################################################
189class VersionCheck:
190 def __init__(self, target_suite: str, force: bool, session):
191 self.target_suite = target_suite
192 self.force = force
193 self.session = session
195 self.must_be_newer_than = [
196 vc.reference.suite_name
197 for vc in get_version_checks(target_suite, "MustBeNewerThan", session)
198 ]
199 self.must_be_older_than = [
200 vc.reference.suite_name
201 for vc in get_version_checks(target_suite, "MustBeOlderThan", session)
202 ]
204 # Must be newer than an existing version in target_suite
205 if target_suite not in self.must_be_newer_than: 205 ↛ exitline 205 didn't return from function '__init__', because the condition on line 205 was never false
206 self.must_be_newer_than.append(target_suite)
208 def __call__(self, package: str, architecture: str, new_version: str):
209 if architecture == "source":
210 suite_version_list = get_suite_version_by_source(package, self.session)
211 else:
212 suite_version_list = get_suite_version_by_package(
213 package, architecture, self.session
214 )
216 violations = False
218 for suite, version in suite_version_list:
219 cmp = apt_pkg.version_compare(new_version, version)
220 # for control-suite we allow equal version (for uploads, we don't)
221 if suite in self.must_be_newer_than and cmp < 0:
222 utils.warn(
223 "%s (%s): version check violated: %s targeted at %s is *not* newer than %s in %s"
224 % (
225 package,
226 architecture,
227 new_version,
228 self.target_suite,
229 version,
230 suite,
231 )
232 )
233 violations = True
234 if suite in self.must_be_older_than and cmp > 0: 234 ↛ 235line 234 didn't jump to line 235, because the condition on line 234 was never true
235 utils.warn(
236 "%s (%s): version check violated: %s targeted at %s is *not* older than %s in %s"
237 % (
238 package,
239 architecture,
240 new_version,
241 self.target_suite,
242 version,
243 suite,
244 )
245 )
246 violations = True
248 if violations:
249 if self.force:
250 utils.warn("Continuing anyway (forced)...")
251 else:
252 utils.fubar("Aborting. Version checks violated and not forced.")
255#######################################################################################
258def cmp_package_version(a, b):
259 """
260 comparison function for tuples of the form (package-name, version, arch, ...)
261 """
262 res = 0
263 if a[2] == "source" and b[2] != "source":
264 res = -1
265 elif a[2] != "source" and b[2] == "source":
266 res = 1
267 if res == 0:
268 res = (a[0] > b[0]) - (a[0] < b[0])
269 if res == 0:
270 res = apt_pkg.version_compare(a[1], b[1])
271 return res
274#######################################################################################
277def copy_to_suites(transaction, pkg, suites):
278 component = pkg.poolfile.component
279 if pkg.arch_string == "source":
280 for s in suites:
281 transaction.copy_source(pkg, s, component)
282 else:
283 for s in suites:
284 transaction.copy_binary(pkg, s, component)
287def check_propups(pkg, psuites_current, propups):
288 key = (pkg.name, pkg.arch_string)
289 for suite_id in psuites_current:
290 if key in psuites_current[suite_id]:
291 old_version = psuites_current[suite_id][key]
292 if apt_pkg.version_compare(pkg.version, old_version) > 0:
293 propups[suite_id].add(pkg)
294 if pkg.arch_string != "source":
295 source = pkg.source
296 propups[suite_id].add(source)
299def get_propup_suites(suite, session):
300 propup_suites = []
301 for rule in Config().value_list("SuiteMappings"):
302 fields = rule.split()
303 if fields[0] == "propup-version" and fields[1] == suite.suite_name:
304 propup_suites.append(
305 session.query(Suite).filter_by(suite_name=fields[2]).one()
306 )
307 return propup_suites
310def set_suite(file, suite, transaction, britney=False, force=False):
311 session = transaction.session
312 suite_id = suite.suite_id
313 lines = file.readlines()
314 suites = [suite] + [q.suite for q in suite.copy_queues]
315 propup_suites = get_propup_suites(suite, session)
317 # Our session is already in a transaction
319 def get_binary_q(suite_id):
320 return session.execute(
321 """SELECT b.package, b.version, a.arch_string, ba.id
322 FROM binaries b, bin_associations ba, architecture a
323 WHERE ba.suite = :suiteid
324 AND ba.bin = b.id AND b.architecture = a.id
325 ORDER BY b.version ASC""",
326 {"suiteid": suite_id},
327 )
329 def get_source_q(suite_id):
330 return session.execute(
331 """SELECT s.source, s.version, 'source', sa.id
332 FROM source s, src_associations sa
333 WHERE sa.suite = :suiteid
334 AND sa.source = s.id
335 ORDER BY s.version ASC""",
336 {"suiteid": suite_id},
337 )
339 # Build up a dictionary of what is currently in the suite
340 current = {}
342 q = get_binary_q(suite_id)
343 for i in q:
344 key = i[:3]
345 current[key] = i[3]
347 q = get_source_q(suite_id)
348 for i in q:
349 key = i[:3]
350 current[key] = i[3]
352 # Build a dictionary of what's currently in the propup suites
353 psuites_current = {}
354 propups_needed = {}
355 for p_s in propup_suites:
356 propups_needed[p_s.suite_id] = set()
357 psuites_current[p_s.suite_id] = {}
358 q = get_binary_q(p_s.suite_id)
359 for i in q:
360 key = (i[0], i[2])
361 # the query is sorted, so we only keep the newest version
362 psuites_current[p_s.suite_id][key] = i[1]
364 q = get_source_q(p_s.suite_id)
365 for i in q:
366 key = (i[0], i[2])
367 # the query is sorted, so we only keep the newest version
368 psuites_current[p_s.suite_id][key] = i[1]
370 # Build up a dictionary of what should be in the suite
371 desired = set()
372 for line in lines:
373 split_line = line.strip().split()
374 if len(split_line) != 3: 374 ↛ 375line 374 didn't jump to line 375, because the condition on line 374 was never true
375 utils.warn(
376 "'%s' does not break into 'package version architecture'." % (line[:-1])
377 )
378 continue
379 desired.add(tuple(split_line))
381 version_check = VersionCheck(suite.suite_name, force, session)
383 # Check to see which packages need added and add them
384 for key in sorted(desired, key=functools.cmp_to_key(cmp_package_version)):
385 if key not in current:
386 (package, version, architecture) = key
387 version_check(package, architecture, version)
388 pkg = get_pkg(package, version, architecture, session)
389 if pkg is None: 389 ↛ 390line 389 didn't jump to line 390, because the condition on line 389 was never true
390 continue
392 copy_to_suites(transaction, pkg, suites)
393 Logger.log(["added", suite.suite_name, " ".join(key)])
395 check_propups(pkg, psuites_current, propups_needed)
397 # Check to see which packages need removed and remove them
398 for key, pkid in current.items():
399 if key not in desired:
400 (package, version, architecture) = key
401 if architecture == "source":
402 session.execute(
403 """DELETE FROM src_associations WHERE id = :pkid""", {"pkid": pkid}
404 )
405 else:
406 session.execute(
407 """DELETE FROM bin_associations WHERE id = :pkid""", {"pkid": pkid}
408 )
409 Logger.log(["removed", suite.suite_name, " ".join(key), pkid])
411 for p_s in propup_suites:
412 for pkg in propups_needed[p_s.suite_id]:
413 copy_to_suites(transaction, pkg, [p_s])
414 info = (pkg.name, pkg.version, pkg.arch_string)
415 Logger.log(["propup", p_s.suite_name, " ".join(info)])
417 session.commit()
419 if britney:
420 britney_changelog(current, suite, session)
423#######################################################################################
426def process_file(file, suite, action, transaction, britney=False, force=False):
427 session = transaction.session
429 if action == "set":
430 set_suite(file, suite, transaction, britney, force)
431 return
433 suite_id = suite.suite_id
434 suites = [suite] + [q.suite for q in suite.copy_queues]
435 extra_archives = [suite.archive]
437 request = []
439 # Our session is already in a transaction
440 for line in file:
441 split_line = line.strip().split()
442 if len(split_line) != 3: 442 ↛ 443line 442 didn't jump to line 443, because the condition on line 442 was never true
443 utils.warn(
444 "'%s' does not break into 'package version architecture'." % (line[:-1])
445 )
446 continue
447 request.append(split_line)
449 request.sort(key=functools.cmp_to_key(cmp_package_version))
451 version_check = VersionCheck(suite.suite_name, force, session)
453 for package, version, architecture in request:
454 pkg = get_pkg(package, version, architecture, session)
455 if pkg is None: 455 ↛ 456line 455 didn't jump to line 456, because the condition on line 455 was never true
456 continue
457 if architecture == "source":
458 pkid = pkg.source_id
459 else:
460 pkid = pkg.binary_id
462 component = pkg.poolfile.component
464 # Do version checks when adding packages
465 if action == "add":
466 version_check(package, architecture, version)
468 if architecture == "source":
469 # Find the existing association ID, if any
470 q = session.execute(
471 """SELECT id FROM src_associations
472 WHERE suite = :suiteid and source = :pkid""",
473 {"suiteid": suite_id, "pkid": pkid},
474 )
475 ql = q.fetchall()
476 if len(ql) < 1:
477 association_id = None
478 else:
479 association_id = ql[0][0]
481 # Take action
482 if action == "add":
483 if association_id: 483 ↛ 484line 483 didn't jump to line 484, because the condition on line 483 was never true
484 utils.warn(
485 "'%s_%s_%s' already exists in suite %s."
486 % (package, version, architecture, suite.suite_name)
487 )
488 continue
489 else:
490 for s in suites:
491 transaction.copy_source(pkg, s, component)
492 Logger.log(
493 [
494 "added",
495 package,
496 version,
497 architecture,
498 suite.suite_name,
499 pkid,
500 ]
501 )
503 elif action == "remove": 503 ↛ 453line 503 didn't jump to line 453, because the condition on line 503 was never false
504 if association_id is None: 504 ↛ 505line 504 didn't jump to line 505, because the condition on line 504 was never true
505 utils.warn(
506 "'%s_%s_%s' doesn't exist in suite %s."
507 % (package, version, architecture, suite)
508 )
509 continue
510 else:
511 session.execute(
512 """DELETE FROM src_associations WHERE id = :pkid""",
513 {"pkid": association_id},
514 )
515 Logger.log(
516 [
517 "removed",
518 package,
519 version,
520 architecture,
521 suite.suite_name,
522 pkid,
523 ]
524 )
525 else:
526 # Find the existing associations ID, if any
527 q = session.execute(
528 """SELECT id FROM bin_associations
529 WHERE suite = :suiteid and bin = :pkid""",
530 {"suiteid": suite_id, "pkid": pkid},
531 )
532 ql = q.fetchall()
533 if len(ql) < 1:
534 association_id = None
535 else:
536 association_id = ql[0][0]
538 # Take action
539 if action == "add":
540 if association_id: 540 ↛ 541line 540 didn't jump to line 541, because the condition on line 540 was never true
541 utils.warn(
542 "'%s_%s_%s' already exists in suite %s."
543 % (package, version, architecture, suite)
544 )
545 continue
546 else:
547 for s in suites:
548 transaction.copy_binary(
549 pkg, s, component, extra_archives=extra_archives
550 )
551 Logger.log(
552 [
553 "added",
554 package,
555 version,
556 architecture,
557 suite.suite_name,
558 pkid,
559 ]
560 )
561 elif action == "remove": 561 ↛ 453line 561 didn't jump to line 453, because the condition on line 561 was never false
562 if association_id is None: 562 ↛ 563line 562 didn't jump to line 563, because the condition on line 562 was never true
563 utils.warn(
564 "'%s_%s_%s' doesn't exist in suite %s."
565 % (package, version, architecture, suite)
566 )
567 continue
568 else:
569 session.execute(
570 """DELETE FROM bin_associations WHERE id = :pkid""",
571 {"pkid": association_id},
572 )
573 Logger.log(
574 [
575 "removed",
576 package,
577 version,
578 architecture,
579 suite.suite_name,
580 pkid,
581 ]
582 )
584 session.commit()
587#######################################################################################
590def get_list(suite, session):
591 suite_id = suite.suite_id
592 # List binaries
593 q = session.execute(
594 """SELECT b.package, b.version, a.arch_string
595 FROM binaries b, bin_associations ba, architecture a
596 WHERE ba.suite = :suiteid
597 AND ba.bin = b.id AND b.architecture = a.id""",
598 {"suiteid": suite_id},
599 )
600 for i in q.fetchall():
601 print(" ".join(i))
603 # List source
604 q = session.execute(
605 """SELECT s.source, s.version
606 FROM source s, src_associations sa
607 WHERE sa.suite = :suiteid
608 AND sa.source = s.id""",
609 {"suiteid": suite_id},
610 )
611 for i in q.fetchall():
612 print(" ".join(i) + " source")
615#######################################################################################
618def main():
619 global Logger
621 cnf = Config()
623 Arguments = [
624 ("a", "add", "Control-Suite::Options::Add", "HasArg"),
625 ("b", "britney", "Control-Suite::Options::Britney"),
626 ("f", "force", "Control-Suite::Options::Force"),
627 ("h", "help", "Control-Suite::Options::Help"),
628 ("l", "list", "Control-Suite::Options::List", "HasArg"),
629 ("r", "remove", "Control-Suite::Options::Remove", "HasArg"),
630 ("s", "set", "Control-Suite::Options::Set", "HasArg"),
631 ]
633 for i in ["add", "britney", "help", "list", "remove", "set", "version"]:
634 key = "Control-Suite::Options::%s" % i
635 if key not in cnf: 635 ↛ 633line 635 didn't jump to line 633, because the condition on line 635 was never false
636 cnf[key] = ""
638 try:
639 file_list = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
640 except SystemError as e:
641 print("%s\n" % e)
642 usage(1)
643 Options = cnf.subtree("Control-Suite::Options")
645 if Options["Help"]:
646 usage()
648 force = "Force" in Options and Options["Force"]
650 action = None
652 for i in ("add", "list", "remove", "set"):
653 if cnf["Control-Suite::Options::%s" % (i)] != "":
654 suite_name = cnf["Control-Suite::Options::%s" % (i)]
656 if action: 656 ↛ 657line 656 didn't jump to line 657, because the condition on line 656 was never true
657 utils.fubar("Can only perform one action at a time.")
659 action = i
661 # Need an action...
662 if action is None: 662 ↛ 663line 662 didn't jump to line 663, because the condition on line 662 was never true
663 utils.fubar("No action specified.")
665 britney = False
666 if action == "set" and cnf["Control-Suite::Options::Britney"]:
667 britney = True
669 if action == "list":
670 session = DBConn().session()
671 suite = get_suite(suite_name, session)
672 get_list(suite, session)
673 else:
674 Logger = daklog.Logger("control-suite")
676 with ArchiveTransaction() as transaction:
677 session = transaction.session
678 suite = get_suite(suite_name, session)
680 if action == "set" and not suite.allowcsset:
681 if force: 681 ↛ 688line 681 didn't jump to line 688, because the condition on line 681 was never false
682 utils.warn(
683 "Would not normally allow setting suite {0} (allowcsset is FALSE), but --force used".format(
684 suite_name
685 )
686 )
687 else:
688 utils.fubar(
689 "Will not reset suite {0} due to its database configuration (allowcsset is FALSE)".format(
690 suite_name
691 )
692 )
694 if file_list: 694 ↛ 695line 694 didn't jump to line 695, because the condition on line 694 was never true
695 for f in file_list:
696 process_file(open(f), suite, action, transaction, britney, force)
697 else:
698 process_file(sys.stdin, suite, action, transaction, britney, force)
700 Logger.close()
703#######################################################################################
706if __name__ == "__main__":
707 main()