Source Code for Module dak.import_repository

  1  #! /usr/bin/env python3 
  2  # 
  3  # Copyright (C) 2015, Ansgar Burchardt <ansgar@debian.org> 
  4  # 
  5  # This program is free software; you can redistribute it and/or modify 
  6  # it under the terms of the GNU General Public License as published by 
  7  # the Free Software Foundation; either version 2 of the License, or 
  8  # (at your option) any later version. 
  9  # 
 10  # This program is distributed in the hope that it will be useful, 
 11  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 12  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 13  # GNU General Public License for more details. 
 14  # 
 15  # You should have received a copy of the GNU General Public License along 
 16  # with this program; if not, write to the Free Software Foundation, Inc., 
 17  # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 18   
 19  import daklib.archive 
 20  import daklib.config 
 21  import daklib.dbconn 
 22  import daklib.import_repository 
 23  import daklib.utils 
 24   
 25  import apt_pkg 
 26  import sys 
 27   
 28  from collections import defaultdict 
 29   
 30   
 31 -def usage(status=0): 
 32      print(""" 
 33  dak import-repository 
 34    --keyring=/usr/share/keyrings/debian-archive-keyring.gpg 
 35    [--key=${fingerprint}] 
 36    [--architectures=a,b,c (default: architectures in origin suite)] 
 37    [--components=main,contrib (default: components in origin suite)] 
 38    [--target-suite=${suite} (default: origin suite name)] 
 39    [--add-overrides] 
 40    [--max-packages=${n} (import at maximum ${n} packages, default: no limit)] 
 41    http://httpredir.debian.org/debian unstable 
 42   
 43  Things to think about: 
 44   - Import Built-Using sources 
 45     - all / only referenced 
 46   - Remove old packages: 
 47     - by-source: remove source X_v, if no X exists upstream 
 48     - by-version: remove source X_v, if no X_v exists upstream 
 49     (X denotes package name, v version, X_v package at a specific version) 
 50   - Import all or only newest? 
 51   - Expire binary packages? 
 52  """) 
 53      sys.exit(status) 
 54   
 55   
 56 -def entry_is_newer(entry, packages): 
 57      version = entry['Version'] 
 58      for p in packages[entry['Package']]: 
 59          if apt_pkg.version_compare(version, p.version) <= 0: 
 60              return False 
 61      return True 
 62   
 63   
 64 -def entry_in_packages(entry, packages): 
 65      return entry['Package'] in packages 
 66   
 67   
 68 -def get_packages_in_suite(suite): 
 69      sources = defaultdict(list) 
 70      for s in suite.sources: 
 71          sources[s.source].append(s) 
 72   
 73      packages = defaultdict(list) 
 74      for b in suite.binaries: 
 75          packages[b.package].append(b) 
 76   
 77      return sources, packages 
 78   
 79   
 80 -def import_sources(base, sources, transaction, target_suite, component, target_sources, extra_sources, extra_sources_comp, max_packages=None): 
 81      n = 0 
 82      for entry in sources: 
 83          if max_packages is not None and n > max_packages: 
 84              break 
 85          if entry.get('Extra-Source-Only', 'no') == 'yes': 
 86              # Remember package, we might need to import it later. 
 87              key = (entry['Package'], entry['Version']) 
 88              extra_sources[key] = entry 
 89              extra_sources_comp[key].add(c) 
 90              continue 
 91          if not entry_in_packages(entry, target_sources) or entry_is_newer(entry, target_sources): 
 92              print("Importing {0}={1}".format(entry['Package'], entry['Version'])) 
 93              daklib.import_repository.import_source_to_suite(base, entry, transaction, target_suite, component) 
 94              n += 1 
 95      return n 
 96   
 97   
 98 -def import_built_using(base, source, version, transaction, target_suite, component, extra_sources, extra_sources_comp): 
 99      if not daklib.import_repository.source_in_archive(bu_source, bu_version, target_suite.archive): 
100          print("Importing extra source {0}={1}".format(bu_source, bu_version)) 
101          key = (bu_source, bu_version) 
102          extra_entry = extra_sources.get(key) 
103          if extra_entry is None: 
104              raise Exception("Extra source {0}={1} referenced by {2}={3} ({4}) not found in source suite.".format(bu_source, bu_version, entry['Package'], entry['Version'], architecture)) 
105          extra_components = extra_sources_comp[key] 
106          if c in components: 
107              extra_component = component 
108          else: 
109              # TODO: Take preferred components from those listed... 
110              raise Exception("Not implemented.") 
111          daklib.import_repository.import_source_to_suite(base, extra_entry, transaction, target_suite, extra_component) 
112   
113   
114 -def import_packages(base, packages, transaction, target_suite, component, architecture, target_binaries, extra_sources, extra_sources_comp, max_packages=None): 
115      n = 0 
116      for entry in packages: 
117          if max_packages is not None and n > max_packages: 
118              break 
119          if not entry_in_packages(entry, target_binaries) or entry_is_newer(entry, target_binaries): 
120              print("Importing {0}={1} ({2})".format(entry['Package'], entry['Version'], architecture)) 
121              # Import Built-Using sources: 
122              for bu_source, bu_version in daklib.utils.parse_built_using(entry): 
123                  import_built_using(base, bu_source, bu_version, transaction, target_suite, component, extra_sources, extra_sources_comp) 
124              # Import binary: 
125              daklib.import_repository.import_package_to_suite(base, entry, transaction, target_suite, component) 
126              n += 1 
127      return n 
128   
129   
130 -def main(argv=None): 
131      if argv is None: 
132          argv = sys.argv 
133   
134      arguments = [ 
135          ('h', 'help', 'Import-Repository::Help'), 
136          ('k', 'keyring', 'Import-Repository::Keyring', 'HasArg'), 
137          ('K', 'key', 'Import-Repository::Key', 'HasArg'), 
138          ('a', 'architectures', 'Import-Repository::Architectures', 'HasArg'), 
139          ('c', 'components', 'Import-Repository::Components', 'HasArg'), 
140          ('t', 'target-suite', 'Import-Repository::Target-Suite', 'HasArg'), 
141          ('A', 'add-overrides', 'Import-Repository::AddOverrides'), 
142          ('n', 'max-packages', 'Import-Repository::MaxPackages', 'HasArg'), 
143      ] 
144   
145      cnf = daklib.config.Config() 
146      argv = apt_pkg.parse_commandline(cnf.Cnf, arguments, argv) 
147      options = cnf.subtree('Import-Repository') 
148   
149      if 'Help' in options or len(argv) < 2: 
150          usage(0) 
151   
152      keyring = options.find('Keyring') or None 
153      if keyring is None: 
154          print("Error: No keyring specified") 
155          print() 
156   
157      if 'Key' in options: 
158          raise Exception('Not implemented.') 
159   
160      if 'AddOverrides' in options: 
161          raise Exception('Not implemented.') 
162   
163      if 'MaxPackages' in options: 
164          max_packages = int(options['MaxPackages']) 
165      else: 
166          max_packages = None 
167   
168      base, suite = argv[0:2] 
169   
170      target_suite_name = options.find('Target-Suite') or suite 
171   
172      print("Importing packages from {0}/dists/{1} to {2}".format(base, suite, target_suite_name)) 
173      with daklib.archive.ArchiveTransaction() as transaction: 
174          target_suite = daklib.dbconn.get_suite(target_suite_name, transaction.session) 
175          if target_suite is None: 
176              daklib.utils.fubar("Target suite '{0}' is unknown.".format(target_suite_name)) 
177   
178          release = daklib.import_repository.obtain_release(base, suite, keyring) 
179          target_sources, target_binaries = get_packages_in_suite(target_suite) 
180   
181          if 'Architectures' in options: 
182              architectures = options['Architectures'].split(',') 
183          else: 
184              architectures = ['all'] + release.architectures() 
185   
186          if 'Components' in options: 
187              components = options['Components'].split(',') 
188          else: 
189              components = release.components() 
190   
191          # TODO: Clean this up... 
192   
193          n = 0 
194   
195          # For Extra-Source-Only sources packages, keep a dict 
196          # (name, version) -> entry and (name, version) -> set of components 
197          # to allow importing needed packages at a later stage 
198          extra_sources = dict() 
199          extra_sources_comp = defaultdict(set) 
200   
201          for c in components: 
202              component = daklib.dbconn.get_component(c, transaction.session) 
203              print("Processing {0}/source...".format(c)) 
204              sources = release.sources(c) 
205              imported = import_sources(base, sources, transaction, target_suite, component, target_sources, extra_sources, extra_sources_comp, max_packages) 
206              print("  imported {0} source packages".format(imported)) 
207              n += imported 
208              if max_packages is not None: 
209                  max_packages -= n 
210   
211          for c in components: 
212              component = daklib.dbconn.get_component(c, transaction.session) 
213              for architecture in architectures: 
214                  print("Processing {0}/{1}...".format(c, architecture)) 
215                  packages = release.packages(c, architecture) 
216                  imported = import_packages(base, packages, transaction, target_suite, component, architecture, target_binaries, extra_sources, extra_sources_comp, max_packages) 
217                  print("  imported {0} binary packages".format(imported)) 
218                  n += imported 
219                  if max_packages is not None: 
220                      max_packages -= n 
221   
222          transaction.rollback() 
223   
224   
225  if __name__ == '__main__': 
226      main() 
227