Package daklib :: Module import_repository
[hide private]
[frames] | no frames]

Source Code for Module daklib.import_repository

  1  # Copyright (C) 2015, Ansgar Burchardt <ansgar@debian.org> 
  2  # 
  3  # This program is free software; you can redistribute it and/or modify 
  4  # it under the terms of the GNU General Public License as published by 
  5  # the Free Software Foundation; either version 2 of the License, or 
  6  # (at your option) any later version. 
  7  # 
  8  # This program is distributed in the hope that it will be useful, 
  9  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 10  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 11  # GNU General Public License for more details. 
 12  # 
 13  # You should have received a copy of the GNU General Public License along 
 14  # with this program; if not, write to the Free Software Foundation, Inc., 
 15  # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 16   
 17  import daklib.compress 
 18  import daklib.config 
 19  import daklib.dakapt 
 20  import daklib.dbconn 
 21  import daklib.gpg 
 22  import daklib.upload 
 23  import daklib.regexes 
 24   
 25  import apt_pkg 
 26  import os 
 27  import shutil 
 28  import tempfile 
 29  import urllib.request 
 30  import urllib.error 
 31  import urllib.parse 
 32   
 33  from daklib.dbconn import DBSource, PoolFile 
 34  from sqlalchemy.orm import object_session 
 35   
 36  # Hmm, maybe use APT directly for all of this? 
 37   
 38  _release_hashes_fields = ('MD5Sum', 'SHA1', 'SHA256') 
 39   
 40   
41 -class Release:
42 - def __init__(self, base, suite_name, data):
43 self._base = base 44 self._suite_name = suite_name 45 self._dict = apt_pkg.TagSection(data) 46 self._hashes = daklib.upload.parse_file_list(self._dict, False, daklib.regexes.re_file_safe_slash, _release_hashes_fields)
47
48 - def architectures(self):
49 return self._dict['Architectures'].split()
50
51 - def components(self):
52 return self._dict['Components'].split()
53
54 - def packages(self, component, architecture):
55 fn = '{0}/binary-{1}/Packages'.format(component, architecture) 56 tmp = obtain_release_file(self, fn) 57 return apt_pkg.TagFile(tmp.fh())
58
59 - def sources(self, component):
60 fn = '{0}/source/Sources'.format(component) 61 tmp = obtain_release_file(self, fn) 62 return apt_pkg.TagFile(tmp.fh())
63
64 - def suite(self):
65 return self._dict['Suite']
66
67 - def codename(self):
68 return self._dict['Codename']
69 # TODO: Handle Date/Valid-Until to make sure we import 70 # a newer version than before 71 72
73 -class File:
74 - def __init__(self):
75 config = daklib.config.Config() 76 self._tmp = tempfile.NamedTemporaryFile(dir=config['Dir::TempPath'])
77
78 - def fh(self):
79 self._tmp.seek(0) 80 return self._tmp
81
82 - def hashes(self):
83 return daklib.dakapt.DakHashes(self.fh())
84 85
86 -def obtain_file(base, path):
87 """Obtain a file 'path' located below 'base' 88 89 Returns: daklib.import_repository.File 90 91 Note: return type can still change 92 """ 93 fn = '{0}/{1}'.format(base, path) 94 tmp = File() 95 if fn.startswith('http://'): 96 fh = urllib.request.urlopen(fn, timeout=300) 97 shutil.copyfileobj(fh, tmp._tmp) 98 fh.close() 99 else: 100 with open(fn, 'rb') as fh: 101 shutil.copyfileobj(fh, tmp._tmp) 102 return tmp
103 104
105 -def obtain_release(base, suite_name, keyring, fingerprint=None):
106 """Obtain release information 107 108 Returns: daklib.import_repository.Release 109 """ 110 tmp = obtain_file(base, 'dists/{0}/InRelease'.format(suite_name)) 111 data = tmp.fh().read() 112 f = daklib.gpg.SignedFile(data, [keyring]) 113 r = Release(base, suite_name, f.contents) 114 if r.suite() != suite_name and r.codename() != suite_name: 115 raise Exception("Suite {0} doesn't match suite or codename from Release file.".format(suite_name)) 116 return r
117 118 119 _compressions = ('.zst', '.xz', '.gz', '.bz2') 120 121
122 -def obtain_release_file(release, filename):
123 """Obtain file referenced from Release 124 125 A compressed version is automatically selected and decompressed if it exists. 126 127 Returns: daklib.import_repository.File 128 """ 129 if filename not in release._hashes: 130 raise ValueError("File {0} not referenced in Release".format(filename)) 131 132 compressed = False 133 for ext in _compressions: 134 compressed_file = filename + ext 135 if compressed_file in release._hashes: 136 compressed = True 137 filename = compressed_file 138 break 139 140 # Obtain file and check hashes 141 tmp = obtain_file(release._base, 'dists/{0}/{1}'.format(release._suite_name, filename)) 142 hashedfile = release._hashes[filename] 143 hashedfile.check_fh(tmp.fh()) 144 145 if compressed: 146 tmp2 = File() 147 daklib.compress.decompress(tmp.fh(), tmp2.fh(), filename) 148 tmp = tmp2 149 150 return tmp
151 152
153 -def import_source_to_archive(base, entry, transaction, archive, component):
154 """Import source package described by 'entry' into the given 'archive' and 'component' 155 156 'entry' needs to be a dict-like object with at least the following 157 keys as used in a Sources index: Directory, Files, Checksums-Sha1, 158 Checksums-Sha256 159 160 Return: daklib.dbconn.DBSource 161 162 """ 163 # Obtain and verify files 164 if not daklib.regexes.re_file_safe_slash.match(entry['Directory']): 165 raise Exception("Unsafe path in Directory field") 166 hashed_files = daklib.upload.parse_file_list(entry, False) 167 files = [] 168 for f in hashed_files.values(): 169 path = os.path.join(entry['Directory'], f.filename) 170 tmp = obtain_file(base, path) 171 f.check_fh(tmp.fh()) 172 files.append(tmp) 173 directory, f.input_filename = os.path.split(tmp.fh().name) 174 175 # Inject files into archive 176 source = daklib.upload.Source(directory, list(hashed_files.values()), [], require_signature=False) 177 # TODO: ugly hack! 178 for f in hashed_files.keys(): 179 if f.endswith('.dsc'): 180 continue 181 source.files[f].input_filename = hashed_files[f].input_filename 182 183 # TODO: allow changed_by to be NULL 184 changed_by = source.dsc['Maintainer'] 185 db_changed_by = daklib.dbconn.get_or_set_maintainer(changed_by, transaction.session) 186 db_source = transaction.install_source_to_archive(directory, source, archive, component, db_changed_by) 187 188 return db_source
189 190
191 -def import_package_to_suite(base, entry, transaction, suite, component):
192 """Import binary package described by 'entry' into the given 'suite' and 'component' 193 194 'entry' needs to be a dict-like object with at least the following 195 keys as used in a Packages index: Filename, Size, MD5sum, SHA1, 196 SHA256 197 198 Returns: daklib.dbconn.DBBinary 199 """ 200 # Obtain and verify file 201 filename = entry['Filename'] 202 tmp = obtain_file(base, filename) 203 directory, fn = os.path.split(tmp.fh().name) 204 hashedfile = daklib.upload.HashedFile(os.path.basename(filename), int(entry['Size']), entry['MD5sum'], entry['SHA1'], entry['SHA256'], input_filename=fn) 205 hashedfile.check_fh(tmp.fh()) 206 207 # Inject file into archive 208 binary = daklib.upload.Binary(directory, hashedfile) 209 db_binary = transaction.install_binary(directory, binary, suite, component) 210 transaction.flush() 211 212 return db_binary
213 214
215 -def import_source_to_suite(base, entry, transaction, suite, component):
216 """Import source package described by 'entry' into the given 'suite' and 'component' 217 218 'entry' needs to be a dict-like object with at least the following 219 keys as used in a Sources index: Directory, Files, Checksums-Sha1, 220 Checksums-Sha256 221 222 Returns: daklib.dbconn.DBBinary 223 """ 224 source = import_source_to_archive(base, entry, transaction, suite.archive, component) 225 source.suites.append(suite) 226 transaction.flush()
227 228
229 -def source_in_archive(source, version, archive, component=None):
230 """Check that source package 'source' with version 'version' exists in 'archive', 231 with an optional check for the given component 'component'. 232 233 @type source: str 234 @type version: str 235 @type archive: daklib.dbconn.Archive 236 @type component: daklib.dbconn.Component or None 237 @rtype: boolean 238 239 Note: This should probably be moved somewhere else 240 """ 241 session = object_session(archive) 242 query = session.query(DBSource).filter_by(source=source, version=version) \ 243 .join(DBSource.poolfile).join(PoolFile.archives).filter_by(archive=archive) 244 if component is not None: 245 query = query.filter_by(component=component) 246 return session.query(query.exists()).scalar()
247