Package daklib :: Module upload
[hide private]
[frames] | no frames]

Source Code for Module daklib.upload

  1  # Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org> 
  2  # 
  3  # This program is free software; you can redistribute it and/or modify 
  4  # it under the terms of the GNU General Public License as published by 
  5  # the Free Software Foundation; either version 2 of the License, or 
  6  # (at your option) any later version. 
  7  # 
  8  # This program is distributed in the hope that it will be useful, 
  9  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 10  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 11  # GNU General Public License for more details. 
 12  # 
 13  # You should have received a copy of the GNU General Public License along 
 14  # with this program; if not, write to the Free Software Foundation, Inc., 
 15  # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 16   
 17  """module to handle uploads not yet installed to the archive 
 18   
 19  This module provides classes to handle uploads not yet installed to the 
 20  archive.  Central is the L{Changes} class which represents a changes file. 
 21  It provides methods to access the included binary and source packages. 
 22  """ 
 23   
 24  import apt_inst 
 25  import apt_pkg 
 26  import errno 
 27  import functools 
 28  import os 
 29   
 30  from daklib.aptversion import AptVersion 
 31  from daklib.gpg import SignedFile 
 32  from daklib.regexes import * 
 33  import daklib.dakapt 
 34  import daklib.packagelist 
35 36 37 -class UploadException(Exception):
38 pass
39
40 41 -class InvalidChangesException(UploadException):
42 pass
43
44 45 -class InvalidBinaryException(UploadException):
46 pass
47
48 49 -class InvalidSourceException(UploadException):
50 pass
51
52 53 -class InvalidHashException(UploadException):
54 - def __init__(self, filename, hash_name, expected, actual):
55 self.filename = filename 56 self.hash_name = hash_name 57 self.expected = expected 58 self.actual = actual
59
60 - def __str__(self):
61 return ("Invalid {0} hash for {1}:\n" 62 "According to the control file the {0} hash should be {2},\n" 63 "but {1} has {3}.\n" 64 "\n" 65 "If you did not include {1} in your upload, a different version\n" 66 "might already be known to the archive software.") \ 67 .format(self.hash_name, self.filename, self.expected, self.actual)
68
69 70 -class InvalidFilenameException(UploadException):
71 - def __init__(self, filename):
72 self.filename = filename
73
74 - def __str__(self):
75 return "Invalid filename '{0}'.".format(self.filename)
76
77 78 -class FileDoesNotExist(UploadException):
79 - def __init__(self, filename):
80 self.filename = filename
81
82 - def __str__(self):
83 return "Refers to non-existing file '{0}'".format(self.filename)
84
85 86 -class HashedFile:
87 """file with checksums 88 """ 89
90 - def __init__(self, filename, size, md5sum, sha1sum, sha256sum, section=None, priority=None, input_filename=None):
91 self.filename = filename 92 """name of the file 93 @type: str 94 """ 95 96 if input_filename is None: 97 input_filename = filename 98 self.input_filename = input_filename 99 """name of the file on disk 100 101 Used for temporary files that should not be installed using their on-disk name. 102 @type: str 103 """ 104 105 self.size = size 106 """size in bytes 107 @type: long 108 """ 109 110 self.md5sum = md5sum 111 """MD5 hash in hexdigits 112 @type: str 113 """ 114 115 self.sha1sum = sha1sum 116 """SHA1 hash in hexdigits 117 @type: str 118 """ 119 120 self.sha256sum = sha256sum 121 """SHA256 hash in hexdigits 122 @type: str 123 """ 124 125 self.section = section 126 """section or C{None} 127 @type: str or C{None} 128 """ 129 130 self.priority = priority 131 """priority or C{None} 132 @type: str of C{None} 133 """
134 135 @classmethod
136 - def from_file(cls, directory, filename, section=None, priority=None):
137 """create with values for an existing file 138 139 Create a C{HashedFile} object that refers to an already existing file. 140 141 @type directory: str 142 @param directory: directory the file is located in 143 144 @type filename: str 145 @param filename: filename 146 147 @type section: str or C{None} 148 @param section: optional section as given in .changes files 149 150 @type priority: str or C{None} 151 @param priority: optional priority as given in .changes files 152 153 @rtype: L{HashedFile} 154 @return: C{HashedFile} object for the given file 155 """ 156 path = os.path.join(directory, filename) 157 with open(path, 'r') as fh: 158 size = os.fstat(fh.fileno()).st_size 159 hashes = daklib.dakapt.DakHashes(fh) 160 return cls(filename, size, hashes.md5, hashes.sha1, hashes.sha256, section, priority)
161
162 - def check(self, directory):
163 """Validate hashes 164 165 Check if size and hashes match the expected value. 166 167 @type directory: str 168 @param directory: directory the file is located in 169 170 @raise InvalidHashException: hash mismatch 171 """ 172 path = os.path.join(directory, self.input_filename) 173 try: 174 with open(path) as fh: 175 self.check_fh(fh) 176 except OSError as e: 177 if e.errno == errno.ENOENT: 178 raise FileDoesNotExist(self.input_filename) 179 raise
180
181 - def check_fh(self, fh):
182 size = os.fstat(fh.fileno()).st_size 183 fh.seek(0) 184 hashes = daklib.dakapt.DakHashes(fh) 185 186 if size != self.size: 187 raise InvalidHashException(self.filename, 'size', self.size, size) 188 189 if hashes.md5 != self.md5sum: 190 raise InvalidHashException(self.filename, 'md5sum', self.md5sum, hashes.md5) 191 192 if hashes.sha1 != self.sha1sum: 193 raise InvalidHashException(self.filename, 'sha1sum', self.sha1sum, hashes.sha1) 194 195 if hashes.sha256 != self.sha256sum: 196 raise InvalidHashException(self.filename, 'sha256sum', self.sha256sum, hashes.sha256)
197
198 199 -def parse_file_list(control, has_priority_and_section, safe_file_regexp=re_file_safe, fields=('Files', 'Checksums-Sha1', 'Checksums-Sha256')):
200 """Parse Files and Checksums-* fields 201 202 @type control: dict-like 203 @param control: control file to take fields from 204 205 @type has_priority_and_section: bool 206 @param has_priority_and_section: Files field include section and priority 207 (as in .changes) 208 209 @raise InvalidChangesException: missing fields or other grave errors 210 211 @rtype: dict 212 @return: dict mapping filenames to L{daklib.upload.HashedFile} objects 213 """ 214 entries = {} 215 216 for line in control.get(fields[0], "").split('\n'): 217 if len(line) == 0: 218 continue 219 220 if has_priority_and_section: 221 (md5sum, size, section, priority, filename) = line.split() 222 entry = dict(md5sum=md5sum, size=int(size), section=section, priority=priority, filename=filename) 223 else: 224 (md5sum, size, filename) = line.split() 225 entry = dict(md5sum=md5sum, size=int(size), filename=filename) 226 227 entries[filename] = entry 228 229 for line in control.get(fields[1], "").split('\n'): 230 if len(line) == 0: 231 continue 232 (sha1sum, size, filename) = line.split() 233 entry = entries.get(filename, None) 234 if entry is None: 235 raise InvalidChangesException('{0} is listed in {1}, but not in {2}.'.format(filename, fields[1], fields[0])) 236 if entry is not None and entry.get('size', None) != int(size): 237 raise InvalidChangesException('Size for {0} in {1} and {2} fields differ.'.format(filename, fields[0], fields[1])) 238 entry['sha1sum'] = sha1sum 239 240 for line in control.get(fields[2], "").split('\n'): 241 if len(line) == 0: 242 continue 243 (sha256sum, size, filename) = line.split() 244 entry = entries.get(filename, None) 245 if entry is None: 246 raise InvalidChangesException('{0} is listed in {1}, but not in {2}.'.format(filename, fields[2], fields[0])) 247 if entry is not None and entry.get('size', None) != int(size): 248 raise InvalidChangesException('Size for {0} in {1} and {2} fields differ.'.format(filename, fields[0], fields[2])) 249 entry['sha256sum'] = sha256sum 250 251 files = {} 252 for entry in entries.values(): 253 filename = entry['filename'] 254 if 'size' not in entry: 255 raise InvalidChangesException('No size for {0}.'.format(filename)) 256 if 'md5sum' not in entry: 257 raise InvalidChangesException('No md5sum for {0}.'.format(filename)) 258 if 'sha1sum' not in entry: 259 raise InvalidChangesException('No sha1sum for {0}.'.format(filename)) 260 if 'sha256sum' not in entry: 261 raise InvalidChangesException('No sha256sum for {0}.'.format(filename)) 262 if safe_file_regexp is not None and not safe_file_regexp.match(filename): 263 raise InvalidChangesException("{0}: References file with unsafe filename {1}.".format(self.filename, filename)) 264 files[filename] = HashedFile(**entry) 265 266 return files
267
268 269 @functools.total_ordering 270 -class Changes:
271 """Representation of a .changes file 272 """ 273
274 - def __init__(self, directory, filename, keyrings, require_signature=True):
275 if not re_file_safe.match(filename): 276 raise InvalidChangesException('{0}: unsafe filename'.format(filename)) 277 278 self.directory = directory 279 """directory the .changes is located in 280 @type: str 281 """ 282 283 self.filename = filename 284 """name of the .changes file 285 @type: str 286 """ 287 288 with open(self.path, 'rb') as fd: 289 data = fd.read() 290 self.signature = SignedFile(data, keyrings, require_signature) 291 self.changes = apt_pkg.TagSection(self.signature.contents) 292 """dict to access fields of the .changes file 293 @type: dict-like 294 """ 295 296 self._binaries = None 297 self._source = None 298 self._files = None 299 self._keyrings = keyrings 300 self._require_signature = require_signature
301 302 @property
303 - def path(self):
304 """path to the .changes file 305 @type: str 306 """ 307 return os.path.join(self.directory, self.filename)
308 309 @property
310 - def primary_fingerprint(self):
311 """fingerprint of the key used for signing the .changes file 312 @type: str 313 """ 314 return self.signature.primary_fingerprint
315 316 @property
317 - def valid_signature(self):
318 """C{True} if the .changes has a valid signature 319 @type: bool 320 """ 321 return self.signature.valid
322 323 @property
324 - def weak_signature(self):
325 """C{True} if the .changes was signed using a weak algorithm 326 @type: bool 327 """ 328 return self.signature.weak_signature
329 330 @property
331 - def signature_timestamp(self):
332 return self.signature.signature_timestamp
333 334 @property
335 - def contents_sha1(self):
336 return self.signature.contents_sha1
337 338 @property
339 - def architectures(self):
340 """list of architectures included in the upload 341 @type: list of str 342 """ 343 return self.changes.get('Architecture', '').split()
344 345 @property
346 - def distributions(self):
347 """list of target distributions for the upload 348 @type: list of str 349 """ 350 return self.changes['Distribution'].split()
351 352 @property
353 - def source(self):
354 """included source or C{None} 355 @type: L{daklib.upload.Source} or C{None} 356 """ 357 if self._source is None: 358 source_files = [] 359 for f in self.files.values(): 360 if re_file_dsc.match(f.filename) or re_file_source.match(f.filename): 361 source_files.append(f) 362 if len(source_files) > 0: 363 self._source = Source(self.directory, source_files, self._keyrings, self._require_signature) 364 return self._source
365 366 @property
367 - def sourceful(self):
368 """C{True} if the upload includes source 369 @type: bool 370 """ 371 return "source" in self.architectures
372 373 @property
374 - def source_name(self):
375 """source package name 376 @type: str 377 """ 378 return re_field_source.match(self.changes['Source']).group('package')
379 380 @property
381 - def binaries(self):
382 """included binary packages 383 @type: list of L{daklib.upload.Binary} 384 """ 385 if self._binaries is None: 386 binaries = [] 387 for f in self.files.values(): 388 if re_file_binary.match(f.filename): 389 binaries.append(Binary(self.directory, f)) 390 self._binaries = binaries 391 return self._binaries
392 393 @property
394 - def byhand_files(self):
395 """included byhand files 396 @type: list of L{daklib.upload.HashedFile} 397 """ 398 byhand = [] 399 400 for f in self.files.values(): 401 if f.section == 'byhand' or f.section[:4] == 'raw-': 402 byhand.append(f) 403 continue 404 if re_file_dsc.match(f.filename) or re_file_source.match(f.filename) or re_file_binary.match(f.filename): 405 continue 406 if re_file_buildinfo.match(f.filename): 407 continue 408 409 raise InvalidChangesException("{0}: {1} looks like a byhand package, but is in section {2}".format(self.filename, f.filename, f.section)) 410 411 return byhand
412 413 @property
414 - def buildinfo_files(self):
415 """included buildinfo files 416 @type: list of L{daklib.upload.HashedFile} 417 """ 418 buildinfo = [] 419 420 for f in self.files.values(): 421 if re_file_buildinfo.match(f.filename): 422 buildinfo.append(f) 423 424 return buildinfo
425 426 @property
427 - def binary_names(self):
428 """names of included binary packages 429 @type: list of str 430 """ 431 return self.changes.get('Binary', '').split()
432 433 @property
434 - def closed_bugs(self):
435 """bugs closed by this upload 436 @type: list of str 437 """ 438 return self.changes.get('Closes', '').split()
439 440 @property
441 - def files(self):
442 """dict mapping filenames to L{daklib.upload.HashedFile} objects 443 @type: dict 444 """ 445 if self._files is None: 446 self._files = parse_file_list(self.changes, True) 447 return self._files
448 449 @property
450 - def bytes(self):
451 """total size of files included in this upload in bytes 452 @type: number 453 """ 454 count = 0 455 for f in self.files.values(): 456 count += f.size 457 return count
458
459 - def _key(self):
460 """tuple used to compare two changes files 461 462 We sort by source name and version first. If these are identical, 463 we sort changes that include source before those without source (so 464 that sourceful uploads get processed first), and finally fall back 465 to the filename (this should really never happen). 466 467 @rtype: tuple 468 """ 469 return ( 470 self.changes.get('Source'), 471 AptVersion(self.changes.get('Version', '')), 472 not self.sourceful, 473 self.filename 474 )
475
476 - def __eq__(self, other):
477 return self._key() == other._key()
478
479 - def __lt__(self, other):
480 return self._key() < other._key()
481
482 483 -class Binary:
484 """Representation of a binary package 485 """ 486
487 - def __init__(self, directory, hashed_file):
488 self.hashed_file = hashed_file 489 """file object for the .deb 490 @type: HashedFile 491 """ 492 493 path = os.path.join(directory, hashed_file.input_filename) 494 data = apt_inst.DebFile(path).control.extractdata("control") 495 496 self.control = apt_pkg.TagSection(data) 497 """dict to access fields in DEBIAN/control 498 @type: dict-like 499 """
500 501 @classmethod
502 - def from_file(cls, directory, filename):
503 hashed_file = HashedFile.from_file(directory, filename) 504 return cls(directory, hashed_file)
505 506 @property
507 - def source(self):
508 """get tuple with source package name and version 509 @type: tuple of str 510 """ 511 source = self.control.get("Source", None) 512 if source is None: 513 return (self.control["Package"], self.control["Version"]) 514 match = re_field_source.match(source) 515 if not match: 516 raise InvalidBinaryException('{0}: Invalid Source field.'.format(self.hashed_file.filename)) 517 version = match.group('version') 518 if version is None: 519 version = self.control['Version'] 520 return (match.group('package'), version)
521 522 @property
523 - def name(self):
524 return self.control['Package']
525 526 @property
527 - def type(self):
528 """package type ('deb' or 'udeb') 529 @type: str 530 """ 531 match = re_file_binary.match(self.hashed_file.filename) 532 if not match: 533 raise InvalidBinaryException('{0}: Does not match re_file_binary'.format(self.hashed_file.filename)) 534 return match.group('type')
535 536 @property
537 - def component(self):
538 """component name 539 @type: str 540 """ 541 fields = self.control['Section'].split('/') 542 if len(fields) > 1: 543 return fields[0] 544 return "main"
545
546 547 -class Source:
548 """Representation of a source package 549 """ 550
551 - def __init__(self, directory, hashed_files, keyrings, require_signature=True):
552 self.hashed_files = hashed_files 553 """list of source files (including the .dsc itself) 554 @type: list of L{HashedFile} 555 """ 556 557 self._dsc_file = None 558 for f in hashed_files: 559 if re_file_dsc.match(f.filename): 560 if self._dsc_file is not None: 561 raise InvalidSourceException("Multiple .dsc found ({0} and {1})".format(self._dsc_file.filename, f.filename)) 562 else: 563 self._dsc_file = f 564 565 if self._dsc_file is None: 566 raise InvalidSourceException("No .dsc included in source files") 567 568 # make sure the hash for the dsc is valid before we use it 569 self._dsc_file.check(directory) 570 571 dsc_file_path = os.path.join(directory, self._dsc_file.input_filename) 572 with open(dsc_file_path, 'rb') as fd: 573 data = fd.read() 574 self.signature = SignedFile(data, keyrings, require_signature) 575 self.dsc = apt_pkg.TagSection(self.signature.contents) 576 """dict to access fields in the .dsc file 577 @type: dict-like 578 """ 579 580 self.package_list = daklib.packagelist.PackageList(self.dsc) 581 """Information about packages built by the source. 582 @type: daklib.packagelist.PackageList 583 """ 584 585 self._files = None
586 587 @classmethod
588 - def from_file(cls, directory, filename, keyrings, require_signature=True):
589 hashed_file = HashedFile.from_file(directory, filename) 590 return cls(directory, [hashed_file], keyrings, require_signature)
591 592 @property
593 - def files(self):
594 """dict mapping filenames to L{HashedFile} objects for additional source files 595 596 This list does not include the .dsc itself. 597 598 @type: dict 599 """ 600 if self._files is None: 601 self._files = parse_file_list(self.dsc, False) 602 return self._files
603 604 @property
605 - def primary_fingerprint(self):
606 """fingerprint of the key used to sign the .dsc 607 @type: str 608 """ 609 return self.signature.primary_fingerprint
610 611 @property
612 - def valid_signature(self):
613 """C{True} if the .dsc has a valid signature 614 @type: bool 615 """ 616 return self.signature.valid
617 618 @property
619 - def weak_signature(self):
620 """C{True} if the .dsc was signed using a weak algorithm 621 @type: bool 622 """ 623 return self.signature.weak_signature
624 625 @property
626 - def component(self):
627 """guessed component name 628 629 Might be wrong. Don't rely on this. 630 631 @type: str 632 """ 633 if 'Section' not in self.dsc: 634 return 'main' 635 fields = self.dsc['Section'].split('/') 636 if len(fields) > 1: 637 return fields[0] 638 return "main"
639 640 @property
641 - def filename(self):
642 """filename of .dsc file 643 @type: str 644 """ 645 return self._dsc_file.filename
646