Package backend :: Package server :: Package importlib :: Module importLib
[hide private]
[frames] | no frames]

Source Code for Module backend.server.importlib.importLib

  1  # 
  2  # Copyright (c) 2008--2018 Red Hat, Inc. 
  3  # 
  4  # This software is licensed to you under the GNU General Public License, 
  5  # version 2 (GPLv2). There is NO WARRANTY for this software, express or 
  6  # implied, including the implied warranties of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 
  8  # along with this software; if not, see 
  9  # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. 
 10  # 
 11  # Red Hat trademarks are not licensed under GPLv2. No permission is 
 12  # granted to use or replicate Red Hat trademarks that are incorporated 
 13  # in this software or its documentation. 
 14  # 
 15  # 
 16  # Common data structures used throughout the import code 
 17  # 
 18   
 19  import os 
 20  import shutil 
 21  from spacewalk.common.usix import IntType, StringType, InstanceType 
 22  from UserDict import UserDict 
 23  try: 
 24      #  python 2 
 25      from UserList import UserList 
 26  except ImportError: 
 27      #  python3 
 28      from collections import UserList 
 29   
 30  from spacewalk.common.checksum import getFileChecksum 
 31  from spacewalk.common.fileutils import createPath 
 32  from spacewalk.common.rhnConfig import CFG 
 33   
 34  # no-op class, used to define the type of an attribute 
 35   
 36   
37 -class DateType:
38 pass
39 40 41 # An Item is just an extension for a dictionary
42 -class Item(UserDict):
43 44 """ 45 First level object, that stores information in a hash-like structure 46 """ 47
48 - def __init__(self, attributes=None):
49 UserDict.__init__(self, attributes)
50
51 - def populate(self, hash):
52 self.update(hash) 53 return self
54
55 - def __repr__(self):
56 return "[<%s instance; attributes=%s]" % (str(self.__class__), 57 str(self.data))
58 59 # BaseInformation is an Item with a couple of other features (an id, an ignored 60 # flag, diff information) 61 62
63 -class BaseInformation(Item):
64 65 """ 66 Second level object. It may contain composite items as attributes 67 """ 68
69 - def __init__(self, dict=None):
70 Item.__init__(self, dict) 71 # Initialize attributes 72 for k in dict.keys(): 73 self[k] = None 74 # Each information object has an id (which is set by the database) 75 self.id = None 76 # If the information is ignored (non-critical) 77 self.ignored = None 78 # Diff with the object already installed 79 self.diff = None 80 # Same as above, except that it doesn't get cleared if the upload was 81 # forced 82 self.diff_result = None
83
84 - def toDict(self):
85 dict = { 86 'ignored': not not self.ignored, 87 'diff': self.diff.toDict(), 88 } 89 return dict
90 91 # This class is handy for reducing code duplication 92 93
94 -class Information(BaseInformation):
95 attributeTypes = {} 96
97 - def __init__(self):
99 100 # Function that validates the insertion of items in a Collection 101 102
103 -def validateInformation(obj):
104 if not isinstance(obj, BaseInformation): 105 if isinstance(obj, InstanceType): 106 strtype = "instance of %s" % obj.__class__ 107 else: 108 strtype = str(type(obj)) 109 raise TypeError("Expected an Information object; got %s" % strtype)
110 111 112 # A list with the needed functions to validate what gets put in it
113 -class Collection(UserList):
114
115 - def __init__(self, list=None):
116 if list: 117 for obj in list: 118 validateInformation(obj) 119 UserList.__init__(self, list)
120
121 - def __setitem__(self, i, item):
122 validateInformation(item) 123 UserList.__setitem__(self, i, item)
124
125 - def append(self, item):
126 validateInformation(item) 127 UserList.append(self, item)
128 129 add = append 130
131 - def insert(self, i, item):
132 validateInformation(item) 133 UserList.insert(self, i, item)
134
135 - def extend(self, other):
136 for obj in other: 137 validateInformation(obj) 138 UserList.extend(self, other)
139
140 - def __setslice__(self, i, j, other):
141 for obj in other: 142 validateInformation(obj) 143 UserList.__setslice__(self, i, j, other)
144
145 - def __add__(self, other):
146 for obj in other: 147 validateInformation(obj) 148 UserList.__add__(self, other)
149
150 - def __radd__(self, other):
151 for obj in other: 152 validateInformation(obj) 153 UserList.__radd__(self, other)
154
155 - def __repr__(self):
156 return "[<%s instance; items=%s]" % (str(self.__class__), 157 str(self.data))
158 159 160 # Import classes 161 # XXX makes sense to put this in a different file
162 -class ChannelFamily(Information):
163 attributeTypes = { 164 'name': StringType, 165 'label': StringType, 166 'product_url': StringType, 167 'channels': [StringType], 168 'org_id': IntType, 169 }
170
171 -class DistChannelMap(Information):
172 attributeTypes = { 173 'os': StringType, 174 'release': StringType, 175 'channel_arch': StringType, 176 'channel': StringType, 177 'org_id': IntType, 178 }
179
180 -class ReleaseChannelMap(Information):
181 attributeTypes = { 182 'product': StringType, 183 'version': StringType, 184 'release': StringType, 185 'channel_arch_id': IntType, 186 'channel_id': IntType 187 }
188 189
190 -class ChannelErratum(Information):
191 attributeTypes = { 192 'id': StringType, 193 'advisory_name': StringType, 194 'last_modified': DateType, 195 }
196 197
198 -class IncompleteSourcePackage(Information):
199 attributeTypes = { 200 'id': StringType, 201 'source_rpm': StringType, 202 'last_modified': DateType, 203 }
204 205
206 -class ChannelTrust(Information):
207 attributeTypes = { 208 'org_trust_id': IntType, 209 }
210 211
212 -class ContentSourceSsl(Information):
213 attributeTypes = { 214 'ssl_ca_cert_id': IntType, 215 'ssl_client_cert_id': IntType, 216 'ssl_client_key_id': IntType, 217 }
218 219
220 -class ContentSource(Information):
221 attributeTypes = { 222 'label': StringType, 223 'source_url': StringType, 224 'type_id': IntType, 225 'org_id': IntType, 226 'ssl-sets': [ContentSourceSsl], 227 'channels': [StringType], 228 }
229 230
231 -class Channel(Information):
232 attributeTypes = { 233 'label': StringType, 234 'org_id': IntType, 235 'channel_arch': StringType, 236 'parent_channel': StringType, 237 'name': StringType, 238 'summary': StringType, 239 'description': StringType, 240 'last_modified': DateType, 241 'comps_last_modified': DateType, 242 'modules_last_modified': DateType, 243 'gpg_key_url': StringType, 244 'product_name_id': IntType, 245 'channel_product_id': IntType, 246 'receiving_updates': StringType, 247 'checksum_type': StringType, # xml dumps >= 3.5 248 'channel_access': StringType, 249 # XXX Not really useful stuff 250 'basedir': StringType, 251 'product_name': StringType, 252 'product_version': StringType, 253 'product_beta': StringType, 254 # Families this channel is subscribed to 255 'families': [ChannelFamily], 256 'packages': [StringType], 257 'source_packages': [IncompleteSourcePackage], 258 'all-packages': [StringType], 259 'dists': [DistChannelMap], 260 'release': [ReleaseChannelMap], 261 'errata': [StringType], 262 'errata_timestamps': [ChannelErratum], 263 'kickstartable_trees': [StringType], 264 'trust_list': [ChannelTrust], 265 'export-type': StringType, 266 'export-end-date': StringType, 267 'export-start-date': StringType, 268 'content-sources': [ContentSource], 269 }
270 271
272 -class OrgTrust(Information):
273 attributeTypes = { 274 'org_id': IntType, 275 }
276 277
278 -class Org(Information):
279 attributeTypes = { 280 'id': IntType, 281 'name': StringType, 282 'org_trust_ids': [OrgTrust], 283 }
284 285
286 -class File(Item):
287 attributeTypes = { 288 'name': StringType, 289 'device': IntType, 290 'inode': IntType, 291 'file_mode': IntType, 292 'username': StringType, 293 'groupname': StringType, 294 'rdev': IntType, 295 'file_size': IntType, 296 'mtime': DateType, 297 'linkto': StringType, 298 'flags': IntType, 299 'verifyflags': IntType, 300 'lang': StringType, 301 'checksum': StringType, 302 'checksum_type': StringType, 303 } 304
305 - def __init__(self):
306 Item.__init__(self, self.attributeTypes)
307 308
309 -class Dependency(Item):
310 attributeTypes = { 311 'name': StringType, 312 'version': StringType, 313 'flags': IntType, 314 } 315
316 - def __init__(self):
317 Item.__init__(self, self.attributeTypes)
318 319
320 -class ChangeLog(Item):
321 attributeTypes = { 322 'name': StringType, 323 'text': StringType, 324 'time': DateType, 325 } 326
327 - def __init__(self):
328 Item.__init__(self, self.attributeTypes)
329 330
331 -class Checksum(Item):
332 attributeTypes = { 333 'type': StringType, 334 'value': StringType, 335 } 336
337 - def __init__(self):
338 Item.__init__(self, self.attributeTypes)
339 340
341 -class IncompletePackage(BaseInformation):
342 attributeTypes = { 343 'package_id': StringType, # RH db id 344 'name': StringType, 345 'epoch': StringType, 346 'version': StringType, 347 'release': StringType, 348 'arch': StringType, 349 'org_id': IntType, 350 'package_size': IntType, 351 'last_modified': DateType, 352 'md5sum': StringType, # xml dumps < 3.5 353 # These attributes are lists of objects 354 'channels': [StringType], 355 'checksum_list': [Checksum], 356 } 357
358 - def __init__(self):
359 BaseInformation.__init__(self, IncompletePackage.attributeTypes) 360 self.name = None 361 self.evr = None 362 self.arch = None 363 self.org_id = None
364
365 - def toDict(self):
366 dict = BaseInformation.toDict(self) 367 evr = list(self.evr) 368 if evr[0] is None: 369 evr[0] = '' 370 371 dict['name'] = self.name 372 dict['evr'] = evr 373 dict['arch'] = self.arch 374 375 org_id = self.org_id 376 if org_id is None: 377 org_id = '' 378 dict['org_id'] = org_id 379 return dict
380
381 - def short_str(self):
382 return "%s-%s-%s.%s.rpm" % (self.name, self.evr[1], self.evr[2], 383 self.arch)
384 385
386 -class Package(IncompletePackage):
387 388 """ 389 A package is a hash of attributes 390 """ 391 attributeTypes = { 392 'description': StringType, 393 'summary': StringType, 394 'license': StringType, 395 'package_group': StringType, 396 'rpm_version': StringType, 397 'payload_size': IntType, 398 'installed_size': IntType, 399 'payload_format': StringType, 400 'build_host': StringType, 401 'build_time': DateType, 402 'cookie': StringType, 403 'vendor': StringType, 404 'source_rpm': StringType, 405 'package_size': IntType, 406 'last_modified': DateType, 407 'sigpgp': StringType, 408 'siggpg': StringType, 409 'sigsize': IntType, 410 'header_start': IntType, 411 'header_end': IntType, 412 'path': StringType, 413 'md5sum': StringType, # xml dumps < 3.5 414 'sigmd5': StringType, 415 'multi_arch': StringType, 416 # These attributes are lists of objects 417 'files': [File], 418 'requires': [Dependency], 419 'provides': [Dependency], 420 'conflicts': [Dependency], 421 'obsoletes': [Dependency], 422 'recommends': [Dependency], 423 'supplements': [Dependency], 424 'enhances': [Dependency], 425 'suggests': [Dependency], 426 'breaks': [Dependency], 427 'predepends': [Dependency], 428 'changelog': [ChangeLog], 429 'channels': [StringType], 430 'checksum_list': [Checksum], 431 } 432
433 - def __init__(self):
434 # Inherit from IncompletePackage 435 IncompletePackage.__init__(self) 436 # And initialize the specific ones 437 for k in self.attributeTypes.keys(): 438 self[k] = None
439 440
441 -class SourcePackage(IncompletePackage):
442 attributeTypes = { 443 'package_group': StringType, 444 'rpm_version': StringType, 445 'source_rpm': StringType, 446 'payload_size': IntType, 447 'payload_format': StringType, 448 'build_host': StringType, 449 'build_time': DateType, 450 'vendor': StringType, 451 'cookie': StringType, 452 'package_size': IntType, 453 'path': StringType, 454 'last_modified': DateType, 455 # these attributes are mutualy exclusive 456 'md5sum': StringType, # xml dumps < 3.5 457 'sigmd5': StringType, # xml dumps < 3.5 and rpms 458 'checksum_list': [Checksum], 459 } 460
461 - def __init__(self):
462 # Inherit from IncompletePackage 463 IncompletePackage.__init__(self) 464 # And initialize the specific ones 465 self.source_rpm = None 466 for k in self.attributeTypes.keys(): 467 self[k] = None
468
469 - def short_str(self):
470 return self.source_rpm
471 472
473 -class Bug(Information):
474 attributeTypes = { 475 'bug_id': StringType, 476 'summary': StringType, 477 'href': StringType, 478 }
479 480
481 -class ErrataFile(Information):
482 attributeTypes = { 483 'filename': StringType, 484 'file_type': StringType, 485 'channel_list': [StringType], 486 'package_id': IntType, 487 # these attributes are mutualy exclusive 488 'md5sum': StringType, # xml dumps < 3.5 489 'checksum_list': [Checksum], 490 }
491 492
493 -class Keyword(Information):
494 attributeTypes = { 495 'keyword': StringType, 496 }
497 498
499 -class Erratum(Information):
500 attributeTypes = { 501 'advisory': StringType, 502 'advisory_name': StringType, 503 'advisory_rel': IntType, 504 'advisory_type': StringType, 505 'product': StringType, 506 'description': StringType, 507 'synopsis': StringType, 508 'topic': StringType, 509 'solution': StringType, 510 'issue_date': DateType, 511 'update_date': DateType, 512 'last_modified': DateType, 513 'notes': StringType, 514 'org_id': IntType, 515 'refers_to': StringType, 516 # These attributes are lists of objects 517 'channels': [Channel], 518 'packages': [IncompletePackage], 519 'files': [ErrataFile], 520 'keywords': [Keyword], 521 'bugs': [Bug], 522 'cve': [StringType], 523 'errata_from': StringType, 524 'severity_id': IntType 525 }
526 527
528 -class BaseArch(Information):
529 attributeTypes = { 530 'label': StringType, 531 'name': StringType, 532 }
533 534
535 -class CPUArch(BaseArch):
536 pass
537 538
539 -class BaseTypedArch(BaseArch):
540 attributeTypes = BaseArch.attributeTypes.copy() 541 attributeTypes.update({ 542 'arch-type-label': StringType, 543 'arch-type-name': StringType, 544 })
545 546
547 -class ServerArch(BaseTypedArch):
548 pass
549 550
551 -class PackageArch(BaseTypedArch):
552 pass
553 554
555 -class ChannelArch(BaseTypedArch):
556 pass
557 558
559 -class ServerPackageArchCompat(Information):
560 attributeTypes = { 561 'server-arch': StringType, 562 'package-arch': StringType, 563 'preference': IntType, 564 }
565 566
567 -class ServerChannelArchCompat(Information):
568 attributeTypes = { 569 'server-arch': StringType, 570 'channel-arch': StringType, 571 }
572 573
574 -class ChannelPackageArchCompat(Information):
575 attributeTypes = { 576 'channel-arch': StringType, 577 'package-arch': StringType, 578 }
579 580
581 -class ServerGroupServerArchCompat(Information):
582 attributeTypes = { 583 'server-arch': StringType, 584 'server-group-type': StringType, 585 }
586 587
588 -class KickstartFile(Information):
589 attributeTypes = { 590 'relative_path': StringType, 591 'last_modified': DateType, 592 'file_size': IntType, 593 'md5sum': StringType, # xml dumps < 3.5 594 'checksum_list': [Checksum], 595 }
596 597
598 -class KickstartableTree(Information):
599 attributeTypes = { 600 'label': StringType, 601 'base_path': StringType, 602 'channel': StringType, 603 'boot_image': StringType, 604 'kstree_type_label': StringType, 605 'install_type_name': StringType, 606 'kstree_type_label': StringType, 607 'install_type_name': StringType, 608 'org_id': IntType, 609 'last_modified': DateType, 610 'files': [KickstartFile], 611 }
612 613
614 -class ProductName(Information):
615 attributeTypes = { 616 'label': StringType, 617 'name': StringType, 618 }
619 620 621 # Generic error object
622 -class Error(Information):
623 attributeTypes = { 624 'error': StringType, 625 }
626 627 628 # Base import class
629 -class Import:
630
631 - def __init__(self, batch, backend):
632 self.batch = batch 633 self.backend = backend 634 # Upload force 635 self.uploadForce = 1 636 # Force object verification 637 self.forceVerify = 0 638 # Ignore already-uploaded objects 639 self.ignoreUploaded = 0 640 # Transactional behaviour 641 self.transactional = 0
642
643 - def setUploadForce(self, value):
644 self.uploadForce = value
645
646 - def setForceVerify(self, value):
647 self.forceVerify = value
648
649 - def setIgnoreUploaded(self, value):
650 self.ignoreUploaded = value
651
652 - def setTransactional(self, value):
653 self.transactional = value
654 655 # This is the generic API exposed by an importer
656 - def preprocess(self):
657 pass
658
659 - def fix(self):
660 pass
661
662 - def submit(self):
663 pass
664
665 - def run(self):
666 self.preprocess() 667 self.fix() 668 self.submit()
669
670 - def cleanup(self):
671 # Clean up the objects in the batch 672 for object in self.batch: 673 self._cleanup_object(object)
674
675 - def _cleanup_object(self, object):
676 object.clear()
677
678 - def status(self):
679 # Report the status back 680 self.cleanup() 681 return self.batch
682
683 - def _processPackage(self, package):
684 # Build the helper data structures 685 evr = [] 686 for f in ('epoch', 'version', 'release'): 687 evr.append(package[f]) 688 package.evr = tuple(evr) 689 package.name = package['name'] 690 package.arch = package['arch'] 691 package.org_id = package['org_id']
692 693 694 # Any package processing import class
695 -class GenericPackageImport(Import):
696
697 - def __init__(self, batch, backend):
698 Import.__init__(self, batch, backend) 699 # Packages have to be pre-processed 700 self.names = {} 701 self.evrs = {} 702 self.checksums = {} 703 self.package_arches = {} 704 self.channels = {} 705 self.channel_package_arch_compat = {}
706
707 - def _processPackage(self, package):
708 Import._processPackage(self, package) 709 710 # Save the fields in the local hashes 711 if package.evr not in self.evrs: 712 self.evrs[package.evr] = None 713 714 if package.name not in self.names: 715 self.names[package.name] = None 716 717 if package.arch not in self.package_arches: 718 self.package_arches[package.arch] = None 719 720 for type, chksum in package['checksums'].items(): 721 checksumTuple = (type, chksum) 722 if not checksumTuple in self.checksums: 723 self.checksums[checksumTuple] = None
724
725 - def _postprocessPackageNEVRA(self, package):
726 arch = self.package_arches[package.arch] 727 if not arch: 728 # Unsupported arch 729 package.ignored = 1 730 raise InvalidArchError(package.arch, 731 "Unknown arch %s" % package.arch) 732 733 # package['package_arch_id'] = arch 734 # package['name_id'] = self.names[package.name] 735 # package['evr_id'] = self.evrs[package.evr] 736 737 nevra = (self.names[package.name], self.evrs[package.evr], arch) 738 nevra_dict = {nevra: None} 739 740 self.backend.lookupPackageNEVRAs(nevra_dict) 741 742 package['name_id'], package['evr_id'], package['package_arch_id'] = nevra 743 package['nevra_id'] = nevra_dict[nevra] 744 package['checksum_id'] = self.checksums[(package['checksum_type'], package['checksum'])]
745 746 # Exceptions 747 748
749 -class ImportException(Exception):
750
751 - def __init__(self, arglist):
752 Exception.__init__(self, *arglist)
753 754
755 -class AlreadyUploadedError(ImportException):
756
757 - def __init__(self, object, *rest):
758 ImportException.__init__(self, rest) 759 self.object = object
760 761
762 -class FileConflictError(AlreadyUploadedError):
763 pass
764 765
766 -class InvalidPackageError(ImportException):
767
768 - def __init__(self, package, *rest):
769 ImportException.__init__(self, rest) 770 self.package = package
771 772
773 -class InvalidArchError(ImportException):
774
775 - def __init__(self, arch, *rest):
776 ImportException.__init__(self, rest) 777 self.arch = arch
778 779
780 -class InvalidChannelError(ImportException):
781
782 - def __init__(self, channel, *rest):
783 ImportException.__init__(self, rest) 784 self.channel = channel
785 786
787 -class MissingParentChannelError(ImportException):
788
789 - def __init__(self, channel, *rest):
790 ImportException.__init__(self, rest) 791 self.channel = channel
792 793
794 -class InvalidChannelFamilyError(ImportException):
795
796 - def __init__(self, channel_family, *rest):
797 ImportException.__init__(self, rest) 798 self.channel_family = channel_family
799 800
801 -class IncompatibleArchError(ImportException):
802
803 - def __init__(self, arch1, arch2, *rest):
804 ImportException.__init__(self, rest) 805 self.arch1 = arch1 806 self.arch2 = arch2
807 808
809 -class InvalidSeverityError(ImportException):
810
811 - def __init__(self, *rest):
812 ImportException.__init__(self, rest)
813 814
815 -class TransactionError(ImportException):
816
817 - def __init__(self, *rest):
818 ImportException.__init__(self, rest)
819 820 # Class that stores diff information 821 822
823 -class Diff(UserList):
824
825 - def __init__(self):
826 UserList.__init__(self) 827 self.level = 0
828
829 - def setLevel(self, level):
830 if self.level < level: 831 self.level = level
832
833 - def toDict(self):
834 # Converts the object to a dictionary 835 l = [] 836 for item in self: 837 l.append(removeNone(item)) 838 return { 839 'level': self.level, 840 'diff': l, 841 }
842 843 844 # Replaces all occurences of None with the empty string
845 -def removeNone(list):
846 return [(x is not None and x) or '' for x in list]
847 848 849 # Assorted functions for various things 850
851 -def move_package(filename, basedir, relpath, checksum_type, checksum, force=None):
852 """ 853 Copies the information from the file descriptor to a file 854 Checks the file's checksum, raising FileConflictErrror if it's different 855 The force flag prevents the exception from being raised, and copies the 856 file even if the checksum has changed 857 """ 858 packagePath = basedir + "/" + relpath 859 # Is the file there already? 860 if os.path.isfile(packagePath): 861 if force: 862 os.unlink(packagePath) 863 else: 864 # Get its checksum 865 localsum = getFileChecksum(checksum_type, packagePath) 866 if checksum == localsum: 867 # Same file, so get outa here 868 return 869 raise FileConflictError(os.path.basename(packagePath)) 870 871 dir = os.path.dirname(packagePath) 872 # Create the directory where the file will reside 873 if not os.path.exists(dir): 874 createPath(dir) 875 876 # Check if the RPM has been downloaded from a remote repository 877 # If so, it is stored in CFG.MOUNT_POINT and we have to move it 878 # If not, the repository is local to the server, so the rpm should be copied 879 if filename.startswith(CFG.MOUNT_POINT): 880 shutil.move(filename, packagePath) 881 else: 882 shutil.copy(filename, packagePath) 883 884 # set the path perms readable by all users 885 os.chmod(packagePath, int('0644', 8))
886 887 888 # Returns a list of containing nevra for the given RPM header 889 NEVRA_TAGS = ['name', 'epoch', 'version', 'release', 'arch'] 890 891
892 -def get_nevra(header):
893 # Get nevra 894 nevra = [] 895 for tag in NEVRA_TAGS: 896 nevra.append(header[tag]) 897 return nevra
898 899
900 -def get_nevra_dict(header):
901 # Get nevra 902 nevra = {} 903 for tag in NEVRA_TAGS: 904 nevra[tag] = header[tag] 905 if nevra['epoch'] == '': 906 nevra['epoch'] = None 907 return nevra
908