backend.server.importlib.packageImport

1 # 2 # Copyright (c) 2008--2018 Red Hat, Inc. 3 # 4 # This software is licensed to you under the GNU General Public License, 5 # version 2 (GPLv2). There is NO WARRANTY for this software, express or 6 # implied, including the implied warranties of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 8 # along with this software; if not, see 9 # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. 10 # 11 # Red Hat trademarks are not licensed under GPLv2. No permission is 12 # granted to use or replicate Red Hat trademarks that are incorporated 13 # in this software or its documentation. 14 # 15 # 16 # Package import process 17 # 18 19 import rpm 20 import sys 21 import os.path 22 from importLib import GenericPackageImport, IncompletePackage, \ 23 Import, InvalidArchError, InvalidChannelError, \ 24 IncompatibleArchError 25 from mpmSource import mpmBinaryPackage 26 from spacewalk.common import rhn_pkg 27 from spacewalk.common.rhnConfig import CFG 28 from spacewalk.server import taskomatic 29 from spacewalk.server.rhnServer import server_packages 30 31

32 -class ChannelPackageSubscription(GenericPackageImport):

33

34 - def __init__(self, batch, backend, caller=None, strict=0, repogen=True):

35 # If strict, the set of packages that was passed in will be the only 36 # one in the channels - everything else will be unlinked 37 GenericPackageImport.__init__(self, batch, backend) 38 self.affected_channels = [] 39 # A hash keyed on the channel id, and with tuples 40 # (added_packages, removed_packages) as values (packages are package 41 # ids) 42 self.affected_channel_packages = {} 43 if not caller: 44 self.caller = "backend.(unknown)" 45 else: 46 self.caller = caller 47 self._strict_subscription = strict 48 self.repogen = repogen

49

50 - def preprocess(self):

51 # Processes the package batch to a form more suitable for database 52 # operations 53 for package in self.batch: 54 # if package object doesn't have multiple checksums (like satellite-sync objects) 55 # then let's fake it 56 if 'checksums' not in package: 57 package['checksums'] = {package['checksum_type']: package['checksum']} 58 if not isinstance(package, IncompletePackage): 59 raise TypeError("Expected an IncompletePackage instance, " 60 "got %s" % package.__class__.__name__) 61 self._processPackage(package)

62

63 - def fix(self):

64 # Look up arches and channels 65 self.backend.lookupPackageArches(self.package_arches) 66 self.backend.lookupChannels(self.channels) 67 # Initialize self.channel_package_arch_compat 68 self.channel_package_arch_compat = {} 69 for channel, channel_row in self.channels.items(): 70 if not channel_row: 71 # Unsupported channel 72 continue 73 self.channel_package_arch_compat[channel_row['channel_arch_id']] = None 74 self.backend.lookupChannelPackageArchCompat(self.channel_package_arch_compat) 75 self.backend.lookupPackageNames(self.names) 76 self.backend.lookupEVRs(self.evrs) 77 self.backend.lookupChecksums(self.checksums) 78 79 # Fix the package information up, and uniquify the packages too 80 uniqdict = {} 81 for package in self.batch: 82 if package.ignored: 83 continue 84 self._postprocessPackageNEVRA(package) 85 if not CFG.ENABLE_NVREA: 86 # nvrea disabled, skip checksum 87 nevrao = ( 88 package['name_id'], 89 package['evr_id'], 90 package['package_arch_id'], 91 package['org_id']) 92 else: 93 # As nvrea is enabled uniquify based on checksum 94 nevrao = ( 95 package['name_id'], 96 package['evr_id'], 97 package['package_arch_id'], 98 package['org_id'], 99 package['checksum_id']) 100 101 if nevrao not in uniqdict: 102 # Uniquify the channel names 103 package['channels'] = {} 104 # Initialize the channels 105 # This is a handy way of checking arch compatibility for this 106 # package with its channels 107 self.__copyChannels(package, package) 108 uniqdict[nevrao] = package 109 else: 110 # Package is found twice in the same batch 111 # Are the packages the same? 112 self._comparePackages(package, uniqdict[nevrao]) 113 # Invalidate it 114 package.ignored = 1 115 firstpackage = uniqdict[nevrao] 116 # Copy any new channels 117 self.__copyChannels(package, firstpackage) 118 # Knowing the id of the referenced package 119 package.first_package = firstpackage

120

121 - def _comparePackages(self, package1, package2):

122 # XXX This should probably do a deep compare of the two packages 123 pass

124

125 - def submit(self):

126 self.backend.lookupPackages(self.batch, self.checksums) 127 try: 128 affected_channels = self.backend.subscribeToChannels(self.batch, 129 strict=self._strict_subscription) 130 except: 131 self.backend.rollback() 132 raise 133 self.compute_affected_channels(affected_channels) 134 135 if len(self.batch) < 10: 136 # update small batch per package 137 name_ids = [pkg['name_id'] for pkg in self.batch] 138 else: 139 # update bigger batch at once 140 name_ids = [] 141 self.backend.update_newest_package_cache(caller=self.caller, 142 affected_channels=self.affected_channel_packages, name_ids=name_ids) 143 # Now that channel is updated, schedule the repo generation 144 if self.repogen: 145 taskomatic.add_to_repodata_queue_for_channel_package_subscription( 146 self.affected_channels, self.batch, self.caller) 147 self.backend.commit()

148

149 - def compute_affected_channels(self, affected_channels):

150 # Fill the list of affected channels 151 self.affected_channel_packages.clear() 152 self.affected_channel_packages.update(affected_channels) 153 for channel_label, channel_row in list(self.channels.items()): 154 channel_id = channel_row['id'] 155 if channel_id in affected_channels: 156 affected_channels[channel_id] = channel_label 157 self.affected_channels = list(affected_channels.values())

158

159 - def _processPackage(self, package):

160 GenericPackageImport._processPackage(self, package) 161 162 # Process channels 163 channels = [] 164 channelHash = {} 165 for channel in package['channels']: 166 channelName = channel['label'] 167 if channelName not in channelHash: 168 channels.append(channelName) 169 channelHash[channelName] = None 170 self.channels[channelName] = None 171 # Replace the channel list with the uniquified list 172 package.channels = channels

173 174 # Copies the channels from one package to the other

175 - def __copyChannels(self, sourcePackage, destPackage):

176 dpHash = destPackage['channels'] 177 for schannelName in sourcePackage.channels: 178 # Check if the package is compatible with the channel 179 channel = self.channels[schannelName] 180 if not channel: 181 # Unknown channel 182 sourcePackage.ignored = 1 183 raise InvalidChannelError(channel, 184 "Unsupported channel %s" % schannelName) 185 # Check channel-package compatibility 186 charch = channel['channel_arch_id'] 187 archCompat = self.channel_package_arch_compat[charch] 188 if not archCompat: 189 # Invalid architecture 190 sourcePackage.ignored = 1 191 raise InvalidArchError(charch, 192 "Invalid channel architecture %s" % charch) 193 194 # Now check if the source package's arch is compatible with the 195 # current channel 196 if sourcePackage['package_arch_id'] not in archCompat: 197 sourcePackage.ignored = 1 198 raise IncompatibleArchError(sourcePackage.arch, charch, 199 "Package arch %s incompatible with channel %s" % 200 (sourcePackage.arch, schannelName)) 201 202 dpHash[channel['id']] = schannelName 203 204 destPackage.channels = list(dpHash.values())

205 206

207 -class PackageImport(ChannelPackageSubscription):

208

209 - def __init__(self, batch, backend, caller=None, update_last_modified=0):

210 ChannelPackageSubscription.__init__(self, batch, backend, 211 caller=caller) 212 self.ignoreUploaded = 1 213 self._update_last_modified = update_last_modified 214 self.capabilities = {} 215 self.groups = {} 216 self.sourceRPMs = {} 217 self.changelog_data = {}

218

219 - def _rpm_knows(self, tag):

220 # See if the installed version of RPM understands a given tag 221 # Assumed attr-format in RPM is 'RPMTAG_<UPPERCASETAG>' 222 return hasattr(rpm, 'RPMTAG_'+tag.upper())

223

224 - def _processPackage(self, package):

225 ChannelPackageSubscription._processPackage(self, package) 226 227 # Process package groups 228 group = package['package_group'] 229 if group not in self.groups: 230 self.groups[group] = None 231 sourceRPM = package['source_rpm'] 232 if (sourceRPM is not None) and (sourceRPM not in self.sourceRPMs): 233 self.sourceRPMs[sourceRPM] = None 234 # Change copyright to license 235 # XXX 236 package['copyright'] = self._fix_encoding(package['license']) 237 238 for tag in ('recommends', 'suggests', 'supplements', 'enhances', 'breaks', 'predepends'): 239 if not self._rpm_knows(tag) or tag not in package or type(package[tag]) != type([]): 240 # older spacewalk server do not export weak deps. 241 # and older RPM doesn't know about them either 242 # lets create an empty list 243 package[tag] = [] 244 245 # Creates all the data structures needed to insert capabilities 246 for tag in ('provides', 'requires', 'conflicts', 'obsoletes', 'recommends', 'suggests', 'supplements', 'enhances', 'breaks', 'predepends'): 247 depList = package[tag] 248 if type(depList) != type([]): 249 sys.stderr.write("!!! packageImport.PackageImport._processPackage: " 250 "erronous depList for '%s', converting to []\n" % tag) 251 depList = [] 252 for dep in depList: 253 nv = [] 254 for f in ('name', 'version'): 255 nv.append(dep[f]) 256 del dep[f] 257 nv = tuple(nv) 258 dep['capability'] = nv 259 if nv not in self.capabilities: 260 self.capabilities[nv] = None 261 # Process files too 262 fileList = package['files'] 263 for f in fileList: 264 filename = self._fix_encoding(f['name']) 265 nv = (filename, '') 266 del f['name'] 267 f['capability'] = nv 268 if nv not in self.capabilities: 269 self.capabilities[nv] = None 270 fchecksumTuple = (f['checksum_type'], f['checksum']) 271 if fchecksumTuple not in self.checksums: 272 self.checksums[fchecksumTuple] = None 273 274 # Uniquify changelog entries 275 unique_package_changelog_hash = {} 276 unique_package_changelog = [] 277 for changelog in package['changelog']: 278 key = (changelog['name'], changelog['time'], changelog['text']) 279 if key not in unique_package_changelog_hash: 280 self.changelog_data[key] = None 281 unique_package_changelog.append(changelog) 282 unique_package_changelog_hash[key] = 1 283 package['changelog'] = unique_package_changelog 284 285 # fix encoding issues in package summary and description 286 package['description'] = self._fix_encoding(package['description']) 287 package['summary'] = self._fix_encoding(package['summary'])

288

289 - def fix(self):

290 # If capabilities are available, process them 291 if self.capabilities: 292 try: 293 self.backend.processCapabilities(self.capabilities) 294 except: 295 # Oops 296 self.backend.rollback() 297 raise 298 # Since this is the bulk of the work, commit 299 self.backend.commit() 300 301 self.backend.processChangeLog(self.changelog_data) 302 303 ChannelPackageSubscription.fix(self) 304 305 self.backend.lookupSourceRPMs(self.sourceRPMs) 306 self.backend.lookupPackageGroups(self.groups) 307 # Postprocess the gathered information 308 self.__postprocess()

309

310 - def submit(self):

311 upload_force = self.uploadForce 312 if not upload_force and self._update_last_modified: 313 # # Force it just a little bit - kind of hacky 314 upload_force = 0.5 315 try: 316 self.backend.processPackages(self.batch, 317 uploadForce=upload_force, 318 forceVerify=self.forceVerify, 319 ignoreUploaded=self.ignoreUploaded, 320 transactional=self.transactional) 321 self._import_signatures() 322 except: 323 # Oops 324 self.backend.rollback() 325 raise 326 self.backend.commit() 327 if not self._update_last_modified: 328 # Go though the list of objects and clear out the ones that have a 329 # force of 0.5 330 for p in self.batch: 331 if p.diff and p.diff.level == 0.5: 332 # Ignore this difference completely 333 p.diff = None

334 # Leave p.diff_result in place 335

336 - def subscribeToChannels(self):

337 affected_channels = self.backend.subscribeToChannels(self.batch) 338 # Fill the list of affected channels 339 self.compute_affected_channels(affected_channels) 340 341 name_ids = [pkg['name_id'] for pkg in self.batch] 342 self.backend.update_newest_package_cache(caller=self.caller, 343 affected_channels=self.affected_channel_packages, name_ids=name_ids) 344 taskomatic.add_to_repodata_queue_for_channel_package_subscription( 345 self.affected_channels, self.batch, self.caller) 346 self.backend.commit()

347

348 - def __postprocess(self):

349 # Gather the IDs we've found 350 351 for package in self.batch: 352 if package.ignored: 353 # Skip it 354 continue 355 # Only deal with packages 356 self.__postprocessPackage(package)

357

358 - def __postprocessPackage(self, package):

359 """ populate the columns foo_id with id numbers from appropriate hashes """ 360 package['package_group'] = self.groups[package['package_group']] 361 source_rpm = package['source_rpm'] 362 if source_rpm is not None: 363 source_rpm = self.sourceRPMs[source_rpm] 364 else: 365 source_rpm = '' 366 package['source_rpm_id'] = source_rpm 367 package['checksum_id'] = self.checksums[(package['checksum_type'], package['checksum'])] 368 369 # Postprocess the dependency information 370 for tag in ('provides', 'requires', 'conflicts', 'obsoletes', 'files', 'recommends', 'suggests', 'supplements', 'enhances', 'breaks', 'predepends'): 371 for entry in package[tag]: 372 nv = entry['capability'] 373 entry['capability_id'] = self.capabilities[nv] 374 for c in package['changelog']: 375 c['changelog_data_id'] = self.changelog_data[(c['name'], c['time'], c['text'])] 376 fileList = package['files'] 377 for f in fileList: 378 f['checksum_id'] = self.checksums[(f['checksum_type'], f['checksum'])]

379

380 - def _comparePackages(self, package1, package2):

381 if (package1['checksum_type'] == package2['checksum_type'] 382 and package1['checksum'] == package2['checksum']): 383 return 384 # XXX Handle this better 385 raise Exception("Different packages in the same batch")

386

387 - def _cleanup_object(self, object):

388 ChannelPackageSubscription._cleanup_object(self, object) 389 if object.ignored: 390 object.id = object.first_package.id

391

392 - def _import_signatures(self):

393 for package in self.batch: 394 # skip missing files and mpm packages 395 if package['path'] and not isinstance(package, mpmBinaryPackage): 396 full_path = os.path.join(CFG.MOUNT_POINT, package['path']) 397 if os.path.exists(full_path): 398 header = rhn_pkg.get_package_header(filename=full_path) 399 server_packages.processPackageKeyAssociations(header, 400 package['checksum_type'], package['checksum'])

401

402 - def _fix_encoding(self, text):

403 if text is None: 404 return None 405 try: 406 return text.decode('utf8') 407 except UnicodeDecodeError: 408 return text.decode('iso8859-1')

409 410

411 -class SourcePackageImport(Import):

412

413 - def __init__(self, batch, backend, caller=None, update_last_modified=0):

414 Import.__init__(self, batch, backend) 415 self._update_last_modified = update_last_modified 416 self.ignoreUploaded = 1 417 self.sourceRPMs = {} 418 self.groups = {} 419 self.checksums = {}

420

421 - def preprocess(self):

422 for package in self.batch: 423 self._processPackage(package)

424

425 - def fix(self):

426 self.backend.lookupSourceRPMs(self.sourceRPMs) 427 self.backend.lookupPackageGroups(self.groups) 428 self.backend.lookupChecksums(self.checksums) 429 self.__postprocess() 430 # Uniquify the packages 431 uniqdict = {} 432 for package in self.batch: 433 # Unique key 434 key = (package['org_id'], package['source_rpm_id']) 435 if key not in uniqdict: 436 uniqdict[key] = package 437 continue 438 else: 439 self._comparePackages(package, uniqdict[key]) 440 # And invalidate it 441 package.ignored = 1 442 package.first_package = uniqdict[key]

443

444 - def submit(self):

445 upload_force = self.uploadForce 446 if not upload_force and self._update_last_modified: 447 # # Force it just a little bit - kind of hacky 448 upload_force = 0.5 449 try: 450 self.backend.processSourcePackages(self.batch, 451 uploadForce=upload_force, 452 forceVerify=self.forceVerify, 453 ignoreUploaded=self.ignoreUploaded, 454 transactional=self.transactional) 455 except: 456 # Oops 457 self.backend.rollback() 458 raise 459 self.backend.commit() 460 if not self._update_last_modified: 461 # Go though the list of objects and clear out the ones that have a 462 # force of 0.5 463 for p in self.batch: 464 if p.diff and p.diff.level == 0.5: 465 # Ignore this difference completely 466 p.diff = None

467 # Leave p.diff_result in place 468

469 - def _comparePackages(self, package1, package2):

470 if (package1['checksum_type'] == package2['checksum_type'] 471 and package1['checksum'] == package2['checksum']): 472 return 473 # XXX Handle this better 474 raise Exception("Different packages in the same batch")

475

476 - def _processPackage(self, package):

477 Import._processPackage(self, package) 478 # Fix the arch 479 package.arch = 'src' 480 package.source_rpm = package['source_rpm'] 481 sourceRPM = package['source_rpm'] 482 if not sourceRPM: 483 # Should not happen 484 raise Exception("Source RPM %s does not exist") 485 self.sourceRPMs[sourceRPM] = None 486 self.groups[package['package_group']] = None 487 488 checksumTuple = (package['checksum_type'], package['checksum']) 489 if checksumTuple not in self.checksums: 490 self.checksums[checksumTuple] = None 491 492 sigchecksumTuple = (package['sigchecksum_type'], package['sigchecksum']) 493 if sigchecksumTuple not in self.checksums: 494 self.checksums[sigchecksumTuple] = None

495

496 - def __postprocess(self):

497 # Gather the IDs we've found 498 499 for package in self.batch: 500 if package.ignored: 501 # Skip it 502 continue 503 # Only deal with packages 504 self.__postprocessPackage(package)

505

506 - def __postprocessPackage(self, package):

507 # Set the ids 508 package['package_group'] = self.groups[package['package_group']] 509 package['source_rpm_id'] = self.sourceRPMs[package['source_rpm']] 510 package['checksum_id'] = self.checksums[(package['checksum_type'], 511 package['checksum'])] 512 package['sigchecksum_id'] = self.checksums[(package['sigchecksum_type'], 513 package['sigchecksum'])]

514

515 - def _cleanup_object(self, object):

516 Import._cleanup_object(self, object) 517 if object.ignored: 518 object.id = object.first_package.id

519 520

521 -def packageImporter(batch, backend, source=0, caller=None):

522 if source: 523 return SourcePackageImport(batch, backend, caller=caller) 524 return PackageImport(batch, backend, caller=caller)

525

Source Code for Module backend.server.importlib.packageImport