backend.satellite_tools.sync

1 # 2 # Copyright (c) 2008--2016 Red Hat, Inc. 3 # 4 # This software is licensed to you under the GNU General Public License, 5 # version 2 (GPLv2). There is NO WARRANTY for this software, express or 6 # implied, including the implied warranties of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 8 # along with this software; if not, see 9 # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. 10 # 11 # Red Hat trademarks are not licensed under GPLv2. No permission is 12 # granted to use or replicate Red Hat trademarks that are incorporated 13 # in this software or its documentation. 14 # 15 16 import sys 17 import string # pylint: disable=W0402 18 19 from spacewalk.common import usix 20 from spacewalk.server.importlib import channelImport, packageImport, errataImport, \ 21 kickstartImport 22 from spacewalk.common.usix import raise_with_tb 23 import diskImportLib 24 import xmlSource 25 import syncCache 26 import syncLib 27 28 DEFAULT_ORG = 1

29 30 # Singleton-like 31 32 33 -class BaseCollection:

34 _shared_state = {} 35

36 - def __init__(self):

37 self.__dict__ = self._shared_state 38 if not list(self._shared_state.keys()): 39 self._items = [] 40 self._cache = None 41 self._items_hash = {} 42 self._init_fields() 43 self._init_cache()

44

45 - def add_item(self, item):

46 item_id = self._get_item_id(item) 47 timestamp = self._get_item_timestamp(item) 48 self._cache.cache_set(item_id, item, timestamp=timestamp) 49 return self

50

51 - def get_item_timestamp(self, item_id):

52 "Returns this item's timestamp" 53 if item_id not in self._items_hash: 54 raise KeyError("Item %s not found in collection" % item_id) 55 return self._items_hash[item_id]

56

57 - def get_item(self, item_id, timestamp):

58 "Retrieve an item from the collection" 59 return self._cache.cache_get(item_id, timestamp=timestamp)

60

61 - def has_item(self, item_id, timestamp):

62 """Return true if the item exists in the collection (with the 63 specified timestamp""" 64 return self._cache.cache_has_key(item_id, timestamp=timestamp)

65

66 - def _init_fields(self):

67 return self

68

69 - def _init_cache(self):

70 return self

71

72 - def _get_item_id(self, item):

73 "Get the item ID out of an item. Override in subclasses" 74 raise NotImplementedError

75

76 - def _get_item_timestamp(self, item):

77 "Get the item timestamp out of an item. Override in subclasses" 78 raise NotImplementedError

79

80 - def reset(self):

81 """Reset the collection""" 82 self._shared_state.clear() 83 self.__init__()

84

85 # Singleton-like 86 87 88 -class ChannelCollection:

89 _shared_state = {} 90

91 - def __init__(self):

92 self.__dict__ = self._shared_state 93 if not list(self._shared_state.keys()): 94 self._channels = [] 95 self._parent_channels = {} 96 self._channels_hash = {} 97 self._cache = syncCache.ChannelCache()

98

99 - def add_item(self, channel_object):

100 """Stores a channel in the collection""" 101 channel_label = channel_object['label'] 102 channel_last_modified = channel_object['last_modified'] 103 last_modified = _to_timestamp(channel_last_modified) 104 self._cache.cache_set(channel_label, channel_object, 105 timestamp=last_modified) 106 t = (channel_label, last_modified) 107 self._channels.append(t) 108 channel_parent = channel_object.get('parent_channel') 109 if channel_parent is not None: 110 # Add this channel to the parent's list 111 l = self._get_list_from_dict(self._parent_channels, channel_parent) 112 l.append(t) 113 else: 114 # Create an empty list 115 self._get_list_from_dict(self._parent_channels, channel_label) 116 self._channels_hash[channel_label] = last_modified 117 return self

118 119 @staticmethod

120 - def _get_list_from_dict(diction, key):

121 # Returns the dictionary's key if present (assumed to be a list), or 122 # sets the value to an empty list and returns it 123 if key in diction: 124 l = diction[key] 125 else: 126 l = diction[key] = [] 127 return l

128

129 - def get_channel_labels(self):

130 """Return the channel labels from this collection""" 131 return [x[0] for x in self._channels]

132

133 - def get_channels(self):

134 """Return a list of (channel label, channel timestamp) from this 135 collection""" 136 return self._channels[:]

137

138 - def get_channel(self, channel_label, timestamp):

139 """Return the channel with the specified label and timestamp from the 140 collection""" 141 return self._cache.cache_get(channel_label, timestamp=timestamp)

142

143 - def get_channel_timestamp(self, channel_label):

144 """Returns the channel's timestamp""" 145 if channel_label not in self._channels_hash: 146 raise KeyError("Channel %s could not be found" % channel_label) 147 return self._channels_hash[channel_label]

148

149 - def get_parent_channel_labels(self):

150 """Return a list of channel labels for parent channels""" 151 l = list(self._parent_channels.keys()) 152 l.sort() 153 return l

154

155 - def get_child_channels(self, channel_label):

156 """Return a list of (channel label, channel timestamp) for this parent 157 channel""" 158 if channel_label not in self._parent_channels: 159 raise Exception("Channel %s is not a parent" % channel_label) 160 return self._parent_channels[channel_label]

161

162 - def reset(self):

163 """Reset the collection""" 164 self._shared_state.clear() 165 self.__init__()

166

167 # pylint: disable=W0232 168 169 170 -class SyncHandlerContainer:

171 collection = object 172 173 # this class has no __init__ for the purpose 174 # it's used in multiple inheritance mode and inherited classes should 175 # use __init__ from the other base class 176

177 - def endItemCallback(self):

178 # reference to xmlSource superclass we redefines 179 xml_superclass = self.__class__.__bases__[1] 180 xml_superclass.endItemCallback(self) 181 # pylint: disable=E1101 182 if not self.batch: 183 return 184 c = self.collection() 185 c.add_item(self.batch[-1]) 186 del self.batch[:]

187

188 - def endContainerCallback(self):

189 # Not much to do here... 190 pass

191

192 193 -def get_sync_handler(container):

194 handler = xmlSource.SatelliteDispatchHandler() 195 handler.set_container(container) 196 return handler

197

198 199 -class ChannelContainer(SyncHandlerContainer, xmlSource.ChannelContainer):

200 collection = ChannelCollection

201

202 203 -def get_channel_handler():

204 return get_sync_handler(ChannelContainer())

205

206 207 -def import_channels(channels, orgid=None, master=None):

208 collection = ChannelCollection() 209 batch = [] 210 org_map = None 211 my_backend = diskImportLib.get_backend() 212 if master: 213 org_map = my_backend.lookupOrgMap(master)['master-id-to-local-id'] 214 for c in channels: 215 try: 216 timestamp = collection.get_channel_timestamp(c) 217 except KeyError: 218 raise_with_tb(Exception("Could not find channel %s" % c), sys.exc_info()[2]) 219 c_obj = collection.get_channel(c, timestamp) 220 if c_obj is None: 221 raise Exception("Channel not found in cache: %s" % c) 222 223 # Check to see if we're asked to sync to an orgid, 224 # make sure the org from the export is not null org, 225 # finally if the orgs differ so we might wanna use 226 # requested org's channel-family. 227 # TODO: Move these checks somewhere more appropriate 228 if not orgid and c_obj['org_id'] is not None: 229 # If the src org is not present default to org 1 230 orgid = DEFAULT_ORG 231 if orgid is not None and c_obj['org_id'] is not None and \ 232 c_obj['org_id'] != orgid: 233 # If we know the master this is coming from and the master org 234 # has been mapped to a local org, transform org_id to the local 235 # org_id. Otherwise just put it in the default org. 236 if (org_map and c_obj['org_id'] in list(org_map.keys()) 237 and org_map[c_obj['org_id']]): 238 c_obj['org_id'] = org_map[c_obj['org_id']] 239 else: 240 c_obj['org_id'] = orgid 241 if c_obj.has_key('trust_list'): 242 del(c_obj['trust_list']) 243 for family in c_obj['families']: 244 family['label'] = 'private-channel-family-' + \ 245 str(c_obj['org_id']) 246 # If there's a trust list on the channel, transform the org ids to 247 # the local ones 248 if c_obj.has_key('trust_list') and c_obj['trust_list']: 249 trusts = [] 250 for trust in c_obj['trust_list']: 251 if trust['org_trust_id'] in org_map: 252 trust['org_trust_id'] = org_map[trust['org_trust_id']] 253 trusts.append(trust) 254 c_obj['trust_list'] = trusts 255 256 syncLib.log(6, "Syncing Channel %s to Org %s " % (c_obj['label'], c_obj['org_id'])) 257 batch.append(c_obj) 258 259 importer = channelImport.ChannelImport(batch, my_backend) 260 # Don't commit just yet 261 importer.will_commit = 0 262 importer.run() 263 return importer

264

265 # Singleton-like 266 267 268 -class ShortPackageCollection:

269 _shared_state = {} 270

271 - def __init__(self):

272 self.__dict__ = self._shared_state 273 if not list(self._shared_state.keys()): 274 self._cache = None 275 self._init_cache()

276

277 - def _init_cache(self):

278 self._cache = syncCache.ShortPackageCache()

279

280 - def add_item(self, package):

281 """Stores a package in the collection""" 282 self._cache.cache_set(package['package_id'], package)

283

284 - def get_package(self, package_id):

285 """Return the package with the specified id from the collection""" 286 return self._cache.cache_get(package_id)

287

288 - def has_package(self, package_id):

289 """Returns true if the package exists in the collection""" 290 return self._cache.cache_has_key(package_id)

291

292 - def reset(self):

293 """Reset the collection""" 294 self._shared_state.clear() 295 self.__init__()

296

297 298 -class ShortPackageContainer(SyncHandlerContainer, xmlSource.IncompletePackageContainer):

299 collection = ShortPackageCollection

300

301 302 -def get_short_package_handler():

303 return get_sync_handler(ShortPackageContainer())

304

305 306 -class PackageCollection(ShortPackageCollection):

307 _shared_state = {} 308

309 - def _init_cache(self):

310 self._cache = syncCache.PackageCache()

311

312 - def get_package_timestamp(self, package_id):

313 raise NotImplementedError

314

315 316 -class PackageContainer(SyncHandlerContainer, xmlSource.PackageContainer):

317 collection = PackageCollection

318

319 320 -def get_package_handler():

321 return get_sync_handler(PackageContainer())

322

323 324 # Singleton-like 325 -class SourcePackageCollection(ShortPackageCollection):

326 _shared_state = {} 327

328 - def _init_cache(self):

329 self._cache = syncCache.SourcePackageCache()

330

331 332 -class SourcePackageContainer(SyncHandlerContainer, xmlSource.SourcePackageContainer):

333 collection = SourcePackageCollection

334

335 336 -def get_source_package_handler():

337 return get_sync_handler(SourcePackageContainer())

338

339 # Singleton-like 340 341 342 -class ErrataCollection:

343 _shared_state = {} 344

345 - def __init__(self):

346 self.__dict__ = self._shared_state 347 if not list(self._shared_state.keys()): 348 self._errata_hash = {} 349 self._cache = None 350 self._init_cache()

351

352 - def _init_cache(self):

353 self._cache = syncCache.ErratumCache()

354

355 - def add_item(self, erratum):

356 """Stores an erratum in the collection""" 357 erratum_id = erratum['erratum_id'] 358 timestamp = _to_timestamp(erratum['last_modified']) 359 self._errata_hash[erratum_id] = timestamp 360 self._cache.cache_set(erratum_id, erratum, timestamp=timestamp)

361

362 - def get_erratum_timestamp(self, erratum_id):

363 """Returns the erratum's timestamp""" 364 if erratum_id not in self._errata_hash: 365 raise KeyError("Erratum %s could not be found" % erratum_id) 366 return self._errata_hash[erratum_id]

367

368 - def get_erratum(self, erratum_id, timestamp):

369 """Return the erratum with the specified id and timestamp from the 370 collection. Note that timestamp can be None, in which case no timetamp 371 matching is performed""" 372 return self._cache.cache_get(erratum_id, timestamp=timestamp)

373

374 - def has_erratum(self, erratum_id, timestamp):

375 """Returns true if the erratum exists in the collection""" 376 return self._cache.cache_has_key(erratum_id, timestamp=timestamp)

377

378 - def reset(self):

379 """Reset the collection""" 380 self._shared_state.clear() 381 self.__init__()

382

383 384 -class ErrataContainer(SyncHandlerContainer, xmlSource.ErrataContainer):

385 collection = ErrataCollection

386

387 388 -def get_errata_handler():

389 return get_sync_handler(ErrataContainer())

390

391 392 -class KickstartableTreesCollection(BaseCollection):

393 _shared_state = {} 394

395 - def _init_cache(self):

396 self._cache = syncCache.KickstartableTreesCache()

397

398 - def _get_item_id(self, item):

399 return item['label']

400

401 - def _get_item_timestamp(self, item):

402 return None

403

404 405 -class KickstartableTreesContainer(SyncHandlerContainer, xmlSource.KickstartableTreesContainer):

406 collection = KickstartableTreesCollection

407

408 409 -def get_kickstarts_handler():

410 return get_sync_handler(KickstartableTreesContainer())

411

412 413 -def import_packages(batch, sources=0):

414 importer = packageImport.PackageImport(batch, diskImportLib.get_backend(), sources) 415 importer.setUploadForce(4) 416 importer.run() 417 importer.status() 418 return importer

419

420 421 -def link_channel_packages(batch, strict=1):

422 importer = packageImport.ChannelPackageSubscription(batch, 423 diskImportLib.get_backend(), 424 caller="satsync.linkPackagesToChannels", strict=strict) 425 importer.run() 426 importer.status() 427 return importer

428

429 430 -def import_errata(batch):

431 importer = errataImport.ErrataImport(batch, diskImportLib.get_backend()) 432 importer.ignoreMissing = 1 433 importer.run() 434 importer.status() 435 return importer

436

437 438 -def import_kickstarts(batch):

439 importer = kickstartImport.KickstartableTreeImport(batch, 440 diskImportLib.get_backend()) 441 importer.run() 442 importer.status() 443 return importer

444

445 446 -def _to_timestamp(t):

447 if isinstance(t, usix.IntType): 448 # Already an int 449 return t 450 # last_modified is YYYY-MM-DD HH24:MI:SS 451 # The cache expects YYYYMMDDHH24MISS as format; so just drop the 452 # spaces, dashes and columns 453 # python 2.4 can't handle t.translate(None, ' -:') 454 last_modified = t.translate(string.maketrans("", ""), ' -:') 455 return last_modified

456

457 # Generic container handler 458 459 460 -class ContainerHandler:

461 462 """generate and set container XML handlers""" 463

464 - def __init__(self, master_label, create_orgs=False):

465 self.handler = xmlSource.SatelliteDispatchHandler() 466 # arch containers 467 self.setServerArchContainer() 468 self.setPackageArchContainer() 469 self.setChannelArchContainer() 470 self.setCPUArchContainer() 471 self.setServerPackageArchContainer() 472 self.setServerChannelArchContainer() 473 self.setServerGroupServerArchContainer() 474 self.setChannelPackageArchContainer() 475 # all other containers 476 self.setChannelFamilyContainer() 477 self.setProductNamesContainer() 478 self.setOrgContainer(master_label, create_orgs)

479

480 - def __del__(self):

481 self.handler.close() # kill the circular reference.

482

483 - def close(self):

484 self.handler.close() # kill the circular reference.

485

486 - def clear(self):

487 self.handler.clear() # clear the batch

488 489 # basic functionality:

490 - def process(self, stream):

491 self.handler.process(stream)

492

493 - def reset(self):

494 self.handler.reset()

495

496 - def getHandler(self):

497 return self.handler

498 499 # set arch containers:

500 - def setServerArchContainer(self):

501 self.handler.set_container(diskImportLib.ServerArchContainer())

502

503 - def setPackageArchContainer(self):

504 self.handler.set_container(diskImportLib.PackageArchContainer())

505

506 - def setChannelArchContainer(self):

507 self.handler.set_container(diskImportLib.ChannelArchContainer())

508

509 - def setCPUArchContainer(self):

510 self.handler.set_container(diskImportLib.CPUArchContainer())

511

512 - def setServerPackageArchContainer(self):

513 self.handler.set_container(diskImportLib.ServerPackageArchCompatContainer())

514

515 - def setServerChannelArchContainer(self):

516 self.handler.set_container(diskImportLib.ServerChannelArchCompatContainer())

517

518 - def setServerGroupServerArchContainer(self):

519 self.handler.set_container(diskImportLib.ServerGroupServerArchCompatContainer())

520

521 - def setChannelPackageArchContainer(self):

522 self.handler.set_container(ChannelPackageArchCompatContainer())

523 # set all other containers: 524

525 - def setChannelFamilyContainer(self):

526 self.handler.set_container(ChannelFamilyContainer())

527

528 - def setProductNamesContainer(self):

529 self.handler.set_container(diskImportLib.ProductNamesContainer())

530

531 - def setOrgContainer(self, master_label, create_orgs):

532 # pylint: disable=E1101,E1103 533 self.handler.set_container(diskImportLib.OrgContainer()) 534 self.handler.get_container('rhn-orgs').set_master_and_create_org_args( 535 master_label, create_orgs)

536

537 # 538 # more containers 539 # 540 # NOTE: we use *most* the Arch Containers from diskImportLib.py 541 # this one is used simply to print out the arches. 542 543 544 -class ChannelPackageArchCompatContainer(diskImportLib.ChannelPackageArchCompatContainer):

545 546 arches = {} 547

548 - def endItemCallback(self):

549 diskImportLib.ChannelPackageArchCompatContainer.endItemCallback(self) 550 if not self.batch: 551 return 552 self.arches[self.batch[-1]['package-arch']] = 1

553

554 - def endContainerCallback(self):

555 arches = list(self.arches.keys()) 556 arches.sort() 557 if arches: 558 for arch in arches: 559 syncLib.log(6, ' parsed arch: %s' % (arch)) 560 diskImportLib.ChannelPackageArchCompatContainer.endContainerCallback(self)

561

562 563 -class ChannelFamilyContainer(xmlSource.ChannelFamilyContainer):

564

565 - def endItemCallback(self):

566 xmlSource.ChannelFamilyContainer.endItemCallback(self) 567 if not self.batch: 568 return 569 syncLib.log(2, ' parsing family: %s' % (self.batch[-1]['name']))

570

571 - def endContainerCallback(self):

572 batch = self.batch 573 # use the copy only; don't want a persistent self.batch 574 self.batch = [] 575 576 importer = channelImport.ChannelFamilyImport(batch, 577 diskImportLib.get_backend()) 578 importer.run()

579

Source Code for Module backend.satellite_tools.sync_handlers