backend.cdn_tools.cdnsync

45 """Main class of CDN sync run.""" 46 47 log_path = '/var/log/rhn/cdnsync.log' 48

49 - def __init__(self, no_packages=False, no_errata=False, no_rpms=False, no_kickstarts=False, 50 log_level=None, mount_point=None, consider_full=False, force_kickstarts=False, 51 force_all_errata=False, email=False, import_batch_size=None):

52 53 if log_level is None: 54 log_level = 0 55 self.log_level = log_level 56 CFG.set('DEBUG', log_level) 57 self.email = email 58 if self.email: 59 initEMAIL_LOG() 60 rhnLog.initLOG(self.log_path, self.log_level) 61 log2disk(0, "Command: %s" % str(sys.argv)) 62 63 rhnSQL.initDB() 64 initCFG('server.satellite') 65 66 self.cdn_repository_manager = CdnRepositoryManager(mount_point) 67 self.no_packages = no_packages 68 self.no_errata = no_errata 69 self.no_rpms = no_rpms 70 if self.no_packages and self.no_rpms: 71 log(0, "Parameter --no-rpms has no effect.") 72 self.no_kickstarts = no_kickstarts 73 self.force_all_errata = force_all_errata 74 self.force_kickstarts = force_kickstarts 75 if self.no_kickstarts and self.force_kickstarts: 76 log(0, "Parameter --force-kickstarts has no effect.") 77 78 if mount_point: 79 self.mount_point = "file://" + mount_point 80 self.consider_full = consider_full 81 else: 82 self.mount_point = CFG.CDN_ROOT 83 self.consider_full = True 84 85 verify_mappings() 86 87 f = None 88 # try block in try block - this is hack for python 2.4 compatibility 89 # to support finally 90 try: 91 try: 92 # Channel families mapping to channels 93 f = open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r') 94 self.families = json.load(f) 95 f.close() 96 97 # Channel metadata 98 f = open(constants.CHANNEL_DEFINITIONS_PATH, 'r') 99 self.channel_metadata = json.load(f) 100 f.close() 101 102 # Dist/Release channel mapping 103 f = open(constants.CHANNEL_DIST_MAPPING_PATH, 'r') 104 self.channel_dist_mapping = json.load(f) 105 f.close() 106 107 # Kickstart metadata 108 f = open(constants.KICKSTART_DEFINITIONS_PATH, 'r') 109 self.kickstart_metadata = json.load(f) 110 f.close() 111 except IOError: 112 e = sys.exc_info()[1] 113 log(1, "Ignoring channel mappings: %s" % e) 114 self.families = {} 115 self.channel_metadata = {} 116 self.channel_dist_mapping = {} 117 self.kickstart_metadata = {} 118 finally: 119 if f is not None: 120 f.close() 121 122 # Map channels to their channel family 123 self.channel_to_family = {} 124 for family in self.families: 125 for channel in self.families[family]['channels']: 126 self.channel_to_family[channel] = family 127 128 # Set already synced channels, entitled null-org channels and custom channels with associated 129 # CDN repositories 130 h = rhnSQL.prepare(""" 131 select distinct c.label, c.org_id 132 from rhnChannelFamilyPermissions cfp inner join 133 rhnChannelFamily cf on cfp.channel_family_id = cf.id inner join 134 rhnChannelFamilyMembers cfm on cf.id = cfm.channel_family_id inner join 135 rhnChannel c on cfm.channel_id = c.id 136 where c.org_id is null 137 or (c.org_id is not null and 138 exists ( 139 select cs.id 140 from rhnContentSource cs inner join 141 rhnChannelContentSource ccs on ccs.source_id = cs.id 142 where ccs.channel_id = c.id 143 and cs.org_id is null 144 ) 145 ) 146 order by c.org_id nulls first, label 147 """) 148 h.execute() 149 channels = h.fetchall_dict() or [] 150 self.synced_channels = {} 151 for channel in channels: 152 # Custom channel repositories not available, don't mark as synced 153 if channel['org_id']: 154 repos = self.cdn_repository_manager.list_associated_repos(channel['label']) 155 if not all([self.cdn_repository_manager.check_repository_availability(r) for r in repos]): 156 continue 157 self.synced_channels[channel['label']] = channel['org_id'] 158 159 # Select available channel families from DB 160 h = rhnSQL.prepare(""" 161 select distinct label 162 from rhnChannelFamilyPermissions cfp inner join 163 rhnChannelFamily cf on cfp.channel_family_id = cf.id 164 where cf.org_id is null 165 """) 166 h.execute() 167 families = h.fetchall_dict() or [] 168 self.entitled_families = [f['label'] for f in families] 169 self.import_batch_size = import_batch_size

170

171 - def _tree_available_channels(self):

172 # collect all channel from available families 173 all_channels = [] 174 channel_tree = {} 175 # Not available parents of child channels 176 not_available_channels = [] 177 for label in self.entitled_families: 178 try: 179 family = self.families[label] 180 except KeyError: 181 log2(2, 2, "WARNING: Can't find channel family in mappings: %s" % label, stream=sys.stderr) 182 continue 183 channels = [c for c in family['channels'] if c is not None] 184 all_channels.extend(channels) 185 186 # filter available channels 187 all_channels = [x for x in all_channels if 188 self.cdn_repository_manager.check_channel_availability(x, self.no_kickstarts)] 189 190 for base_channel in [x for x in all_channels if not self.channel_metadata[x]['parent_channel']]: 191 channel_tree[base_channel] = [] 192 for child_channel in [x for x in all_channels if self.channel_metadata[x]['parent_channel']]: 193 parent_channel = self.channel_metadata[child_channel]['parent_channel'] 194 # Parent not available, orphaned child channel 195 if parent_channel not in channel_tree: 196 channel_tree[parent_channel] = [] 197 not_available_channels.append(parent_channel) 198 channel_tree[parent_channel].append(child_channel) 199 200 return channel_tree, not_available_channels

201

202 - def _list_available_channels(self):

203 channel_tree, not_available_channels = self._tree_available_channels() 204 # Collect all channels 205 channel_list = [] 206 for base_channel in channel_tree: 207 channel_list.extend(channel_tree[base_channel]) 208 if base_channel not in not_available_channels: 209 channel_list.append(base_channel) 210 return channel_list

211

212 - def _can_add_repos(self, db_channel, repos):

213 # Adding custom repositories to custom channel, need to check: 214 # 1. Repositories availability - if there are SSL certificates for them 215 # 2. Channel is custom 216 # 3. Repositories are not already associated with any channels in mapping files 217 if not db_channel or not db_channel['org_id']: 218 log2(0, 0, "ERROR: Channel doesn't exist or is not custom.", stream=sys.stderr) 219 return False 220 # Repositories can't be part of any channel from mappings 221 channels = [] 222 for repo in repos: 223 channels.extend(self.cdn_repository_manager.list_channels_containing_repository(repo)) 224 if channels: 225 log2(0, 0, "ERROR: Specified repositories can't be synced because they are part of following channels: %s" % 226 ", ".join(channels), stream=sys.stderr) 227 return False 228 # Check availability of repositories 229 not_available = [] 230 for repo in repos: 231 if not self.cdn_repository_manager.check_repository_availability(repo): 232 not_available.append(repo) 233 if not_available: 234 log2(0, 0, "ERROR: Following repositories are not available: %s" % ", ".join(not_available), 235 stream=sys.stderr) 236 return False 237 return True

238

239 - def _is_channel_available(self, label):

240 # Checking channel availability, it means either: 241 # 1. Trying to sync custom channel - in this case, it has to have already associated CDN repositories, 242 # it's ensured by query populating synced_channels variable 243 # 2. Trying to sync channel from mappings - it may not exists so we check requirements from mapping files 244 db_channel = channel_info(label) 245 if db_channel and db_channel['org_id']: 246 # Custom channel doesn't have any null-org repositories assigned 247 if label not in self.synced_channels: 248 log2(0, 0, "ERROR: Custom channel '%s' doesn't contain any CDN repositories." % label, 249 stream=sys.stderr) 250 return False 251 else: 252 if label not in self.channel_metadata: 253 log2(1, 1, "WARNING: Channel '%s' not found in channel metadata mapping." % label, stream=sys.stderr) 254 return False 255 elif label not in self.channel_to_family: 256 log2(0, 0, "ERROR: Channel '%s' not found in channel family mapping." % label, stream=sys.stderr) 257 return False 258 family = self.channel_to_family[label] 259 if family not in self.entitled_families: 260 log2(0, 0, "ERROR: Channel family '%s' containing channel '%s' is not entitled." % (family, label), 261 stream=sys.stderr) 262 return False 263 elif not self.cdn_repository_manager.check_channel_availability(label, self.no_kickstarts): 264 log2(0, 0, "ERROR: Channel '%s' repositories are not available." % label, stream=sys.stderr) 265 return False 266 return True

267

268 - def _update_product_names(self, channels):

269 backend = SQLBackend() 270 batch = [] 271 272 for label in channels: 273 channel = self.channel_metadata[label] 274 if channel['product_label'] and channel['product_name']: 275 product_name = ProductName() 276 product_name['label'] = channel['product_label'] 277 product_name['name'] = channel['product_name'] 278 batch.append(product_name) 279 280 importer = ProductNamesImport(batch, backend) 281 importer.run()

282

283 - def _update_channels_metadata(self, channels):

284 # First populate rhnProductName table 285 self._update_product_names(channels) 286 287 backend = SQLBackend() 288 channels_batch = [] 289 content_sources_batch = [] 290 291 for label in channels: 292 channel = self.channel_metadata[label] 293 channel_object = Channel() 294 for k in channel.keys(): 295 channel_object[k] = channel[k] 296 297 family_object = ChannelFamily() 298 family_object['label'] = self.channel_to_family[label] 299 300 channel_object['families'] = [family_object] 301 channel_object['label'] = label 302 channel_object['basedir'] = '/' 303 304 # Backend expects product_label named as product_name 305 # To have correct value in rhnChannelProduct and reference 306 # to rhnProductName in rhnChannel 307 channel_object['product_name'] = channel['product_label'] 308 309 dists = [] 310 releases = [] 311 312 # Distribution/Release channel mapping available 313 if label in self.channel_dist_mapping: 314 dist_map = self.channel_dist_mapping[label] 315 for item in dist_map: 316 if item['eus_release']: 317 r = ReleaseChannelMap() 318 r['product'] = item['os'] 319 r['version'] = item['release'] 320 r['release'] = item['eus_release'] 321 r['channel_arch'] = item['channel_arch'] 322 releases.append(r) 323 else: 324 d = DistChannelMap() 325 for k in item: 326 d[k] = item[k] 327 dists.append(d) 328 329 channel_object['dists'] = dists 330 channel_object['release'] = releases 331 332 sources = self.cdn_repository_manager.get_content_sources_import_batch(label, backend) 333 content_sources_batch.extend(sources) 334 channel_object['content-sources'] = sources 335 336 # Set default channel access to private 337 channel_object['channel_access'] = 'private' 338 339 channels_batch.append(channel_object) 340 341 importer = ContentSourcesImport(content_sources_batch, backend) 342 importer.run() 343 344 importer = ChannelImport(channels_batch, backend) 345 importer.run()

346

347 - def _create_yum_repo(self, repo_source):

348 repo_label = self.cdn_repository_manager.get_content_source_label(repo_source) 349 try: 350 keys = self.cdn_repository_manager.get_repository_crypto_keys(repo_source['relative_url']) 351 except CdnRepositoryNotFoundError: 352 keys = [] 353 log2(1, 1, "WARNING: Repository '%s' was not found." % repo_source['relative_url'], stream=sys.stderr) 354 if keys: 355 (ca_cert_file, client_cert_file, client_key_file) = reposync.write_ssl_set_cache( 356 keys[0]['ca_cert'], keys[0]['client_cert'], keys[0]['client_key']) 357 else: 358 (ca_cert_file, client_cert_file, client_key_file) = (None, None, None) 359 log2(1, 1, "WARNING: No valid SSL certificates were found for repository '%s'." 360 % repo_source['relative_url'], stream=sys.stderr) 361 return yum_src.ContentSource(self.mount_point + str(repo_source['relative_url']), 362 str(repo_label), org=None, no_mirrors=True, 363 ca_cert_file=ca_cert_file, client_cert_file=client_cert_file, 364 client_key_file=client_key_file)

365

366 - def _sync_channel(self, channel):

367 excluded_urls = [] 368 kickstart_trees = [] 369 370 if channel in self.kickstart_metadata: 371 kickstart_trees = self.kickstart_metadata[channel] 372 excluded_urls.extend(self.cdn_repository_manager.excluded_urls) 373 374 if self.no_kickstarts: 375 kickstart_repos = self.cdn_repository_manager.get_content_sources_kickstart(channel) 376 excluded_urls.extend([x['relative_url'] for x in kickstart_repos]) 377 378 log(0, "======================================") 379 log(0, "| Channel: %s" % channel) 380 log(0, "======================================") 381 382 # Print note if channel is already EOL 383 if self._is_channel_eol(channel): 384 log(0, "NOTE: This channel reached end-of-life on %s." % 385 datetime.strptime(self.channel_metadata[channel]['eol'], "%Y-%m-%d %H:%M:%S").strftime("%Y-%m-%d")) 386 387 log(0, "Sync of channel started.") 388 log2disk(0, "Please check 'cdnsync/%s.log' for sync log of this channel." % channel, notimeYN=True) 389 sync = reposync.RepoSync(channel, 390 repo_type="yum", 391 url=None, 392 fail=False, 393 filters=False, 394 no_packages=self.no_packages, 395 no_errata=self.no_errata, 396 sync_kickstart=(not self.no_kickstarts), 397 force_all_errata=self.force_all_errata, 398 force_kickstart=self.force_kickstarts, 399 latest=False, 400 metadata_only=self.no_rpms, 401 excluded_urls=excluded_urls, 402 strict=self.consider_full, 403 log_dir="cdnsync", 404 log_level=self.log_level, 405 check_ssl_dates=True, 406 force_null_org_content=True) 407 sync.set_ks_tree_type('rhn-managed') 408 if self.import_batch_size: 409 sync.set_import_batch_size(self.import_batch_size) 410 if kickstart_trees: 411 # Assuming all trees have same install type 412 sync.set_ks_install_type(kickstart_trees[0]['ks_install_type']) 413 sync.set_urls_prefix(self.mount_point) 414 return sync.sync()

415

416 - def sync(self, channels=None):

417 # If no channels specified, sync already synced channels 418 if not channels: 419 channels = set(self.synced_channels) 420 421 # Check channel availability before doing anything 422 not_available = set() 423 available = set() 424 all_channel_list = None 425 for channel in channels: 426 # Try to expand wildcards in channel labels 427 if '*' in channel or '?' in channel or '[' in channel: 428 if all_channel_list is None: 429 all_channel_list = self._list_available_channels() + [c for c in self.synced_channels 430 if self.synced_channels[c]] 431 expanded = fnmatch.filter(all_channel_list, channel) 432 log(2, "Expanding channel '%s' to: %s" % (channel, ", ".join(expanded))) 433 if expanded: 434 for expanded_channel in expanded: 435 if not self._is_channel_available(expanded_channel): 436 not_available.add(expanded_channel) 437 else: 438 available.add(expanded_channel) 439 else: 440 not_available.add(channel) 441 elif not self._is_channel_available(channel): 442 not_available.add(channel) 443 else: 444 available.add(channel) 445 446 channels = available 447 448 error_messages = [] 449 450 # if we have not_available channels log the error immediately 451 if not_available: 452 msg = "ERROR: these channels either do not exist or are not available for synchronization:\n " + \ 453 "\n ".join(not_available) 454 error_messages.append(msg) 455 456 # BZ 1434913 - let user know if system is not activated if no available channels 457 if not available: 458 error_messages.extend(self._msg_array_if_not_activated()) 459 460 # Need to update channel metadata 461 self._update_channels_metadata([ch for ch in channels if ch in self.channel_metadata]) 462 # Make sure custom channels are properly connected with repos 463 for channel in channels: 464 if channel in self.synced_channels and self.synced_channels[channel]: 465 self.cdn_repository_manager.assign_repositories_to_channel(channel) 466 467 reposync.clear_ssl_cache() 468 469 # Finally, sync channel content 470 total_time = timedelta() 471 for channel in channels: 472 cur_time, failed_packages = self._sync_channel(channel) 473 if failed_packages < 0: 474 error_messages.append("Problems occurred during syncing channel %s. Please check " 475 "/var/log/rhn/cdnsync/%s.log for the details\n" % (channel, channel)) 476 if failed_packages > 0: 477 error_messages.append("%d packages in channel %s failed to sync. Please check " 478 "/var/log/rhn/cdnsync/%s.log for the details\n" % (failed_packages, channel, 479 channel)) 480 total_time += cur_time 481 # Switch back to cdnsync log 482 rhnLog.initLOG(self.log_path, self.log_level) 483 log2disk(0, "Sync of channel completed.") 484 485 log(0, "Total time: %s" % str(total_time).split('.')[0]) 486 487 return error_messages

488

489 - def setup_repos_and_sync(self, channels=None, add_repos=None, delete_repos=None):

490 # Fix format of relative url 491 if add_repos: 492 repos = set() 493 for repo in add_repos: 494 repo = repo.replace(CFG.CDN_ROOT, '') 495 repo_dirs = self.cdn_repository_manager.repository_tree.normalize_url(repo) 496 repo = os.path.join('/', '/'.join(repo_dirs)) 497 repos.add(repo) 498 add_repos = list(repos) 499 if delete_repos: 500 repos = set() 501 for repo in delete_repos: 502 repo = repo.replace(CFG.CDN_ROOT, '') 503 repo_dirs = self.cdn_repository_manager.repository_tree.normalize_url(repo) 504 repo = os.path.join('/', '/'.join(repo_dirs)) 505 repos.add(repo) 506 delete_repos = list(repos) 507 # We need single custom channel 508 if not channels or len(channels) > 1: 509 raise CustomChannelSyncError("Single custom channel needed.") 510 channel = list(channels)[0] 511 db_channel = channel_info(channel) 512 if add_repos and not self._can_add_repos(db_channel, add_repos): 513 raise CustomChannelSyncError("Unable to attach requested repositories to this channel.") 514 # Add custom repositories to custom channel 515 changed = self.cdn_repository_manager.assign_repositories_to_channel(channel, delete_repos=delete_repos, 516 add_repos=add_repos) 517 # Add to synced channels and sync if there are any changed repos 518 if changed and channel not in self.synced_channels: 519 self.synced_channels[channel] = db_channel['org_id'] 520 return self.sync(channels=channels)

521

522 - def count_packages(self, channels=None):

523 start_time = datetime.now() 524 reposync.clear_ssl_cache() 525 # Both entitled channels and custom channels with null-org repositories. 526 channel_list = self._list_available_channels() 527 if not channel_list: 528 error_messages = self._msg_array_if_not_activated() 529 if error_messages: 530 log(0, "\n".join(error_messages)) 531 sys.exit(1) 532 channel_list.extend([c for c in self.synced_channels if self.synced_channels[c]]) 533 534 # Only some channels specified by parameter 535 if channels: 536 new_channel_list = [] 537 for channel in channels: 538 new_channel_list.extend(fnmatch.filter(channel_list, channel)) 539 channel_list = list(set(new_channel_list)) 540 541 log(0, "Number of channels: %d" % len(channel_list)) 542 543 # Prepare repositories 544 repo_tree = {} 545 repository_count = 0 546 for channel in channel_list: 547 sources = self.cdn_repository_manager.get_content_sources(channel) 548 # Custom channel 549 if not sources: 550 repos = self.cdn_repository_manager.list_associated_repos(channel) 551 sources = [] 552 for index, repo in enumerate(sorted(repos)): 553 repo_label = "%s-%d" % (channel, index) 554 sources.append({'relative_url': repo, 'pulp_repo_label_v2': repo_label}) 555 repository_count += len(sources) 556 repo_tree[channel] = sources 557 log(0, "Number of repositories: %d" % repository_count) 558 559 downloader = ThreadedDownloader() 560 for channel in repo_tree: 561 for source in repo_tree[channel]: 562 yum_repo = self._create_yum_repo(source) 563 params = {} 564 yum_repo.set_download_parameters(params, "repodata/repomd.xml", 565 os.path.join(yum_repo.repo.basecachedir, 566 yum_repo.name, "repomd.xml.new")) 567 downloader.add(params) 568 569 progress_bar = ProgressBarLogger("Downloading repomd: ", repository_count) 570 downloader.set_log_obj(progress_bar) 571 # Overwrite existing files 572 downloader.set_force(True) 573 log2background(0, "Downloading repomd started.") 574 downloader.run() 575 log2background(0, "Downloading repomd finished.") 576 577 progress_bar = ProgressBarLogger("Comparing repomd: ", len(repo_tree)) 578 to_download_count = 0 579 repo_tree_to_update = {} 580 log2background(0, "Comparing repomd started.") 581 582 is_missing_repomd = False 583 for channel in repo_tree: 584 cdn_repodata_path = os.path.join(constants.CDN_REPODATA_ROOT, channel) 585 packages_num_path = os.path.join(cdn_repodata_path, "packages_num") 586 packages_size_path = os.path.join(cdn_repodata_path, "packages_size") 587 588 sources = repo_tree[channel] 589 yum_repos = [self._create_yum_repo(source) for source in sources] 590 591 # check all repomd files were downloaded 592 for yum_repo in yum_repos: 593 new_repomd = os.path.join(yum_repo.repo.basecachedir, yum_repo.name, "repomd.xml.new") 594 if not os.path.isfile(new_repomd): 595 is_missing_repomd = True 596 597 # packages_num file exists and all cached repomd files are up to date => skip 598 if os.path.isfile(packages_num_path) and os.path.isfile(packages_size_path) and all( 599 [x.repomd_up_to_date() for x in yum_repos]): 600 progress_bar.log(True, None) 601 continue 602 603 update_channel = False 604 for yum_repo in yum_repos: 605 # use new repomd 606 new_repomd = os.path.join(yum_repo.repo.basecachedir, yum_repo.name, "repomd.xml.new") 607 if os.path.isfile(new_repomd): 608 update_channel = True 609 os.rename(new_repomd, 610 os.path.join(yum_repo.repo.basecachedir, yum_repo.name, "repomd.xml")) 611 else: 612 # it wasn't downloaded 613 continue 614 615 for path, checksum_pair in yum_repo.get_metadata_paths(): 616 params = {} 617 yum_repo.set_download_parameters(params, path, 618 os.path.join(yum_repo.repo.basecachedir, yum_repo.name, 619 os.path.basename(path)), 620 checksum_type=checksum_pair[0], checksum_value=checksum_pair[1]) 621 downloader.add(params) 622 to_download_count += 1 623 624 # If there is at least one repo with new repomd, pass through this channel 625 if update_channel: 626 repo_tree_to_update[channel] = sources 627 628 progress_bar.log(True, None) 629 log2background(0, "Comparing repomd finished.") 630 631 progress_bar = ProgressBarLogger("Downloading metadata:", to_download_count) 632 downloader.set_log_obj(progress_bar) 633 downloader.set_force(False) 634 log2background(0, "Downloading metadata started.") 635 downloader.run() 636 log2background(0, "Downloading metadata finished.") 637 638 progress_bar = ProgressBarLogger("Counting packages: ", len(repo_tree_to_update)) 639 log2background(0, "Counting packages started.") 640 for channel in repo_tree_to_update: 641 cdn_repodata_path = os.path.join(constants.CDN_REPODATA_ROOT, channel) 642 packages_num_path = os.path.join(cdn_repodata_path, "packages_num") 643 packages_size_path = os.path.join(cdn_repodata_path, "packages_size") 644 645 sources = repo_tree_to_update[channel] 646 yum_repos = [self._create_yum_repo(source) for source in sources] 647 648 packages = {} 649 for yum_repo in yum_repos: 650 for pkg in yum_repo.raw_list_packages(): 651 nvrea = str(pkg) 652 packages[nvrea] = pkg.packagesize 653 654 # create directory for repo data if it doesn't exist 655 try: 656 os.makedirs(cdn_repodata_path) 657 except OSError: 658 exc = sys.exc_info()[1] 659 if exc.errno == errno.EEXIST and os.path.isdir(cdn_repodata_path): 660 pass 661 else: 662 raise 663 f_num_out = open(packages_num_path, 'w') 664 f_size_out = open(packages_size_path, 'w') 665 try: 666 f_num_out.write(str(len(packages))) 667 f_size_out.write(str(sum(packages.values()))) 668 finally: 669 if f_num_out is not None: 670 f_num_out.close() 671 if f_size_out is not None: 672 f_size_out.close() 673 # Delete cache to save space 674 for yum_repo in yum_repos: 675 yum_repo.clear_cache(keep_repomd=True) 676 progress_bar.log(True, None) 677 log2background(0, "Counting packages finished.") 678 679 end_time = datetime.now() 680 log(0, "Total time: %s" % str(end_time - start_time).split('.')[0]) 681 if is_missing_repomd: 682 raise CountingPackagesError("Cannot download some repomd.xml files. " 683 "Please, check /var/log/rhn/cdnsync.log for details")

684

685 - def _channel_line_format(self, channel, longest_label):

686 if self._is_channel_eol(channel): 687 eol_status = "EOL" 688 else: 689 eol_status = "" 690 if channel in self.synced_channels: 691 sync_status = 'p' 692 else: 693 sync_status = '.' 694 try: 695 packages_number = open(constants.CDN_REPODATA_ROOT + '/' + channel + "/packages_num", 'r').read() 696 # pylint: disable=W0703 697 except Exception: 698 packages_number = '?' 699 700 try: 701 packages_size = open(constants.CDN_REPODATA_ROOT + '/' + channel + "/packages_size", 'r').read() 702 packages_size = human_readable_size(int(packages_size)) 703 # pylint: disable=W0703 704 except Exception: 705 packages_size = '?B' 706 707 packages_size = "(%s)" % packages_size 708 space = " " 709 offset = longest_label - len(channel) 710 space += " " * offset 711 712 return "%3s %s %s%s%6s packages %9s" % (eol_status, sync_status, channel, space, 713 packages_number, packages_size)

714

715 - def _is_channel_eol(self, channel):

716 if channel in self.channel_metadata: 717 if 'eol' in self.channel_metadata[channel] and self.channel_metadata[channel]['eol']: 718 if datetime.now() > datetime.strptime(self.channel_metadata[channel]['eol'], "%Y-%m-%d %H:%M:%S"): 719 return True 720 return False

721

722 - def _print_unmapped_channels(self):

723 unmapped_channels = [ch for ch in self.synced_channels if not self.synced_channels[ch] 724 and ch not in self.channel_metadata] 725 if unmapped_channels: 726 log(0, "Previously synced channels not available to update from CDN:") 727 for channel in sorted(unmapped_channels): 728 log(0, " p %s" % channel)

729

730 - def print_channel_tree(self, repos=False):

731 channel_tree, not_available_channels = self._tree_available_channels() 732 733 if not channel_tree: 734 error_messages = self._msg_array_if_not_activated() 735 if not error_messages: 736 log(0, "WARNING: No available channels from channel mappings were found. " 737 "Is %s package installed?" % constants.MAPPINGS_RPM_NAME) 738 else: 739 log(0, "\n".join(error_messages)) 740 sys.exit(1) 741 742 available_base_channels = [x for x in sorted(channel_tree) if x not in not_available_channels] 743 custom_cdn_channels = [ch for ch in self.synced_channels if self.synced_channels[ch]] 744 longest_label = len(max(available_base_channels + custom_cdn_channels + 745 [i for l in channel_tree.values() for i in l] + [""], key=len)) 746 747 log(0, "p = previously imported/synced channel") 748 log(0, ". = channel not yet imported/synced") 749 log(0, "? = package count not available (try to run cdn-sync --count-packages)") 750 log(0, "EOL = channel reached end-of-life") 751 752 log(0, "Entitled base channels:") 753 if not available_base_channels: 754 log(0, " NONE") 755 for channel in available_base_channels: 756 log(0, "%s" % self._channel_line_format(channel, longest_label)) 757 if repos: 758 sources = self.cdn_repository_manager.get_content_sources(channel) 759 paths = [s['relative_url'] for s in sources] 760 for path in sorted(paths): 761 log(0, " %s" % path) 762 763 log(0, "Entitled child channels:") 764 if not (any([channel_tree[ch] for ch in channel_tree])): 765 log(0, " NONE") 766 # print information about child channels 767 for channel in sorted(channel_tree): 768 # Print only if there are any child channels 769 if channel_tree[channel]: 770 log(0, "%s:" % channel) 771 for child in sorted(channel_tree[channel]): 772 log(0, "%s" % self._channel_line_format(child, longest_label)) 773 if repos: 774 sources = self.cdn_repository_manager.get_content_sources(child) 775 paths = [s['relative_url'] for s in sources] 776 for path in sorted(paths): 777 log(0, " %s" % path) 778 779 # Not-null org_id channels 780 log(0, "Custom channels syncing from CDN:") 781 if not custom_cdn_channels: 782 log(0, " NONE") 783 for channel in sorted(custom_cdn_channels): 784 log(0, "%s" % self._channel_line_format(channel, longest_label)) 785 if repos: 786 paths = self.cdn_repository_manager.list_associated_repos(channel) 787 for path in sorted(paths): 788 log(0, " %s" % path) 789 790 # Previously synced null-org channels not available in cdn-sync-mappings 791 self._print_unmapped_channels()

792

793 - def clear_cache(self):

794 # Clear packages outside channels from DB and disk 795 log(0, "Cleaning imported packages outside channels.") 796 contentRemove.delete_outside_channels(None) 797 if os.path.isdir(constants.PACKAGE_STAGE_DIRECTORY): 798 log(0, "Cleaning package stage directory.") 799 for pkg in os.listdir(constants.PACKAGE_STAGE_DIRECTORY): 800 os.unlink(os.path.join(constants.PACKAGE_STAGE_DIRECTORY, pkg)) 801 log(0, "Cleaning orphaned CDN repositories in DB.") 802 self.cdn_repository_manager.cleanup_orphaned_repos()

803 804 @staticmethod

805 - def _get_cdn_certificate_keys_and_certs():

806 h = rhnSQL.prepare(""" 807 SELECT ck.id, ck.description, ck.key 808 FROM rhnCryptoKeyType ckt, 809 rhnCryptoKey ck 810 WHERE ckt.label = 'SSL' 811 AND ckt.id = ck.crypto_key_type_id 812 AND ck.description LIKE 'CDN_%' 813 AND ck.org_id is NULL 814 ORDER BY ck.description 815 """) 816 h.execute() 817 data = [] 818 while True: 819 row = h.fetchone_dict() 820 if row is None: 821 break 822 row['key'] = rhnSQL.read_lob(row['key']) 823 data.append(row) 824 return data

825

826 - def print_cdn_certificates_info(self, repos=False):

827 keys = self._get_cdn_certificate_keys_and_certs() 828 if not keys: 829 log2(0, 0, "No SSL certificates were found. Is your %s activated for CDN?" 830 % PRODUCT_NAME, stream=sys.stderr) 831 sys.exit(1) 832 833 for key in keys: 834 log(0, "======================================") 835 log(0, "| Certificate/Key: %s" % key['description']) 836 log(0, "======================================") 837 if constants.CA_CERT_NAME == key['description'] or constants.CLIENT_CERT_PREFIX in key['description']: 838 if not verify_certificate_dates(str(key['key'])): 839 log(0, "WARNING: This certificate is not valid.") 840 cn, serial_number, not_before, not_after = get_certificate_info(str(key['key'])) 841 log(0, "Common name: %s" % str(cn)) 842 log(0, "Serial number: %s" % str(serial_number)) 843 log(0, "Valid from: %s" % str(not_before)) 844 log(0, "Valid to: %s" % str(not_after)) 845 if constants.CLIENT_CERT_PREFIX in key['description']: 846 manager = CdnRepositoryManager(client_cert_id=int(key['id'])) 847 self.cdn_repository_manager = manager 848 log(0, "Provided channels:") 849 channel_tree, not_available_channels = self._tree_available_channels() 850 if not channel_tree: 851 log(0, " NONE") 852 for base_channel in sorted(channel_tree): 853 if base_channel not in not_available_channels: 854 log(0, " * %s" % base_channel) 855 elif channel_tree[base_channel]: 856 log(0, " * %s (only child channels provided)" % base_channel) 857 for child_channel in sorted(channel_tree[base_channel]): 858 log(0, " * %s" % child_channel) 859 if repos: 860 log(0, "Provided repositories:") 861 provided_repos = self.cdn_repository_manager.list_provided_repos(key['id']) 862 for repo in sorted(provided_repos): 863 log(0, " %s" % repo) 864 log(0, "")

865

866 - def print_eol_channel_list(self):

867 available_channels = self._list_available_channels() 868 869 # Filter only channels with EOL date defined 870 eol_channels = {} 871 for channel in available_channels: 872 if 'eol' in self.channel_metadata[channel] and self.channel_metadata[channel]['eol']: 873 eol_channels[channel] = datetime.strptime(self.channel_metadata[channel]['eol'], "%Y-%m-%d %H:%M:%S") 874 875 if eol_channels: 876 longest_label = len(max(eol_channels, key=len)) 877 else: 878 longest_label = 0 879 880 already_eol_channels = [] 881 notyet_eol_channels = [] 882 883 # Split into 2 channel groups based on current date 884 for channel in eol_channels: 885 if datetime.now() > eol_channels[channel]: 886 already_eol_channels.append(channel) 887 else: 888 notyet_eol_channels.append(channel) 889 890 # Print these channel groups, sorted by date 891 def print_channel_line(ch): 892 if ch in self.synced_channels: 893 sync_status = 'p' 894 else: 895 sync_status = '.' 896 space = " " 897 offset = longest_label - len(ch) 898 space += " " * offset 899 log(0, " %s %s%s%s" % (sync_status, channel, space, eol_channels[channel].strftime("%Y-%m-%d")))

900 901 log(0, "p = previously imported/synced channel") 902 log(0, ". = channel not yet imported/synced") 903 log(0, "Channels reached end-of-life already:") 904 if not already_eol_channels: 905 log(0, " NONE") 906 for channel in sorted(already_eol_channels, key=lambda channel: eol_channels[channel]): 907 print_channel_line(channel) 908 log(0, "Channels not reached end-of-life yet:") 909 if not notyet_eol_channels: 910 log(0, " NONE") 911 for channel in sorted(notyet_eol_channels, key=lambda channel: eol_channels[channel]): 912 print_channel_line(channel) 913 914 # Previously synced null-org channels not available in cdn-sync-mappings 915 self._print_unmapped_channels()

Source Code for Module backend.cdn_tools.cdnsync