Trees       Indices       Help   
Package backend :: Package cdn_tools :: Module repository
[hide private]
[frames] | no frames]

Source Code for Module backend.cdn_tools.repository

  1  # Copyright (c) 2016--2018 Red Hat, Inc. 
  2  # 
  3  # This software is licensed to you under the GNU General Public License, 
  4  # version 2 (GPLv2). There is NO WARRANTY for this software, express or 
  5  # implied, including the implied warranties of MERCHANTABILITY or FITNESS 
  6  # FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 
  7  # along with this software; if not, see 
  8  # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. 
  9  # 
 10  # Red Hat trademarks are not licensed under GPLv2. No permission is 
 11  # granted to use or replicate Red Hat trademarks that are incorporated 
 12  # in this software or its documentation. 
 13  # 
 14   
 15  import os 
 16  import sys 
 17  import json 
 18   
 19  from spacewalk.server import rhnSQL 
 20  from spacewalk.server.importlib.backendOracle import SQLBackend 
 21  from spacewalk.server.importlib.contentSourcesImport import ContentSourcesImport 
 22  from spacewalk.satellite_tools.satCerts import verify_certificate_dates 
 23  from spacewalk.satellite_tools.syncLib import log, log2 
 24  from spacewalk.server.importlib.importLib import ContentSource, ContentSourceSsl 
 25   
 26  import constants 

 27   
 28   
 29 -class CdnRepositoryManager(object): 

 30      """Class managing CDN repositories, connected channels etc.""" 
 31   

 32 -    def __init__(self, local_mount_point=None, client_cert_id=None): 

 33          rhnSQL.initDB() 
 34          self.local_mount_point = local_mount_point 
 35          self.repository_tree = CdnRepositoryTree() 
 36          self._populate_repository_tree(client_cert_id=client_cert_id) 
 37          self.excluded_urls = [] 
 38   
 39          f = None 
 40          try: 
 41              try: 
 42                  # Channel to repositories mapping 
 43                  f = open(constants.CONTENT_SOURCE_MAPPING_PATH, 'r') 
 44                  self.content_source_mapping = json.load(f) 
 45                  f.close() 
 46   
 47                  # Channel to kickstart repositories mapping 
 48                  f = open(constants.KICKSTART_SOURCE_MAPPING_PATH, 'r') 
 49                  self.kickstart_source_mapping = json.load(f) 
 50                  f.close() 
 51   
 52                  # Kickstart metadata 
 53                  f = open(constants.KICKSTART_DEFINITIONS_PATH, 'r') 
 54                  self.kickstart_metadata = json.load(f) 
 55                  f.close() 
 56              except IOError: 
 57                  e = sys.exc_info()[1] 
 58                  log(1, "Ignoring channel mappings: %s" % e) 
 59                  self.content_source_mapping = {} 
 60                  self.kickstart_source_mapping = {} 
 61                  self.kickstart_metadata = {} 
 62          finally: 
 63              if f is not None: 
 64                  f.close() 
 65   
 66          self.__init_repository_to_channels_mapping() 

 67   
 68      # Map repositories to channels 

 69 -    def __init_repository_to_channels_mapping(self): 

 70          self.repository_to_channels = {} 
 71          for channel in self.content_source_mapping: 
 72              for source in self.content_source_mapping[channel]: 
 73                  relative_url = source['relative_url'] 
 74                  if relative_url in self.repository_to_channels: 
 75                      self.repository_to_channels[relative_url].append(channel) 
 76                  else: 
 77                      self.repository_to_channels[relative_url] = [channel] 
 78   
 79          for channel in self.kickstart_metadata: 
 80              for tree in self.kickstart_metadata[channel]: 
 81                  tree_label = tree['ks_tree_label'] 
 82                  if tree_label in self.kickstart_source_mapping: 
 83                      relative_url = self.kickstart_source_mapping[tree_label][0]['relative_url'] 
 84                      if relative_url in self.repository_to_channels: 
 85                          self.repository_to_channels[relative_url].append(channel) 
 86                      else: 
 87                          self.repository_to_channels[relative_url] = [channel] 

 88   

 89 -    def _populate_repository_tree(self, client_cert_id=None): 

 90          sql = """ 
 91              select cs.label, cs.source_url, csssl.ssl_ca_cert_id, 
 92                     csssl.ssl_client_cert_id, csssl.ssl_client_key_id 
 93              from rhnContentSource cs inner join 
 94                   rhnContentSourceSsl csssl on cs.id = csssl.content_source_id 
 95              where cs.org_id is null 
 96                and cs.label like :prefix || '%%' 
 97          """ 
 98          # Create repository tree containing only repositories provided from single client certificate 
 99          if client_cert_id: 
100              sql += " and csssl.ssl_client_cert_id = :client_cert_id" 
101          query = rhnSQL.prepare(sql) 
102          query.execute(prefix=constants.MANIFEST_REPOSITORY_DB_PREFIX, client_cert_id=client_cert_id) 
103          rows = query.fetchall_dict() or [] 
104          cdn_repositories = {} 
105          # Loop all rows from DB 
106          for row in rows: 
107              label = row['label'] 
108              if label in cdn_repositories: 
109                  cdn_repository = cdn_repositories[label] 
110              else: 
111                  cdn_repository = CdnRepository(label, row['source_url']) 
112                  cdn_repositories[label] = cdn_repository 
113   
114              # Append SSL cert, key set to repository 
115              ssl_set = CdnRepositorySsl(row['ssl_ca_cert_id'], row['ssl_client_cert_id'], row['ssl_client_key_id']) 
116              cdn_repository.add_ssl_set(ssl_set) 
117   
118          # Add populated repository to tree 
119          for cdn_repository in cdn_repositories.values(): 
120              self.repository_tree.add_repository(cdn_repository) 

121   

122 -    def get_content_sources_regular(self, channel_label, source=False): 

123          if channel_label in self.content_source_mapping: 
124              return [x for x in self.content_source_mapping[channel_label] 
125                      if source or x['pulp_content_category'] != "source"] 
126          else: 
127              return [] 

128   

129 -    def get_content_sources_kickstart(self, channel_label): 

130          repositories = [] 
131          if channel_label in self.kickstart_metadata: 
132              for tree in self.kickstart_metadata[channel_label]: 
133                  tree_label = tree['ks_tree_label'] 
134                  if tree_label in self.kickstart_source_mapping: 
135                      # One tree comes from one repo, one repo for each tree is in the mapping, 
136                      # in future there may be multiple repos for one tree and we will need to select 
137                      # correct repo 
138                      repository = self.kickstart_source_mapping[tree_label][0] 
139                      repository['ks_tree_label'] = tree_label 
140                      repositories.append(repository) 
141                  else: 
142                      log2(1, 1, "WARNING: Can't find repository for kickstart tree in mappings: %s" 
143                           % tree_label, stream=sys.stderr) 
144          return repositories 

145   

146 -    def get_content_sources(self, channel_label, source=False): 

147          sources = self.get_content_sources_regular(channel_label, source=source) 
148          kickstart_sources = self.get_content_sources_kickstart(channel_label) 
149          return sources + sorted(kickstart_sources) 

150   

151 -    def check_channel_availability(self, channel_label, no_kickstarts=False): 

152          """Checks if all repositories for channel are available.""" 
153          if no_kickstarts: 
154              sources = self.get_content_sources_regular(channel_label) 
155          else: 
156              sources = self.get_content_sources(channel_label) 
157   
158          # No content, no channel 
159          if not sources: 
160              return False 
161   
162          for source in sources: 
163              if not self.check_repository_availability(source['relative_url'], channel_label=channel_label): 
164                  if source.get('ks_tree_label', None): 
165                      # don't fail if kickstart is missing, just warn (bz1626797) 
166                      log2(0, 0, "WARNING: kickstart tree '%s' is unavailable" % source['ks_tree_label'], 
167                           stream=sys.stderr) 
168                      self.excluded_urls.append(source['relative_url']) 
169                  else: 
170                      return False 
171          return True 

172   

173 -    def check_repository_availability(self, relative_url, channel_label=None): 

174          try: 
175              crypto_keys = self.get_repository_crypto_keys(relative_url) 
176          except CdnRepositoryNotFoundError: 
177              log2(1, 1, "ERROR: No SSL certificates were found for repository '%s'" % relative_url, stream=sys.stderr) 
178              return False 
179   
180          # Check SSL certificates 
181          if not crypto_keys: 
182              if channel_label: 
183                  log2(1, 1, "ERROR: No valid SSL certificates were found for repository '%s'" 
184                             " required for channel '%s'." % (relative_url, channel_label), stream=sys.stderr) 
185              else: 
186                  log2(1, 1, "ERROR: No valid SSL certificates were found for repository '%s'." % relative_url, 
187                       stream=sys.stderr) 
188              return False 
189   
190          # Try to look for repomd file 
191          if self.local_mount_point and not os.path.isfile(os.path.join( 
192                  self.local_mount_point, relative_url[1:], "repodata/repomd.xml")): 
193              return False 
194   
195          return True 

196   

197 -    def get_content_sources_import_batch(self, channel_label, backend, repos=None): 

198          batch = [] 
199   
200          # No custom repos specified, look into channel mappings 
201          if not repos: 
202              sources = self.get_content_sources(channel_label) 
203              for source in sources: 
204                  if 'ks_tree_label' in source: 
205                      content_source = self._create_content_source_obj(source['ks_tree_label'], 
206                                                                       source['relative_url'], backend) 
207                  else: 
208                      content_source = self._create_content_source_obj(source['pulp_repo_label_v2'], 
209                                                                       source['relative_url'], backend) 
210                  batch.append(content_source) 
211          # We want to sync not-mapped repositories 
212          else: 
213              for index, repo in enumerate(repos): 
214                  repo_label = "%s-%d" % (channel_label, index) 
215                  content_source = self._create_content_source_obj(repo_label, repo, backend) 
216                  batch.append(content_source) 
217   
218          return batch 

219   

220 -    def _create_content_source_obj(self, label, source_url, backend): 

221          type_id = backend.lookupContentSourceType('yum') 
222          content_source = ContentSource() 
223          content_source['label'] = label 
224          content_source['source_url'] = source_url 
225          content_source['org_id'] = None 
226          content_source['type_id'] = type_id 
227          content_source['ssl-sets'] = [] 
228          repository = self.repository_tree.find_repository(source_url) 
229          for ssl_set in repository.get_ssl_sets(): 
230              content_source_ssl = ContentSourceSsl() 
231              content_source_ssl['ssl_ca_cert_id'] = ssl_set.get_ca_cert() 
232              content_source_ssl['ssl_client_cert_id'] = ssl_set.get_client_cert() 
233              content_source_ssl['ssl_client_key_id'] = ssl_set.get_client_key() 
234              content_source['ssl-sets'].append(content_source_ssl) 
235          return content_source 

236   

237 -    def get_repository_crypto_keys(self, url): 

238          repo = self.repository_tree.find_repository(url) 
239          crypto_keys = [] 
240          for ssl_set in repo.get_ssl_sets(): 
241              keys = ssl_set.get_crypto_keys(check_dates=True) 
242              if keys: 
243                  crypto_keys.append(keys) 
244          return crypto_keys 

245   

246 -    def assign_repositories_to_channel(self, channel_label, delete_repos=None, add_repos=None): 

247          backend = SQLBackend() 
248          self.unlink_all_repos(channel_label, custom_only=True) 
249          repos = self.list_associated_repos(channel_label) 
250          changed = 0 
251          if delete_repos: 
252              for to_delete in delete_repos: 
253                  if to_delete in repos: 
254                      repos.remove(to_delete) 
255                      log(0, "Removing repository '%s' from channel." % to_delete) 
256                      changed += 1 
257                  else: 
258                      log2(0, 0, "WARNING: Repository '%s' is not attached to channel." % to_delete, stream=sys.stderr) 
259          if add_repos: 
260              for to_add in add_repos: 
261                  if to_add not in repos: 
262                      repos.append(to_add) 
263                      log(0, "Attaching repository '%s' to channel." % to_add) 
264                      changed += 1 
265                  else: 
266                      log2(0, 0, "WARNING: Repository '%s' is already attached to channel." % to_add, stream=sys.stderr) 
267   
268          # If there are any repositories intended to be attached to channel 
269          if repos: 
270              content_sources_batch = self.get_content_sources_import_batch( 
271                  channel_label, backend, repos=sorted(repos)) 
272              for content_source in content_sources_batch: 
273                  content_source['channels'] = [channel_label] 
274                  importer = ContentSourcesImport(content_sources_batch, backend) 
275                  importer.run() 
276          else: 
277              # Make sure everything is unlinked 
278              self.unlink_all_repos(channel_label) 
279          return changed 

280   
281      @staticmethod 
294   
295      @staticmethod 

296 -    def list_associated_repos(channel_label): 

297          h = rhnSQL.prepare(""" 
298                  select cs.source_url 
299                  from rhnChannel c inner join 
300                       rhnChannelContentSource ccs on c.id = ccs.channel_id inner join 
301                       rhnContentSource cs on ccs.source_id = cs.id 
302                  where c.label = :label 
303                    and cs.org_id is null 
304              """) 
305          h.execute(label=channel_label) 
306          paths = [r['source_url'] for r in h.fetchall_dict() or []] 
307          return paths 

308   
309      @staticmethod 

310 -    def list_provided_repos(crypto_key_id): 

311          h = rhnSQL.prepare(""" 
312                  select cs.source_url 
313                  from rhnContentSource cs inner join 
314                       rhnContentSourceSsl csssl on cs.id = csssl.content_source_id 
315                  where cs.label like :prefix || '%%' 
316                    and csssl.ssl_client_cert_id = :client_cert_id 
317              """) 
318          h.execute(prefix=constants.MANIFEST_REPOSITORY_DB_PREFIX, client_cert_id=crypto_key_id) 
319          paths = [r['source_url'] for r in h.fetchall_dict() or []] 
320          return paths 

321   
322      @staticmethod 

324          h = rhnSQL.prepare(""" 
325              delete from rhnContentSource cs 
326              where cs.org_id is null 
327                and cs.label not like :prefix || '%%' 
328                and not exists (select channel_id from rhnChannelContentSource where source_id = cs.id) 
329          """) 
330          h.execute(prefix=constants.MANIFEST_REPOSITORY_DB_PREFIX) 
331          rhnSQL.commit() 

332   
333      @staticmethod 

334 -    def get_content_source_label(source): 

335          if 'pulp_repo_label_v2' in source: 
336              return source['pulp_repo_label_v2'] 
337          elif 'ks_tree_label' in source: 
338              return source['ks_tree_label'] 
339          else: 
340              raise InvalidContentSourceType() 

341   

342 -    def list_channels_containing_repository(self, relative_path): 

343          if relative_path in self.repository_to_channels: 
344              return self.repository_to_channels[relative_path] 
345          else: 
346              return [] 

347   

348   
349 -class CdnRepositoryTree(object): 

350      """Class representing activated CDN repositories in tree structure. 
351         Leafs contains CdnRepository instances. 
352         Allows us to match direct CDN URLs without variables (coming from mapping) 
353         to CDN URLs with variables (coming from manifest and having SSL keys/certs assigned)""" 
354   
355      VARIABLES = ['$releasever', '$basearch'] 
356   

357 -    def __init__(self): 

358          self.root = {} 

359   

360 -    def add_repository(self, repository): 

361          """Add new CdnRepository to tree.""" 
362   
363          url = repository.get_url() 
364          path = [x for x in url.split('/') if x] 
365          node = self.root 
366          for part in path[:-1]: 
367              if part not in node: 
368                  node[part] = {} 
369              node = node[part] 
370          # Save repository into leaf 
371          node[path[-1]] = repository 

372   

373 -    def _browse_node(self, node, keys): 

374          """Recursive function going through tree.""" 
375          # Return leaf 
376          is_leaf = not isinstance(node, dict) 
377          if is_leaf and not keys: 
378              return node 
379          elif (is_leaf and keys) or (not is_leaf and not keys): 
380              raise CdnRepositoryNotFoundError() 
381          step = keys[0] 
382          to_check = [x for x in node.keys() if x in self.VARIABLES or x == step] 
383          # Remove first step in path, create new list 
384          next_keys = keys[1:] 
385   
386          # Check all available paths 
387          for key in to_check: 
388              try: 
389                  # Try to get next node and run this function recursively 
390                  next_node = node[key] 
391                  return self._browse_node(next_node, next_keys) 
392              # From here 
393              except KeyError: 
394                  pass 
395              # From recurrent call 
396              except CdnRepositoryNotFoundError: 
397                  pass 
398   
399          raise CdnRepositoryNotFoundError() 

400   
401      @staticmethod 

402 -    def normalize_url(url): 

403          """Splits repository URL, removes redundant characters and returns list with directory names.""" 
404          path = [] 
405          for part in url.split('/'): 
406              if part == '..': 
407                  if path: 
408                      del path[-1] 
409                  else: 
410                      # Can't go upper in directory structure, keep it in path 
411                      path.append(part) 
412              elif part and part != '.': 
413                  path.append(part) 
414   
415          return path 

416   

417 -    def find_repository(self, url): 

418          """Finds matching repository in tree. 
419             url is relative CDN url - e.g. /content/dist/rhel/server/6/6Server/x86_64/os""" 
420          node = self.root 
421          try: 
422              path = self.normalize_url(url) 
423              found = self._browse_node(node, path) 
424          except CdnRepositoryNotFoundError: 
425              raise CdnRepositoryNotFoundError("ERROR: Repository '%s' was not found." % url) 
426   
427          return found 

428   

429   
430 -class CdnRepositoryNotFoundError(Exception): 

431      pass 

432   

433   
434 -class InvalidContentSourceType(Exception): 

435      pass 

436   

437   
438 -class CdnRepository(object): 

439      """Class representing CDN repository.""" 
440   

441 -    def __init__(self, label, url): 

442          self.label = label 
443          self.url = url 
444          self.ssl_sets = [] 

445   
446      # CdnRepositorySsl instance 

447 -    def add_ssl_set(self, ssl_set): 

448          self.ssl_sets.append(ssl_set) 

449   

450 -    def get_ssl_sets(self): 

451          return self.ssl_sets 

452   

453 -    def get_label(self): 

454          return self.label 

455   

456 -    def get_url(self): 

457          return self.url 

458   

459   
460 -class CdnRepositorySsl(object): 

461      """Class representing single SSL certificate, key set for single CDN repository""" 
462   

463 -    def __init__(self, ca_cert, client_cert, client_key): 

464          self.ca_cert = int(ca_cert) 
465          self.client_cert = int(client_cert) 
466          self.client_key = int(client_key) 

467   

468 -    def get_ca_cert(self): 

469          return self.ca_cert 

470   

471 -    def get_client_cert(self): 

472          return self.client_cert 

473   

474 -    def get_client_key(self): 

475          return self.client_key 

476   

477 -    def get_crypto_keys(self, check_dates=False): 

478          ssl_query = rhnSQL.prepare(""" 
479              select description, key, org_id from rhnCryptoKey where id = :id 
480          """) 
481          keys = {} 
482          ssl_query.execute(id=self.ca_cert) 
483          row = ssl_query.fetchone_dict() 
484          keys['ca_cert'] = (str(row['description']), str(row['key']), row['org_id']) 
485          ssl_query.execute(id=self.client_cert) 
486          row = ssl_query.fetchone_dict() 
487          keys['client_cert'] = (str(row['description']), str(row['key']), row['org_id']) 
488          ssl_query.execute(id=self.client_key) 
489          row = ssl_query.fetchone_dict() 
490          keys['client_key'] = (str(row['description']), str(row['key']), row['org_id']) 
491   
492          # Check if SSL certificates are usable 
493          if check_dates: 
494              failed = 0 
495              for key in (keys['ca_cert'], keys['client_cert']): 
496                  if not verify_certificate_dates(key[1]): 
497                      log(1, "WARNING: Problem with dates in certificate '%s'. " 
498                             "Please check validity of this certificate." % key[0]) 
499                      failed += 1 
500              if failed: 
501                  return {} 
502          return keys 

503

   Trees       Indices       Help   
Generated by Epydoc 3.0.1 on Wed Mar 4 07:37:43 2020 http://epydoc.sourceforge.net