backend.satellite_tools.repo_plugins.yum

1 # 2 # Copyright (c) 2008--2020 Red Hat, Inc. 3 # Copyright (c) 2010--2011 SUSE LINUX Products GmbH, Nuernberg, Germany. 4 # 5 # This software is licensed to you under the GNU General Public License, 6 # version 2 (GPLv2). There is NO WARRANTY for this software, express or 7 # implied, including the implied warranties of MERCHANTABILITY or FITNESS 8 # FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 9 # along with this software; if not, see 10 # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. 11 # 12 # Red Hat trademarks are not licensed under GPLv2. No permission is 13 # granted to use or replicate Red Hat trademarks that are incorporated 14 # in this software or its documentation. 15 # 16 17 import sys 18 import logging 19 import os.path 20 from os import makedirs 21 from shutil import rmtree 22 23 import yum 24 from yum.Errors import RepoMDError 25 from yum.comps import Comps 26 from yum.config import ConfigParser 27 from yum.packageSack import ListPackageSack 28 from yum.update_md import UpdateMetadata, UpdateNoticeException, UpdateNotice 29 from yum.yumRepo import YumRepository 30 from yum.yumRepo import Errors as YumErrors 31 try: 32 from yum.misc import cElementTree_iterparse as iterparse 33 except ImportError: 34 try: 35 from xml.etree import cElementTree 36 except ImportError: 37 # pylint: disable=F0401 38 import cElementTree 39 iterparse = cElementTree.iterparse 40 from urlgrabber.grabber import URLGrabError 41 try: 42 # python 2 43 import urlparse 44 except ImportError: 45 # python3 46 import urllib.parse as urlparse # pylint: disable=F0401,E0611 47 48 from spacewalk.common import fileutils, checksum 49 from spacewalk.satellite_tools.download import get_proxies 50 from spacewalk.satellite_tools.repo_plugins import ContentPackage, CACHE_DIR 51 from spacewalk.common.rhnConfig import CFG, initCFG 52 53 YUMSRC_CONF = '/etc/rhn/spacewalk-repo-sync/yum.conf'

54 55 56 -class YumWarnings:

57

58 - def __init__(self):

59 self.saved_stdout = None 60 self.errors = None

61

62 - def write(self, s):

63 pass

64

65 - def disable(self):

66 self.saved_stdout = sys.stdout 67 sys.stdout = self

68

69 - def restore(self):

70 sys.stdout = self.saved_stdout

71

72 73 -class YumUpdateMetadata(UpdateMetadata):

74 75 """The root update metadata object supports getting all updates""" 76 77 # pylint: disable=W0221

78 - def add(self, obj, mdtype='updateinfo', all_versions=False):

79 """ Parse a metadata from a given YumRepository, file, or filename. """ 80 if not obj: 81 raise UpdateNoticeException 82 if isinstance(obj, (type(''), type(u''))): 83 infile = fileutils.decompress_open(obj) 84 elif isinstance(obj, YumRepository): 85 if obj.id not in self._repos: 86 self._repos.append(obj.id) 87 md = obj.retrieveMD(mdtype) 88 if not md: 89 raise UpdateNoticeException() 90 infile = fileutils.decompress_open(md) 91 else: # obj is a file object 92 infile = obj 93 94 for _event, elem in iterparse(infile): 95 if elem.tag == 'update': 96 un = UpdateNotice(elem) 97 key = un['update_id'] 98 if all_versions: 99 key = "%s-%s" % (un['update_id'], un['version']) 100 if key not in self._notices: 101 self._notices[key] = un 102 for pkg in un['pkglist']: 103 for pkgfile in pkg['packages']: 104 self._cache['%s-%s-%s' % (pkgfile['name'], 105 pkgfile['version'], 106 pkgfile['release'])] = un 107 no = self._no_cache.setdefault(pkgfile['name'], set()) 108 no.add(un)

109

110 111 -class RepoMDNotFound(Exception):

112 pass

113

114 115 -class ContentSource(object):

116

117 - def __init__(self, url, name, yumsrc_conf=YUMSRC_CONF, org="1", channel_label="", 118 no_mirrors=False, ca_cert_file=None, client_cert_file=None, 119 client_key_file=None):

120 self.url = url 121 self.name = name 122 self.yumbase = yum.YumBase() 123 self.yumbase.preconf.fn = yumsrc_conf 124 if not os.path.exists(yumsrc_conf): 125 self.yumbase.preconf.fn = '/dev/null' 126 self.configparser = ConfigParser() 127 if org: 128 self.org = org 129 else: 130 self.org = "NULL" 131 132 self.proxy_addr = None 133 self.proxy_user = None 134 self.proxy_pass = None 135 self.authtoken = None 136 137 # read the proxy configuration 138 # /etc/rhn/rhn.conf has more priority than yum.conf 139 initCFG('server.satellite') 140 141 # keep authtokens for mirroring 142 (_scheme, _netloc, _path, query, _fragid) = urlparse.urlsplit(url) 143 if query: 144 self.authtoken = query 145 146 if CFG.http_proxy: 147 self.proxy_addr = CFG.http_proxy 148 self.proxy_user = CFG.http_proxy_username 149 self.proxy_pass = CFG.http_proxy_password 150 else: 151 yb_cfg = self.yumbase.conf.cfg 152 section_name = None 153 154 if yb_cfg.has_section(self.name): 155 section_name = self.name 156 elif yb_cfg.has_section(channel_label): 157 section_name = channel_label 158 elif yb_cfg.has_section('main'): 159 section_name = 'main' 160 161 if section_name: 162 if yb_cfg.has_option(section_name, option='proxy'): 163 self.proxy_addr = yb_cfg.get(section_name, option='proxy') 164 165 if yb_cfg.has_option(section_name, 'proxy_username'): 166 self.proxy_user = yb_cfg.get(section_name, 'proxy_username') 167 168 if yb_cfg.has_option(section_name, 'proxy_password'): 169 self.proxy_pass = yb_cfg.get(section_name, 'proxy_password') 170 171 self._authenticate(url) 172 173 # Check for settings in yum configuration files (for custom repos/channels only) 174 if org: 175 repos = self.yumbase.repos.repos 176 else: 177 repos = None 178 if repos and name in repos: 179 repo = repos[name] 180 elif repos and channel_label in repos: 181 repo = repos[channel_label] 182 # In case we are using Repo object based on channel config, override it's id to name of the repo 183 # To not create channel directories in cache directory 184 repo.id = name 185 else: 186 # Not using values from config files 187 repo = yum.yumRepo.YumRepository(name) 188 repo.populate(self.configparser, name, self.yumbase.conf) 189 self.repo = repo 190 191 self.setup_repo(repo, no_mirrors, ca_cert_file, client_cert_file, client_key_file) 192 self.num_packages = 0 193 self.num_excluded = 0 194 self.groupsfile = None

195

196 - def __del__(self):

197 # close log files for yum plugin 198 for handler in logging.getLogger("yum.filelogging").handlers: 199 handler.close() 200 self.repo.close()

201

202 - def _authenticate(self, url):

203 pass

204 205 @staticmethod

206 - def interrupt_callback(*args, **kwargs): # pylint: disable=W0613

207 # Just re-raise 208 e = sys.exc_info()[1] 209 raise e

210

211 - def setup_repo(self, repo, no_mirrors, ca_cert_file, client_cert_file, client_key_file):

212 """Fetch repository metadata""" 213 repo.cache = 0 214 repo.mirrorlist = self.url 215 repo.baseurl = [self.url] 216 repo.basecachedir = os.path.join(CACHE_DIR, self.org) 217 # base_persistdir have to be set before pkgdir 218 if hasattr(repo, 'base_persistdir'): 219 repo.base_persistdir = repo.basecachedir 220 221 pkgdir = os.path.join(CFG.MOUNT_POINT, CFG.PREPENDED_DIR, self.org, 'stage') 222 if not os.path.isdir(pkgdir): 223 fileutils.makedirs(pkgdir, user='apache', group='apache') 224 repo.pkgdir = pkgdir 225 repo.sslcacert = ca_cert_file 226 repo.sslclientcert = client_cert_file 227 repo.sslclientkey = client_key_file 228 repo.proxy = None 229 repo.proxy_username = None 230 repo.proxy_password = None 231 232 if "file://" in self.url: 233 repo.copy_local = 1 234 235 if self.proxy_addr: 236 repo.proxy = self.proxy_addr if '://' in self.proxy_addr else 'http://' + self.proxy_addr 237 repo.proxy_username = self.proxy_user 238 repo.proxy_password = self.proxy_pass 239 240 # Do not try to expand baseurl to other mirrors 241 if no_mirrors: 242 repo.urls = repo.baseurl 243 # Make sure baseurl ends with / and urljoin will work correctly 244 if repo.urls[0][-1] != '/': 245 repo.urls[0] += '/' 246 else: 247 warnings = YumWarnings() 248 warnings.disable() 249 try: 250 repo.baseurlSetup() 251 except: 252 warnings.restore() 253 raise 254 warnings.restore() 255 # if self.url is metalink it will be expanded into 256 # real urls in repo.urls and also save this metalink 257 # in begin of the url list ("for repolist -v ... or anything else wants to know the baseurl") 258 # Remove it from the list, we don't need it to download content of repo 259 repo.urls = [url for url in repo.urls if '?' not in url] 260 repo.interrupt_callback = self.interrupt_callback 261 repo.setup(0)

262

263 - def number_of_packages(self):

264 for dummy_index in range(3): 265 try: 266 self.repo.getPackageSack().populate(self.repo, 'metadata', None, 0) 267 break 268 except YumErrors.RepoError: 269 pass 270 return len(self.repo.getPackageSack().returnPackages())

271

272 - def raw_list_packages(self, filters=None):

273 for dummy_index in range(3): 274 try: 275 self.repo.getPackageSack().populate(self.repo, 'metadata', None, 0) 276 break 277 except YumErrors.RepoError: 278 pass 279 280 rawpkglist = self.repo.getPackageSack().returnPackages() 281 self.num_packages = len(rawpkglist) 282 283 if not filters: 284 filters = [] 285 # if there's no include/exclude filter on command line or in database 286 for p in self.repo.includepkgs: 287 filters.append(('+', [p])) 288 for p in self.repo.exclude: 289 filters.append(('-', [p])) 290 291 if filters: 292 rawpkglist = self._filter_packages(rawpkglist, filters) 293 rawpkglist = self._get_package_dependencies(self.repo.getPackageSack(), rawpkglist) 294 295 self.num_excluded = self.num_packages - len(rawpkglist) 296 297 return rawpkglist

298

299 - def list_packages(self, filters, latest):

300 """ list packages""" 301 self.repo.getPackageSack().populate(self.repo, 'metadata', None, 0) 302 pkglist = ListPackageSack(self.repo.getPackageSack().returnPackages()) 303 self.num_packages = len(pkglist) 304 if latest: 305 pkglist = pkglist.returnNewestByNameArch() 306 pkglist = yum.misc.unique(pkglist) 307 pkglist.sort(self._sort_packages) 308 309 if not filters: 310 # if there's no include/exclude filter on command line or in database 311 # check repository config file 312 for p in self.repo.includepkgs: 313 filters.append(('+', [p])) 314 for p in self.repo.exclude: 315 filters.append(('-', [p])) 316 317 filters = self._expand_package_groups(filters) 318 319 if filters: 320 pkglist = self._filter_packages(pkglist, filters) 321 pkglist = self._get_package_dependencies(self.repo.getPackageSack(), pkglist) 322 323 self.num_excluded = self.num_packages - len(pkglist) 324 to_return = [] 325 for pack in pkglist: 326 if pack.arch == 'src': 327 continue 328 new_pack = ContentPackage() 329 new_pack.setNVREA(pack.name, pack.version, pack.release, 330 pack.epoch, pack.arch) 331 new_pack.unique_id = pack 332 new_pack.checksum_type = pack.checksums[0][0] 333 if new_pack.checksum_type == 'sha': 334 new_pack.checksum_type = 'sha1' 335 new_pack.checksum = pack.checksums[0][1] 336 to_return.append(new_pack) 337 return to_return

338 339 @staticmethod

340 - def _sort_packages(pkg1 ,pkg2):

341 """sorts a list of yum package objects by name""" 342 if pkg1.name > pkg2.name: 343 return 1 344 elif pkg1.name == pkg2.name: 345 return 0 346 else: 347 return -1

348 349 @staticmethod

350 - def _find_comps_type(comps_type, environments, groups, name):

351 # Finds environment or regular group by name or label 352 found = None 353 if comps_type == "environment": 354 for e in environments: 355 if e.environmentid == name or e.name == name: 356 found = e 357 break 358 elif comps_type == "group": 359 for g in groups: 360 if g.groupid == name or g.name == name: 361 found = g 362 break 363 return found

364

365 - def _expand_comps_type(self, comps_type, environments, groups, filters):

366 new_filters = [] 367 # Rebuild filter list 368 for sense, pkg_list in filters: 369 new_pkg_list = [] 370 for pkg in pkg_list: 371 # Package group id 372 if pkg and pkg[0] == '@': 373 group_name = pkg[1:].strip() 374 found = self._find_comps_type(comps_type, environments, groups, group_name) 375 if found and comps_type == "environment": 376 # Save expanded groups to the package list 377 new_pkg_list.extend(['@' + grp for grp in found.allgroups]) 378 elif found and comps_type == "group": 379 # Replace with package list, simplified to not evaluate if packages are default, optional etc. 380 new_pkg_list.extend(found.packages) 381 else: 382 # Invalid group, save group id back 383 new_pkg_list.append(pkg) 384 else: 385 # Regular package 386 new_pkg_list.append(pkg) 387 if new_pkg_list: 388 new_filters.append((sense, new_pkg_list)) 389 return new_filters

390

391 - def _expand_package_groups(self, filters):

392 if not self.groupsfile: 393 return filters 394 comps = Comps() 395 comps.add(self.groupsfile) 396 groups = comps.get_groups() 397 398 if hasattr(comps, 'get_environments'): 399 # First expand environment groups, then regular groups 400 environments = comps.get_environments() 401 filters = self._expand_comps_type("environment", environments, groups, filters) 402 else: 403 environments = [] 404 filters = self._expand_comps_type("group", environments, groups, filters) 405 return filters

406 407 @staticmethod

408 - def _filter_packages(packages, filters):

409 """ implement include / exclude logic 410 filters are: [ ('+', includelist1), ('-', excludelist1), 411 ('+', includelist2), ... ] 412 """ 413 if filters is None: 414 return [] 415 416 selected = [] 417 excluded = [] 418 if filters[0][0] == '-': 419 # first filter is exclude, start with full package list 420 # and then exclude from it 421 selected = packages 422 else: 423 excluded = packages 424 425 for filter_item in filters: 426 sense, pkg_list = filter_item 427 if sense == '+': 428 # include 429 exactmatch, matched, _unmatched = yum.packages.parsePackages( 430 excluded, pkg_list) 431 allmatched = yum.misc.unique(exactmatch + matched) 432 selected = yum.misc.unique(selected + allmatched) 433 for pkg in allmatched: 434 if pkg in excluded: 435 excluded.remove(pkg) 436 elif sense == '-': 437 # exclude 438 exactmatch, matched, _unmatched = yum.packages.parsePackages( 439 selected, pkg_list) 440 allmatched = yum.misc.unique(exactmatch + matched) 441 for pkg in allmatched: 442 if pkg in selected: 443 selected.remove(pkg) 444 excluded = yum.misc.unique(excluded + allmatched) 445 else: 446 raise UpdateNoticeException("Invalid filter sense: '%s'" % sense) 447 return selected

448

449 - def _get_package_dependencies(self, sack, packages):

450 self.yumbase.pkgSack = sack 451 known_deps = set() 452 resolved_deps = self.yumbase.findDeps(packages) 453 while resolved_deps: 454 next_level_deps = [] 455 for deps in resolved_deps.values(): 456 for _dep, dep_packages in deps.items(): 457 if _dep not in known_deps: 458 next_level_deps.extend(dep_packages) 459 packages.extend(dep_packages) 460 known_deps.add(_dep) 461 462 resolved_deps = self.yumbase.findDeps(next_level_deps) 463 464 return yum.misc.unique(packages)

465

466 - def get_package(self, package, metadata_only=False):

467 """ get package """ 468 pack = package.unique_id 469 check = (self.verify_pkg, (pack, 1), {}) 470 if metadata_only: 471 # Include also data before header section 472 pack.hdrstart = 0 473 data = self.repo.getHeader(pack, checkfunc=check) 474 else: 475 data = self.repo.getPackage(pack, checkfunc=check) 476 return data

477 478 @staticmethod

479 - def verify_pkg(_fo, pkg, _fail):

480 return pkg.verifyLocalPkg()

481

482 - def clear_cache(self, directory=None, keep_repomd=False):

483 if directory is None: 484 directory = os.path.join(CACHE_DIR, self.org, self.name) 485 486 # remove content in directory 487 for item in os.listdir(directory): 488 path = os.path.join(directory, item) 489 if os.path.isfile(path) and not (keep_repomd and item == "repomd.xml"): 490 os.unlink(path) 491 elif os.path.isdir(path): 492 rmtree(path) 493 494 # restore empty directories 495 makedirs(directory + "/packages", int('0755', 8)) 496 makedirs(directory + "/gen", int('0755', 8))

497

498 - def get_updates(self):

499 if 'updateinfo' not in self.repo.repoXML.repoData: 500 return [] 501 um = YumUpdateMetadata() 502 um.add(self.repo, all_versions=True) 503 return um.notices

504

505 - def get_groups(self):

506 try: 507 groups = self.repo.getGroups() 508 except RepoMDError: 509 groups = None 510 return groups

511

512 - def get_modules(self):

513 try: 514 modules = self.repo.retrieveMD('modules') 515 except RepoMDError: 516 modules = None 517 return modules

518

519 - def get_file(self, path, local_base=None):

520 try: 521 try: 522 temp_file = "" 523 if local_base is not None: 524 target_file = os.path.join(local_base, path) 525 target_dir = os.path.dirname(target_file) 526 if not os.path.exists(target_dir): 527 os.makedirs(target_dir, int('0755', 8)) 528 temp_file = target_file + '..download' 529 if os.path.exists(temp_file): 530 os.unlink(temp_file) 531 downloaded = self.repo.grab.urlgrab(path, temp_file) 532 os.rename(downloaded, target_file) 533 return target_file 534 else: 535 return self.repo.grab.urlread(path) 536 except URLGrabError: 537 return None 538 finally: 539 if os.path.exists(temp_file): 540 os.unlink(temp_file) 541 return None

542

543 - def repomd_up_to_date(self):

544 repomd_old_path = os.path.join(self.repo.basecachedir, self.name, "repomd.xml") 545 # No cached repomd? 546 if not os.path.isfile(repomd_old_path): 547 return False 548 repomd_new_path = os.path.join(self.repo.basecachedir, self.name, "repomd.xml.new") 549 # Newer file not available? Don't do anything. It should be downloaded before this. 550 if not os.path.isfile(repomd_new_path): 551 return True 552 return (checksum.getFileChecksum('sha256', filename=repomd_old_path) == 553 checksum.getFileChecksum('sha256', filename=repomd_new_path))

554 555 # Get download parameters for threaded downloader

556 - def set_download_parameters(self, params, relative_path, target_file, checksum_type=None, checksum_value=None, 557 bytes_range=None):

558 # Create directories if needed 559 target_dir = os.path.dirname(target_file) 560 if not os.path.exists(target_dir): 561 os.makedirs(target_dir, int('0755', 8)) 562 563 params['urls'] = self.repo.urls 564 params['relative_path'] = relative_path 565 params['authtoken'] = self.authtoken 566 params['target_file'] = target_file 567 params['ssl_ca_cert'] = self.repo.sslcacert 568 params['ssl_client_cert'] = self.repo.sslclientcert 569 params['ssl_client_key'] = self.repo.sslclientkey 570 params['checksum_type'] = checksum_type 571 params['checksum'] = checksum_value 572 params['bytes_range'] = bytes_range 573 params['proxy'] = self.repo.proxy 574 params['proxy_username'] = self.repo.proxy_username 575 params['proxy_password'] = self.repo.proxy_password 576 params['http_headers'] = self.repo.http_headers 577 # Older urlgrabber compatibility 578 params['proxies'] = get_proxies(self.repo.proxy, self.repo.proxy_username, self.repo.proxy_password)

579 580 # Simply load primary and updateinfo path from repomd

581 - def get_metadata_paths(self):

582 def get_location(data_item): 583 for sub_item in data_item: 584 if sub_item.tag.endswith("location"): 585 return sub_item.attrib.get("href") 586 return None

587 588 def get_checksum(data_item): 589 for sub_item in data_item: 590 if sub_item.tag.endswith("checksum"): 591 return sub_item.attrib.get("type"), sub_item.text 592 return None 593 594 repomd_path = os.path.join(self.repo.basecachedir, self.name, "repomd.xml") 595 if not os.path.isfile(repomd_path): 596 raise RepoMDNotFound(repomd_path) 597 repomd = open(repomd_path, 'rb') 598 files = {} 599 for _event, elem in iterparse(repomd): 600 if elem.tag.endswith("data"): 601 if elem.attrib.get("type") == "primary_db": 602 files['primary'] = (get_location(elem), get_checksum(elem)) 603 elif elem.attrib.get("type") == "primary" and 'primary' not in files: 604 files['primary'] = (get_location(elem), get_checksum(elem)) 605 elif elem.attrib.get("type") == "updateinfo": 606 files['updateinfo'] = (get_location(elem), get_checksum(elem)) 607 elif elem.attrib.get("type") == "group_gz": 608 files['group'] = (get_location(elem), get_checksum(elem)) 609 elif elem.attrib.get("type") == "group" and 'group' not in files: 610 files['group'] = (get_location(elem), get_checksum(elem)) 611 elif elem.attrib.get("type") == "modules": 612 files['modules'] = (get_location(elem), get_checksum(elem)) 613 repomd.close() 614 return files.values() 615

Source Code for Module backend.satellite_tools.repo_plugins.yum_src