proxy.broker.rhnRepository

1 # rhnRepository.py - Perform local repository functions. 2 #------------------------------------------------------------------------------- 3 # This module contains the functionality for providing local packages. 4 # 5 # Copyright (c) 2008--2017 Red Hat, Inc. 6 # 7 # This software is licensed to you under the GNU General Public License, 8 # version 2 (GPLv2). There is NO WARRANTY for this software, express or 9 # implied, including the implied warranties of MERCHANTABILITY or FITNESS 10 # FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 11 # along with this software; if not, see 12 # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. 13 # 14 # Red Hat trademarks are not licensed under GPLv2. No permission is 15 # granted to use or replicate Red Hat trademarks that are incorporated 16 # in this software or its documentation. 17 # 18 #------------------------------------------------------------------------------- 19 20 ## language imports 21 import os 22 import time 23 import glob 24 import cPickle 25 import sys 26 import types 27 from operator import truth 28 import xmlrpclib 29 30 ## common imports 31 from spacewalk.common.rhnLib import parseRPMName 32 from spacewalk.common.rhnLog import log_debug 33 from spacewalk.common.rhnException import rhnFault 34 from spacewalk.common.rhnConfig import CFG 35 from spacewalk.common import rhnRepository 36 from spacewalk.common.rhnTranslate import _ 37 38 ## local imports 39 from rhn import rpclib 40 41 42 PKG_LIST_DIR = os.path.join(CFG.PKG_DIR, 'list') 43 PREFIX = "rhn"

44 45 46 -class NotLocalError(Exception):

47 pass

48

49 50 -class Repository(rhnRepository.Repository):

51 # pylint: disable=R0902 52 53 """ Proxy local package repository lookup and manipulation code. """ 54

55 - def __init__(self, 56 channelName, channelVersion, clientInfo, 57 rhnParent=None, rhnParentXMLRPC=None, httpProxy=None, httpProxyUsername=None, 58 httpProxyPassword=None, caChain=None):

59 60 log_debug(3, channelName) 61 rhnRepository.Repository.__init__(self, channelName) 62 self.functions = CFG.PROXY_LOCAL_FLIST 63 self.channelName = channelName 64 self.channelVersion = channelVersion 65 self.clientInfo = clientInfo 66 self.rhnParent = rhnParent 67 self.rhnParentXMLRPC = rhnParentXMLRPC 68 self.httpProxy = httpProxy 69 self.httpProxyUsername = httpProxyUsername 70 self.httpProxyPassword = httpProxyPassword 71 self.caChain = caChain

72

73 - def getPackagePath(self, pkgFilename, redirect=0):

74 """ OVERLOADS getPackagePath in common/rhnRepository. 75 Returns complete path to an RPM file. 76 """ 77 78 log_debug(3, pkgFilename) 79 mappingName = "package_mapping:%s:" % self.channelName 80 mapping = self._cacheObj(mappingName, self.channelVersion, 81 self.__channelPackageMapping, ()) 82 83 # If the file name has parameters, it's a different kind of package. 84 # Determine the architecture requested so we can construct an 85 # appropriate filename. 86 if isinstance(pkgFilename, types.ListType): 87 arch = pkgFilename[3] 88 # Not certain if anything is needed here for Debian, but since what I've tested 89 # works. Leave it alone. 90 if isSolarisArch(arch): 91 pkgFilename = "%s-%s-%s.%s.pkg" % \ 92 (pkgFilename[0], 93 pkgFilename[1], 94 pkgFilename[2], 95 pkgFilename[3]) 96 97 if not mapping.has_key(pkgFilename): 98 log_debug(3, "Package not in mapping: %s" % pkgFilename) 99 raise NotLocalError 100 # A list of possible file paths. Always a list, channel mappings are 101 # cleared on package upgrade so we don't have to worry about the old 102 # behavior of returning a string 103 filePaths = mapping[pkgFilename] 104 # Can we see a file at any of the possible filepaths? 105 for filePath in filePaths: 106 filePath = "%s/%s" % (CFG.PKG_DIR, filePath) 107 log_debug(4, "File path", filePath) 108 if os.access(filePath, os.R_OK): 109 return filePath 110 log_debug(4, "Package not found locally: %s" % pkgFilename) 111 raise NotLocalError(filePaths[0], pkgFilename)

112

113 - def getSourcePackagePath(self, pkgFilename):

114 """ OVERLOADS getSourcePackagePath in common/rhnRepository. 115 snag src.rpm and nosrc.rpm from local repo, after ensuring 116 we are authorized to fetch it. 117 """ 118 119 log_debug(3, pkgFilename) 120 if pkgFilename[-8:] != '.src.rpm' and pkgFilename[-10:] != '.nosrc.rpm': 121 raise rhnFault(17, _("Invalid SRPM package requested: %s") 122 % pkgFilename) 123 124 # Connect to the server to get an authorization for downloading this 125 # package 126 server = rpclib.Server(self.rhnParentXMLRPC, proxy=self.httpProxy, 127 username=self.httpProxyUsername, 128 password=self.httpProxyPassword) 129 if self.caChain: 130 server.add_trusted_cert(self.caChain) 131 132 try: 133 retval = server.proxy.package_source_in_channel( 134 pkgFilename, self.channelName, self.clientInfo) 135 except xmlrpclib.Fault, e: 136 raise rhnFault(1000, 137 _("Error retrieving source package: %s") % str(e)), None, sys.exc_info()[2] 138 if not retval: 139 raise rhnFault(17, _("Invalid SRPM package requested: %s") 140 % pkgFilename) 141 142 if pkgFilename[-8:] != '.src.rpm': 143 # We already know the filename ends in .src.rpm 144 nvrea = list(parseRPMName(pkgFilename[:-8])) 145 nvrea.append("src") 146 else: 147 # We already know the filename ends in .nosrc.rpm 148 # otherwise we did not pass first if in this func 149 nvrea = list(parseRPMName(pkgFilename[:-10])) 150 nvrea.append("nosrc") 151 152 filePaths = computePackagePaths(nvrea, source=1, prepend=PREFIX) 153 for filePath in filePaths: 154 filePath = "%s/%s" % (CFG.PKG_DIR, filePath) 155 log_debug(4, "File path", filePath) 156 if os.access(filePath, os.R_OK): 157 return filePath 158 log_debug(4, "Source package not found locally: %s" % pkgFilename) 159 raise NotLocalError(filePaths[0], pkgFilename)

160

161 - def _cacheObj(self, fileName, version, dataProducer, params=None):

162 """ The real workhorse for all flavors of listall 163 It tries to pull data out of a file; if it doesn't work, 164 it calls the data producer with the specified params to generate 165 the data, which is also cached. 166 167 Returns a string from a cache file or, if the cache file is not 168 there, calls dataProducer to generate the object and caches the 169 results 170 """ 171 172 log_debug(4, fileName, version, params) 173 fileDir = self._getPkgListDir() 174 filePath = "%s/%s-%s" % (fileDir, fileName, version) 175 if os.access(filePath, os.R_OK): 176 try: 177 # Slurp the file 178 f = open(filePath, "r") 179 data = f.read() 180 f.close() 181 stringObject = cPickle.loads(data) 182 return stringObject 183 except (IOError, cPickle.UnpicklingError): # corrupted cache file 184 pass # do nothing, we'll fetch / write it again 185 186 # The file's not there; query the DB or whatever dataproducer used. 187 if params is None: 188 params = () 189 stringObject = dataProducer(*params) 190 # Cache the thing 191 cache(cPickle.dumps(stringObject, 1), fileDir, fileName, version) 192 # Return the string 193 return stringObject

194 195 @staticmethod

196 - def _getPkgListDir():

197 """ Creates and returns the directory for cached lists of packages. 198 Used by _cacheObj. 199 200 XXX: Problem exists here. If PKG_LIST_DIR can't be created 201 due to ownership... this is bad... need to fix. 202 """ 203 204 log_debug(3, PKG_LIST_DIR) 205 if not os.access(PKG_LIST_DIR, os.R_OK | os.X_OK): 206 os.makedirs(PKG_LIST_DIR) 207 return PKG_LIST_DIR

208

209 - def _listPackages(self):

210 """ Generates a list of objects by calling the function """ 211 server = rpclib.GETServer(self.rhnParentXMLRPC, proxy=self.httpProxy, 212 username=self.httpProxyUsername, password=self.httpProxyPassword, 213 headers=self.clientInfo) 214 if self.caChain: 215 server.add_trusted_cert(self.caChain) 216 return server.listAllPackagesChecksum(self.channelName, 217 self.channelVersion)

218

219 - def __channelPackageMapping(self):

220 """ fetch package list on behalf of the client """ 221 222 log_debug(6, self.rhnParentXMLRPC, self.httpProxy, self.httpProxyUsername, self.httpProxyPassword) 223 log_debug(6, self.clientInfo) 224 225 try: 226 packageList = self._listPackages() 227 except xmlrpclib.ProtocolError, e: 228 errcode, errmsg = rpclib.reportError(e.headers) 229 raise rhnFault(1000, "SpacewalkProxy error (xmlrpclib.ProtocolError): " 230 "errode=%s; errmsg=%s" % (errcode, errmsg)), None, sys.exc_info()[2] 231 232 # Hash the list 233 _hash = {} 234 for package in packageList: 235 arch = package[4] 236 237 extension = "rpm" 238 if isSolarisArch(arch): 239 extension = "pkg" 240 if isDebianArch(arch): 241 extension = "deb" 242 243 filename = "%s-%s-%s.%s.%s" % (package[0], package[1], 244 package[2], package[4], extension) 245 # if the package contains checksum info 246 if len(package) > 6: 247 filePaths = computePackagePaths(package, source=0, 248 prepend=PREFIX, checksum=package[7]) 249 else: 250 filePaths = computePackagePaths(package, source=0, 251 prepend=PREFIX) 252 _hash[filename] = filePaths 253 254 if CFG.DEBUG > 4: 255 log_debug(5, "Mapping: %s[...snip snip...]%s" % (str(_hash)[:40], str(_hash)[-40:])) 256 return _hash

257

258 259 -class KickstartRepository(Repository):

260 261 """ Kickstarts always end up pointing to a channel that they're getting 262 rpms from. Lookup what channel that is and then just use the regular 263 repository """ 264

265 - def __init__(self, kickstart, clientInfo, rhnParent=None, 266 rhnParentXMLRPC=None, httpProxy=None, httpProxyUsername=None, 267 httpProxyPassword=None, caChain=None, orgId=None, child=None, 268 session=None, systemId=None):

269 log_debug(3, kickstart) 270 271 self.systemId = systemId 272 self.kickstart = kickstart 273 self.ks_orgId = orgId 274 self.ks_child = child 275 self.ks_session = session 276 277 # have to look up channel name and version for this kickstart 278 # we have no equievanet to the channel version for kickstarts, 279 # expire the cache after an hour 280 fileName = "kickstart_mapping:%s-%s-%s-%s:" % (str(kickstart), 281 str(orgId), str(child), str(session)) 282 283 mapping = self._lookupKickstart(fileName, rhnParentXMLRPC, httpProxy, 284 httpProxyUsername, httpProxyPassword, caChain) 285 Repository.__init__(self, mapping['channel'], mapping['version'], 286 clientInfo, rhnParent, rhnParentXMLRPC, httpProxy, 287 httpProxyUsername, httpProxyPassword, caChain)

288

289 - def _lookupKickstart(self, fileName, rhnParentXMLRPC, httpProxy, 290 httpProxyUsername, httpProxyPassword, caChain):

291 fileDir = self._getPkgListDir() 292 filePath = "%s/%s-1" % (fileDir, fileName) 293 mapping = None 294 if os.access(filePath, os.R_OK): 295 try: 296 # Slurp the file 297 f = open(filePath, "r") 298 mapping = cPickle.loads(f.read()) 299 f.close() 300 except (IOError, cPickle.UnpicklingError): # corrupt cached file 301 mapping = None # ignore it, we'll get and write it again 302 303 now = int(time.time()) 304 if not mapping or mapping['expires'] < now: 305 # Can't use the normal GETServer handler because there is no client 306 # to auth. Instead this is something the Proxy has to be able to 307 # do, so read the serverid and send that up. 308 server = rpclib.Server(rhnParentXMLRPC, proxy=httpProxy, 309 username=httpProxyUsername, password=httpProxyPassword) 310 if caChain: 311 server.add_trusted_cert(caChain) 312 try: 313 response = self._getMapping(server) 314 mapping = {'channel': str(response['label']), 315 'version': str(response['last_modified']), 316 'expires': int(time.time()) + 3600} # 1 hour from now 317 except Exception: 318 # something went wrong. Punt, we just won't serve this request 319 # locally 320 raise NotLocalError 321 322 # Cache the thing 323 cache(cPickle.dumps(mapping, 1), fileDir, fileName, "1") 324 325 return mapping

326

327 - def _listPackages(self):

328 """ Generates a list of objects by calling the function""" 329 # Can't use the normal GETServer handler because there is no client 330 # to auth. Instead this is something the Proxy has to be able to do, 331 # so read the serverid and send that up. 332 server = rpclib.Server(self.rhnParentXMLRPC, proxy=self.httpProxy, 333 username=self.httpProxyUsername, password=self.httpProxyPassword) 334 if self.caChain: 335 server.add_trusted_cert(self.caChain) 336 # Versionless package listing from Server. This saves us from erroring 337 # unnecessarily if the channel has changed since the kickstart mapping. 338 # No problem, newer channel listings will work fine with kickstarts 339 # unless they have removed the kernel or something, in which case it's 340 # not supposed to work. 341 # Worst case scenario is that we cache listing using an older version 342 # than it actually is, and the next time we serve a file from the 343 # regular Repository it'll get replace with the same info but newer 344 # version in filename. 345 return server.proxy.listAllPackagesKickstart(self.channelName, 346 self.systemId)

347

348 - def _getMapping(self, server):

349 """ Generate a hash that tells us what channel this 350 kickstart is looking at. We have no equivalent to channel version, 351 so expire the cached file after an hour.""" 352 if self.ks_orgId: 353 return server.proxy.getKickstartOrgChannel(self.kickstart, 354 self.ks_orgId, self.systemId) 355 if self.ks_session: 356 return server.proxy.getKickstartSessionChannel(self.kickstart, 357 self.ks_session, self.systemId) 358 if self.ks_child: 359 return server.proxy.getKickstartChildChannel(self.kickstart, 360 self.ks_child, self.systemId) 361 return server.proxy.getKickstartChannel(self.kickstart, self.systemId)

362

363 364 -class TinyUrlRepository(KickstartRepository):

365 # pylint: disable=W0233,W0231 366 367 """ TinyURL kickstarts have actually already made a HEAD request up to the 368 Satellite to to get the checksum for the rpm, however we can't just use 369 that data because the epoch information is not in the filename so we'd 370 never find files with a non-None epoch. Instead do the same thing we do 371 for non-tiny-urlified kickstarts and look up what channel it maps to.""" 372

373 - def __init__(self, tinyurl, clientInfo, rhnParent=None, 374 rhnParentXMLRPC=None, httpProxy=None, httpProxyUsername=None, 375 httpProxyPassword=None, caChain=None, systemId=None):

376 log_debug(3, tinyurl) 377 378 self.systemId = systemId 379 self.tinyurl = tinyurl 380 381 # have to look up channel name and version for this kickstart 382 # we have no equievanet to the channel version for kickstarts, 383 # expire the cache after an hour 384 fileName = "tinyurl_mapping:%s:" % (str(tinyurl)) 385 386 mapping = self._lookupKickstart(fileName, rhnParentXMLRPC, httpProxy, 387 httpProxyUsername, httpProxyPassword, caChain) 388 Repository.__init__(self, mapping['channel'], mapping['version'], 389 clientInfo, rhnParent, rhnParentXMLRPC, httpProxy, 390 httpProxyUsername, httpProxyPassword, caChain)

391

392 - def _getMapping(self, server):

393 return server.proxy.getTinyUrlChannel(self.tinyurl, self.systemId)

394

395 396 -def isSolarisArch(arch):

397 """ 398 Returns true if the given arch string represents a solaris architecture. 399 """ 400 return arch.find("solaris") != -1

401

402 403 -def isDebianArch(arch):

404 """ 405 Returns true if the given arch string represents a Debian architecture.. 406 """ 407 return arch[-4:] == "-deb"

408

409 410 -def computePackagePaths(nvrea, source=0, prepend="", checksum=None):

411 """ Finds the appropriate paths, prepending something if necessary """ 412 paths = [] 413 name = nvrea[0] 414 release = nvrea[2] 415 416 if source: 417 dirarch = 'SRPMS' 418 pkgarch = 'src' 419 else: 420 dirarch = pkgarch = nvrea[4] 421 422 extension = "rpm" 423 if isSolarisArch(pkgarch): 424 extension = "pkg" 425 if isDebianArch(pkgarch): 426 extension = "deb" 427 428 version = nvrea[1] 429 epoch = nvrea[3] 430 if epoch not in [None, '']: 431 version = str(epoch) + ':' + version 432 # The new prefered path template avoides collisions if packages with the 433 # same nevra but different checksums are uploaded. It also should be the 434 # same as the /var/satellite/redhat/NULL/* paths upstream. 435 # We can't reliably look up the checksum for source packages, so don't 436 # use it in the source path. 437 if checksum and not source: 438 checksum_template = prepend + "/%s/%s/%s-%s/%s/%s/%s-%s-%s.%s.%s" 439 checksum_template = '/'.join(filter(truth, checksum_template.split('/'))) 440 paths.append(checksum_template % (checksum[:3], name, version, release, 441 dirarch, checksum, name, nvrea[1], release, pkgarch, extension)) 442 template = prepend + "/%s/%s-%s/%s/%s-%s-%s.%s.%s" 443 # Sanitize the path: remove duplicated / 444 template = '/'.join(filter(truth, template.split('/'))) 445 paths.append(template % (name, version, release, dirarch, name, nvrea[1], 446 release, pkgarch, extension)) 447 return paths

448

449 450 -def cache(stringObject, directory, filename, version):

451 """ Caches stringObject into a file and removes older files """ 452 453 # The directory should be readable, writable, seekable 454 if not os.access(directory, os.R_OK | os.W_OK | os.X_OK): 455 os.makedirs(directory) 456 filePath = "%s/%s-%s" % (directory, filename, version) 457 # Create a temp file based on the filename, version and stuff 458 tempfile = "%s-%.20f" % (filePath, time.time()) 459 # Try to create the temp file 460 tries = 10 461 while tries > 0: 462 # Try to create this new file 463 try: 464 fd = os.open(tempfile, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 465 0644) 466 except OSError, e: 467 if e.errno == 17: 468 # File exists; give it another try 469 tries = tries - 1 470 tempfile = tempfile + "%.20f" % time.time() 471 continue 472 # Another error 473 raise 474 else: 475 # We've got the file; everything's nice and dandy 476 break 477 else: 478 # Could not create the file 479 raise Exception("Could not create the file") 480 # Write the object into the cache 481 os.write(fd, stringObject) 482 os.close(fd) 483 # Now rename the temp file 484 os.rename(tempfile, filePath) 485 # Expire the cached copies 486 _list = glob.glob("%s/%s-*" % (directory, filename)) 487 for _file in _list: 488 if _file < filePath: 489 # Older than this 490 os.unlink(_file)

491

Source Code for Module proxy.broker.rhnRepository