rhnpush.uploadLib

1 # 2 # Copyright (c) 2008--2018 Red Hat, Inc. 3 # 4 # This software is licensed to you under the GNU General Public License, 5 # version 2 (GPLv2). There is NO WARRANTY for this software, express or 6 # implied, including the implied warranties of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 8 # along with this software; if not, see 9 # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. 10 # 11 # Red Hat trademarks are not licensed under GPLv2. No permission is 12 # granted to use or replicate Red Hat trademarks that are incorporated 13 # in this software or its documentation. 14 # 15 16 # system imports 17 import os 18 import sys 19 import fnmatch 20 import getpass 21 22 # imports 23 # pylint: disable=F0401,E0611 24 25 # exceptions 26 # pylint: disable=W0702,W0703 27 28 import inspect 29 from rhnpush import rhnpush_cache 30 from rhn.i18n import sstr 31 from up2date_client import rhnserver 32 from spacewalk.common import rhn_mpm 33 from spacewalk.common import rhn_rpm 34 from spacewalk.common.rhn_pkg import package_from_filename, get_package_header 35 from spacewalk.common.usix import raise_with_tb 36 37 if sys.version_info[0] == 3: 38 import xmlrpc.client as xmlrpclib 39 else: 40 import xmlrpclib 41 42 try: 43 from rhn import rpclib # pylint: disable=C0412 44 Binary = rpclib.xmlrpclib.Binary 45 Output = rpclib.transports.Output 46 except ImportError: 47 # old-style xmlrpclib library 48 rpclib = xmlrpclib 49 Binary = rpclib.Binary 50 # pylint: disable=F0401 51 import cgiwrap 52 Output = cgiwrap.Output 53 54 # Buffer size we use for copying 55 BUFFER_SIZE = 65536 56 HEADERS_PER_CALL = 25

57 58 # Exception class 59 60 61 -class UploadError(Exception):

62 pass

63

64 65 -class ServerFault(Exception):

66

67 - def __init__(self, faultCode=None, faultString="", faultExplanation=""):

68 Exception.__init__(self) 69 self.faultCode = faultCode 70 self.faultString = faultString 71 self.faultExplanation = faultExplanation

72

73 74 -class UploadClass:

75 76 """Functionality for an uploading tool 77 """ 78

79 - def __init__(self, options, files=None):

80 self.options = options 81 self.username = None 82 self.password = None 83 self.proxy = None 84 self.proxyUsername = None 85 self.proxyPassword = None 86 self.ca_chain = None 87 self.force = None 88 self.files = files or [] 89 self.new_sat = None 90 self.url = None 91 self.channels = None 92 self.count = None 93 self.server = None 94 self.session = None 95 self.orgId = None 96 self.relativeDir = None 97 self.use_session = True 98 self.use_checksum_paths = False

99

100 - def warn(self, verbose, *args):

101 if self.options.verbose >= verbose: 102 ReportError(*args)

103 104 @staticmethod

105 - def die(errcode, *args):

106 ReportError(*args) 107 # pkilambi:bug#176358:this should exit with error code 108 sys.exit(errcode)

109

110 - def setURL(self):

111 # Redefine this in derived classes 112 self.url = None

113

114 - def setUsernamePassword(self):

115 # Use the stored values, if available 116 username = self.username or self.options.username 117 password = self.password or self.options.password 118 self.username, self.password = getUsernamePassword(username, password)

119

120 - def setProxyUsernamePassword(self):

121 self.proxyUsername = None 122 self.proxyPassword = None

123

124 - def setCAchain(self):

125 self.ca_chain = self.options.ca_chain

126

127 - def setProxy(self):

128 if self.options.proxy is None or self.options.proxy == '': 129 self.proxy = None 130 else: 131 self.proxy = "http://%s" % self.options.proxy

132

133 - def setForce(self):

134 self.force = None

135

136 - def setServer(self):

137 # set the proxy 138 self.setProxy() 139 140 if self.proxy is None: 141 self.warn(1, "Connecting to %s" % self.url) 142 else: 143 self.warn(1, "Connecting to %s (via proxy '%s')" % (self.url, self.proxy)) 144 145 # set the CA chain 146 self.setCAchain() 147 # set the proxy username and password 148 self.setProxyUsernamePassword() 149 self.server = getServer(self.url, self.proxy, self.proxyUsername, 150 self.proxyPassword, self.ca_chain) 151 # Compress the output, just to be fast 152 self.server.set_transport_flags( 153 transfer=Output.TRANSFER_BINARY, 154 encoding=Output.ENCODE_GZIP)

155

156 - def setChannels(self):

157 if not self.options.channel: 158 self.die(-1, "No channel was specified") 159 self.channels = self.options.channel 160 self.warn(1, "Channels: %s" % ' '.join(self.channels))

161 162 setNoChannels = setChannels 163

164 - def setOrg(self):

165 self.orgId = -1

166

167 - def setCount(self):

168 if not self.options.count: 169 self.count = HEADERS_PER_CALL 170 else: 171 self.count = self.options.count

172

173 - def setRelativeDir(self):

174 self.relativeDir = None

175

176 - def directory(self):

177 self.warn(2, "Uploading files from directory", self.options.dir) 178 179 for filename in listdir(self.options.dir): 180 # only add packages 181 if filename[-3:] in ("rpm", "mpm"): 182 self.files.append(filename)

183

184 - def filter_excludes(self):

185 if not self.options.exclude: 186 return self 187 for f in self.files[:]: 188 bf = os.path.basename(f) 189 for pattern in self.options.exclude: 190 if fnmatch.fnmatch(bf, pattern): 191 self.warn(1, "Ignoring %s" % f) 192 self.files.remove(f) 193 return self

194

195 - def readStdin(self):

196 self.warn(1, "Reading package names from stdin") 197 self.files = self.files + readStdin()

198

199 - def _listChannelSource(self):

200 if self.use_session: 201 return listChannelSourceBySession(self.server, 202 self.session.getSessionString(), 203 self.channels) 204 205 return listChannelSource(self.server, 206 self.username, self.password, 207 self.channels)

208

209 - def _listChannel(self):

210 if self.use_session: 211 if self.use_checksum_paths: 212 return listChannelChecksumBySession(self.server, 213 self.session.getSessionString(), self.channels) 214 215 return listChannelBySession(self.server, 216 self.session.getSessionString(), 217 self.channels) 218 219 if self.use_checksum_paths: 220 return listChannelChecksum(self.server, 221 self.username, self.password, 222 self.channels) 223 224 return listChannel(self.server, 225 self.username, self.password, 226 self.channels)

227

228 - def list(self):

229 # set the URL 230 self.setURL() 231 # set the channels 232 self.setChannels() 233 # set the server 234 self.setServer() 235 236 self.authenticate() 237 238 if self.options.source: 239 channel_list = self._listChannelSource() 240 else: 241 # List the channel's contents 242 channel_list = self._listChannel() 243 244 for p in channel_list: 245 print(p[:6])

246

247 - def newest(self):

248 # set the URL 249 self.setURL() 250 # set the channels 251 self.setChannels() 252 # set the server 253 self.setServer() 254 255 self.authenticate() 256 257 sources = self.options.source 258 259 if sources: 260 return self.get_missing_source_packages() 261 262 return self.get_newest_binary_packages()

263

264 - def get_newest_binary_packages(self):

265 # Loop through the args and only keep the newest ones 266 localPackagesHash = {} 267 for filename in self.files: 268 nvrea = self._processFile(filename, nosig=1)['nvrea'] 269 name = nvrea[0] 270 if name not in localPackagesHash: 271 localPackagesHash[name] = {nvrea: filename} 272 continue 273 274 same_names_hash = localPackagesHash[name] 275 # Already saw this name 276 if nvrea in same_names_hash: 277 # Already seen this nvrea 278 continue 279 skip_rpm = 0 280 for local_nvrea in same_names_hash.keys(): 281 # XXX is_mpm should be set accordingly 282 ret = packageCompare(local_nvrea, nvrea, 283 is_mpm=0) 284 if ret == 0 and local_nvrea[4] == nvrea[4]: 285 # Weird case, we've already compared the two 286 skip_rpm = 1 287 break 288 289 if ret > 0: 290 # nvrea is older than local_nvrea 291 skip_rpm = 1 292 break 293 294 if ret < 0: 295 # nvrea is newer than local_nvrea 296 del same_names_hash[local_nvrea] 297 298 # Different arches - go on 299 300 if skip_rpm: 301 # Older 302 continue 303 304 same_names_hash[nvrea] = filename 305 306 # Now get the list from the server 307 pkglist = self._listChannel() 308 309 for p in pkglist: 310 name = p[0] 311 if name not in localPackagesHash: 312 # Not in the local list 313 continue 314 same_names_hash = localPackagesHash[name] 315 remote_nvrea = tuple(p[:5]) 316 if remote_nvrea in same_names_hash: 317 # The same package is already uploaded 318 del same_names_hash[remote_nvrea] 319 continue 320 321 for local_nvrea in list(same_names_hash.keys()): 322 # XXX is_mpm sould be set accordingly 323 ret = packageCompare(local_nvrea, remote_nvrea, 324 is_mpm=0) 325 if ret < 0: 326 # The remote package is newer than the local one 327 del same_names_hash[local_nvrea] 328 continue 329 if ret == 0 and local_nvrea[4] == remote_nvrea[4]: 330 # Same arch 331 del same_names_hash[local_nvrea] 332 continue 333 # This means local is newer 334 335 # Return the list of files to push 336 l = [] 337 for fhash in localPackagesHash.values(): 338 for filename in fhash.values(): 339 l.append(filename) 340 l.sort() 341 self.files = l

342

343 - def _listMissingSourcePackages(self):

344 if self.use_session: 345 return listMissingSourcePackagesBySession(self.server, 346 self.session.getSessionString(), self.channels) 347 348 return listMissingSourcePackages(self.server, 349 self.username, self.password, self.channels)

350

351 - def get_missing_source_packages(self):

352 localPackagesHash = {} 353 for filename in self.files: 354 localPackagesHash[os.path.basename(filename)] = filename 355 356 # Now get the list from the server 357 pkglist = self._listMissingSourcePackages() 358 359 to_push = [] 360 for pkg in pkglist: 361 pkg_name, _pkg_channel = pkg[:2] 362 if pkg_name not in localPackagesHash: 363 # We don't have it 364 continue 365 to_push.append(localPackagesHash[pkg_name]) 366 367 to_push.sort() 368 self.files = to_push 369 return self.files

370

371 - def test(self):

372 # Test only 373 for p in self.files: 374 print(p)

375

376 - def _get_files(self):

377 return self.files[:]

378

379 - def _uploadSourcePackageInfo(self, info):

380 if self.use_session: 381 return call(self.server.packages.uploadSourcePackageInfoBySession, 382 self.session.getSessionString(), info) 383 384 return call(self.server.packages.uploadSourcePackageInfo, 385 self.username, self.password, info)

386

387 - def _uploadPackageInfo(self, info):

388 if self.use_session: 389 return call(self.server.packages.uploadPackageInfoBySession, 390 self.session.getSessionString(), info) 391 392 return call(self.server.packages.uploadPackageInfo, 393 self.username, self.password, info)

394

395 - def uploadHeaders(self):

396 # Set the forcing factor 397 self.setForce() 398 # Relative directory 399 self.setRelativeDir() 400 # Set the count 401 self.setCount() 402 # set the org 403 self.setOrg() 404 # set the URL 405 self.setURL() 406 # set the channels 407 self.setNoChannels() 408 409 # set the server 410 self.setServer() 411 412 self.authenticate() 413 414 source = self.options.source 415 file_list = self._get_files() 416 417 while file_list: 418 chunk = file_list[:self.count] 419 del file_list[:self.count] 420 uploadedPackages, headersList = self._processBatch(chunk, 421 relativeDir=self.relativeDir, source=self.options.source, 422 verbose=self.options.verbose, nosig=self.options.nosig) 423 424 if not headersList: 425 # Nothing to do here... 426 continue 427 428 # Send the big hash 429 info = {'packages': headersList} 430 if self.orgId > 0 or self.orgId == '': 431 info['orgId'] = self.orgId 432 433 if self.force: 434 info['force'] = self.force 435 436 if self.channels: 437 info['channels'] = self.channels 438 439 # Some feedback 440 if self.options.verbose: 441 ReportError("Uploading batch:") 442 for p in list(uploadedPackages.values())[0]: 443 ReportError("\t\t%s" % p) 444 445 if source: 446 ret = self._uploadSourcePackageInfo(info) 447 else: 448 ret = self._uploadPackageInfo(info) 449 450 if ret is None: 451 self.die(-1, "Upload attempt failed") 452 453 # Append the package information 454 alreadyUploaded, newPackages = ret 455 pkglists = (alreadyUploaded, newPackages) 456 457 for idx, item in enumerate(pkglists): 458 for p in item: 459 key = tuple(p[:5]) 460 if key not in uploadedPackages: 461 # XXX Hmm 462 self.warn(1, "XXX XXX %s" % str(p)) 463 filename, checksum = uploadedPackages[key] 464 # Some debugging 465 if self.options.verbose: 466 if idx == 0: 467 pattern = "Already uploaded: %s" 468 else: 469 pattern = "Uploaded: %s" 470 print(pattern % filename) 471 # Per-package post actions 472 # For backwards-compatibility with old spacewalk-proxy 473 try: 474 self.processPackage(p, filename, checksum) 475 except TypeError: 476 self.processPackage(p, filename)

477

478 - def processPackage(self, package, filename, checksum=None):

479 pass

480

481 - def checkSession(self, session):

482 return call(self.server.packages.check_session, session)

483

484 - def readSession(self):

485 # pylint: disable=W0703 486 try: 487 self.session = rhnpush_cache.RHNPushSession() 488 self.session.readSession() 489 except Exception: 490 self.session = None

491

492 - def writeSession(self, session):

493 if self.session: 494 self.session.setSessionString(session) 495 else: 496 self.session = rhnpush_cache.RHNPushSession() 497 self.session.setSessionString(session) 498 499 if not self.options.no_session_caching: 500 self.session.writeSession()

501

502 - def authenticate(self):

503 # Only use the session token stuff if we're talking to a sat that supports session-token authentication. 504 self.readSession() 505 if self.session and not self.options.new_cache and self.options.username == self.username: 506 chksession = self.checkSession(self.session.getSessionString()) 507 if chksession: 508 return 509 self.setUsernamePassword() 510 sessstr = call(self.server.packages.login, self.username, self.password) 511 self.writeSession(sessstr) 512 513 # set whether we should use checksum paths or not (if upstream supports 514 # it we should). 515 self.use_checksum_paths = hasChannelChecksumCapability(self.server)

516 517 @staticmethod

518 - def _processFile(filename, relativeDir=None, source=None, nosig=None):

519 """ Processes a file 520 Returns a hash containing: 521 header 522 packageSize 523 checksum 524 relativePath 525 nvrea 526 """ 527 528 # Is this a file? 529 if not os.access(filename, os.R_OK): 530 raise UploadError("Could not stat the file %s" % filename) 531 if not os.path.isfile(filename): 532 raise UploadError("%s is not a file" % filename) 533 534 # Size 535 size = os.path.getsize(filename) 536 537 try: 538 a_pkg = package_from_filename(filename) 539 a_pkg.read_header() 540 a_pkg.payload_checksum() 541 assert a_pkg.header 542 except: 543 raise_with_tb(UploadError("%s is not a valid package" % filename), sys.exc_info()[2]) 544 545 if nosig is None and not a_pkg.header.is_signed(): 546 raise UploadError("ERROR: %s: unsigned rpm (use --nosig to force)" 547 % filename) 548 549 # Get the name, version, release, epoch, arch 550 lh = [] 551 for k in ['name', 'version', 'release', 'epoch']: 552 if k == 'epoch' and not a_pkg.header[k]: 553 # Fix the epoch 554 lh.append(sstr("")) 555 else: 556 lh.append(sstr(a_pkg.header[k])) 557 558 if source: 559 lh.append('src') 560 else: 561 lh.append(sstr(a_pkg.header['arch'])) 562 563 # Build the header hash to be sent 564 info = {'header': Binary(a_pkg.header.unload()), 565 'checksum_type': a_pkg.checksum_type, 566 'checksum': a_pkg.checksum, 567 'packageSize': size, 568 'header_start': a_pkg.header_start, 569 'header_end': a_pkg.header_end} 570 if relativeDir: 571 # Append the relative dir too 572 info["relativePath"] = "%s/%s" % (relativeDir, 573 os.path.basename(filename)) 574 info['nvrea'] = tuple(lh) 575 return info

576

577 - def _processBatch(self, batch, relativeDir, source, verbose, nosig=None):

578 sentPackages = {} 579 headersList = [] 580 for filename in batch: 581 if verbose: 582 print("Uploading %s" % filename) 583 info = self._processFile(filename, relativeDir=relativeDir, source=source, 584 nosig=nosig) 585 # Get nvrea 586 nvrea = info['nvrea'] 587 del info['nvrea'] 588 589 sentPackages[nvrea] = (filename, info['checksum']) 590 591 # Append the header to the list of headers to be sent out 592 headersList.append(info) 593 return sentPackages, headersList

594

595 596 -def readStdin():

597 # Reads the standard input lines and returns a list 598 l = [] 599 while 1: 600 line = sys.stdin.readline() 601 if not line: 602 break 603 l.append(line.strip()) 604 return l

605

606 607 -def getUsernamePassword(cmdlineUsername, cmdlinePassword):

608 # Returns a username and password (either by returning the ones passed as 609 # args, or the user's input 610 if cmdlineUsername and cmdlinePassword: 611 return cmdlineUsername, cmdlinePassword 612 613 username = cmdlineUsername 614 password = cmdlinePassword 615 616 # Read the username, if not already specified 617 tty = open("/dev/tty", "w") 618 tty.write("Username: ") 619 tty.close() 620 tty = open("/dev/tty", "r") 621 622 while not username: 623 try: 624 username = tty.readline() 625 except KeyboardInterrupt: 626 tty.write("\n") 627 sys.exit(0) 628 if username is None: 629 # EOF 630 tty.write("\n") 631 sys.exit(0) 632 username = username.strip() 633 if username: 634 break 635 636 # Now read the password 637 while not password: 638 try: 639 password = getpass.getpass("Password: ") 640 except KeyboardInterrupt: 641 tty.write("\n") 642 sys.exit(0) 643 tty.close() 644 return username, password

645

646 647 -def listdir(directory):

648 directory = os.path.abspath(os.path.normpath(directory)) 649 if not os.access(directory, os.R_OK | os.X_OK): 650 raise UploadError("Cannot read from directory %s" % directory) 651 if not os.path.isdir(directory): 652 raise UploadError("%s not a directory" % directory) 653 # Build the package list 654 packagesList = [] 655 for f in os.listdir(directory): 656 packagesList.append("%s/%s" % (directory, f)) 657 return packagesList

658

659 660 -def call(function, *params, **kwargs):

661 # Wrapper function 662 try: 663 ret = function(*params) 664 except xmlrpclib.Fault: 665 e = sys.exc_info()[1] 666 x = parseXMLRPCfault(e) 667 if x.faultString: 668 print(x.faultString) 669 if x.faultExplanation: 670 print(x.faultExplanation) 671 sys.exit(-1) 672 except xmlrpclib.ProtocolError: 673 e = sys.exc_info()[1] 674 if kwargs.get('raise_protocol_error'): 675 raise 676 print(e.errmsg) 677 sys.exit(-1) 678 679 return ret

680

681 682 -def parseXMLRPCfault(fault):

683 if not isinstance(fault, xmlrpclib.Fault): 684 return None 685 faultCode = fault.faultCode 686 if faultCode and isinstance(faultCode, type(1)): 687 faultCode = -faultCode 688 return ServerFault(faultCode, "", fault.faultString)

689

690 # pylint: disable=C0103 691 692 693 -def listChannel(server, username, password, channels):

694 return call(server.packages.listChannel, channels, username, password)

695

696 697 -def listChannelChecksum(server, username, password, channels):

698 return call(server.packages.listChannelChecksum, channels, username, 699 password)

700

701 702 -def listChannelBySession(server, session_string, channels):

703 return call(server.packages.listChannelBySession, channels, session_string)

704

705 706 -def listChannelChecksumBySession(server, session_string, channels):

707 return call(server.packages.listChannelChecksumBySession, channels, 708 session_string)

709

710 711 -def listChannelSource(server, username, password, channels):

712 return call(server.packages.listChannelSource, channels, username, password)

713

714 715 -def listChannelSourceBySession(server, session_string, channels):

716 return call(server.packages.listChannelSourceBySession, channels, session_string)

717

718 719 -def listMissingSourcePackages(server, username, password, channels):

720 return call(server.packages.listMissingSourcePackages, channels, username, password)

721

722 723 -def listMissingSourcePackagesBySession(server, session_string, channels):

724 return call(server.packages.listMissingSourcePackagesBySession, channels, session_string)

725

726 727 -def getPackageChecksumBySession(server, session_string, info):

728 return call(server.packages.getPackageChecksumBySession, session_string, info)

729

730 731 -def getSourcePackageChecksumBySession(server, session_string, info):

732 return call(server.packages.getSourcePackageChecksumBySession, session_string, info)

733

734 735 -def getSourcePackageChecksum(server, username, password, info):

736 return call(server.packages.getSourcePackageChecksum, username, password, info)

737

738 # for backward compatibility with satellite <5.4.0 739 740 741 -def getPackageMD5sumBySession(server, session_string, info):

742 return call(server.packages.getPackageMD5sumBySession, session_string, info)

743

744 745 -def getSourcePackageMD5sumBySession(server, session_string, info):

746 return call(server.packages.getSourcePackageMD5sumBySession, session_string, info)

747

748 749 -def getServer(uri, proxy=None, username=None, password=None, ca_chain=None):

750 s = rpclib.Server(uri, proxy=proxy, username=username, password=password) 751 if ca_chain: 752 s.add_trusted_cert(ca_chain) 753 return s

754

755 # pylint: disable=E1123 756 -def hasChannelChecksumCapability(rpc_server):

757 """ check whether server supports getPackageChecksumBySession function""" 758 # pylint: disable=W1505 759 if 'rpcServerOverride' in inspect.getargspec(rhnserver.RhnServer.__init__)[0]: 760 server = rhnserver.RhnServer(rpcServerOverride=rpc_server) 761 else: 762 server = rhnserver.RhnServer() 763 # pylint: disable=W0212 764 server._server = rpc_server 765 return server.capabilities.hasCapability('xmlrpc.packages.checksums')

766

767 768 -def exists_getPackageChecksumBySession(rpc_server):

769 """ check whether server supports getPackageChecksumBySession function""" 770 # unfortunatelly we do not have capability for getPackageChecksumBySession function, 771 # but extended_profile in version 2 has been created just 2 months before 772 # getPackageChecksumBySession lets use it instead 773 # pylint: disable=W1505 774 if 'rpcServerOverride' in inspect.getargspec(rhnserver.RhnServer.__init__)[0]: 775 server = rhnserver.RhnServer(rpcServerOverride=rpc_server) 776 else: 777 server = rhnserver.RhnServer() 778 # pylint: disable=W0212 779 server._server = rpc_server 780 return server.capabilities.hasCapability('xmlrpc.packages.extended_profile', 2)

781

782 # compare two package [n,v,r,e] tuples 783 784 785 -def packageCompare(pkg1, pkg2, is_mpm=None):

786 if pkg1[0] != pkg2[0]: 787 raise ValueError("You should only compare packages with the same name") 788 packages = [] 789 for pkg in (pkg1, pkg2): 790 e = pkg[3] 791 if e == "": 792 e = None 793 elif e is not None: 794 e = str(e) 795 evr = (e, str(pkg[1]), str(pkg[2])) 796 packages.append(evr) 797 if is_mpm: 798 func = rhn_mpm.labelCompare 799 else: 800 func = rhn_rpm.labelCompare 801 return func(packages[0], packages[1])

802

803 804 # returns a header from a package file on disk. 805 -def get_header(filename, fildes=None, source=None):

806 try: 807 h = get_package_header(filename=filename, fd=fildes) 808 except: 809 raise_with_tb(UploadError("Package is invalid"), sys.exc_info()[2]) 810 811 # Verify that this is indeed a binary/source. xor magic 812 # xor doesn't work with None values, so compare the negated values - the 813 # results are identical 814 if (not source) ^ (not h.is_source): 815 raise UploadError("Unexpected RPM package type") 816 return h

817

818 819 -def ReportError(*args):

820 sys.stderr.write(' '.join(map(str, args)) + "\n")

821

Source Code for Module rhnpush.uploadLib