rhnpush.rhnpush

62 # Initialize a command-line processing object with a table of options 63 optionsTable = [ 64 Option('-v', '--verbose', action='count', help='Increase verbosity', 65 default=0), 66 Option('-d', '--dir', action='store', 67 help='Process packages from this directory'), 68 Option('-c', '--channel', action='append', 69 help='Manage this channel (specified by label)'), 70 Option('-n', '--count', action='store', 71 help='Process this number of headers per call', type='int'), 72 Option('-l', '--list', action='store_true', 73 help='Only list the specified channels'), 74 Option('-r', '--reldir', action='store', 75 help='Relative dir to associate with the file'), 76 Option('-o', '--orgid', action='store', 77 help='Org ID', type='int'), 78 Option('-u', '--username', action='store', 79 help='Use this username to connect to RHN/Satellite'), 80 Option('-p', '--password', action='store', 81 help='Use this password to connect to RHN/Satellite'), 82 Option('-s', '--stdin', action='store_true', 83 help='Read the package names from stdin'), 84 Option('-X', '--exclude', action='append', 85 help='Exclude packages that match this glob expression'), 86 Option('--force', action='store_true', 87 help='Force the package upload (overwrites if already uploaded)'), 88 Option('--nosig', action='store_true', help='Push unsigned packages'), 89 Option('--newest', action='store_true', 90 help='Only push the packages that are newer than the server ones'), 91 Option('--nullorg', action='store_true', help='Use the null org id'), 92 Option('--header', action='store_true', 93 help='Upload only the header(s)'), 94 Option('--source', action='store_true', 95 help='Upload source package information'), 96 Option('--server', action='store', 97 help='Push to this server (http[s]://<hostname>/APP)'), 98 Option('--proxy', action='store', 99 help='Use proxy server (<server>:<port>)'), 100 Option('--test', action='store_true', 101 help='Only print the packages to be pushed'), 102 Option('-?', '--usage', action='store_true', 103 help='Briefly describe the options'), 104 Option('-N', '--new-cache', action='store_true', 105 help='Create a new username/password cache'), 106 Option('--extended-test', action='store_true', 107 help='Perform a more verbose test'), 108 Option('--no-session-caching', action='store_true', 109 help='Disables session-token authentication.'), 110 Option('--tolerant', action='store_true', 111 help='If rhnpush errors while uploading a package, continue uploading the rest of the packages.'), 112 Option('--ca-chain', action='store', help='alternative SSL CA Cert'), 113 Option('--timeout', action='store', type='int', metavar='SECONDS', 114 help='Change default connection timeout.') 115 ] 116 117 # Having to maintain a store_true list is ugly. I'm trying to get rid of this. 118 true_list = ['usage', 'test', 'source', 'header', 'nullorg', 'newest', 119 'nosig', 'force', 'list', 'stdin', 'new_cache', 120 'extended_test', 'no_session_caching', 'tolerant'] 121 # pylint: disable=E1101,E1103 122 optionParser = OptionParser(option_list=optionsTable, usage="%prog [OPTION] [<package>]") 123 manager = rhnpush_confmanager.ConfManager(optionParser, true_list) 124 options = manager.get_config() 125 126 upload = UploadClass(options, files=options.files) 127 128 if options.usage: 129 optionParser.print_usage() 130 sys.exit(0) 131 132 if options.proxy: 133 options.proxy = idn_ascii_to_puny(options.proxy) 134 135 if options.list: 136 if not options.channel: 137 upload.die(1, "Must specify a channel for --list to work") 138 upload.list() 139 return 0 140 141 if options.dir and not options.stdin: 142 upload.directory() 143 144 elif options.stdin and not options.dir: 145 upload.readStdin() 146 147 elif options.dir and options.stdin: 148 upload.readStdin() 149 upload.directory() 150 151 if options.exclude: 152 upload.filter_excludes() 153 154 if options.newest: 155 if not options.channel: 156 upload.die(1, "Must specify a channel for --newest to work") 157 158 upload.newest() 159 160 if not upload.files: 161 if upload.newest: 162 print("No new files to upload; exiting") 163 else: 164 print("Nothing to do (try --help for more options)") 165 sys.exit(0) 166 167 if options.test: 168 upload.test() 169 return 0 170 171 if options.extended_test: 172 upload.extended_test() 173 return 0 174 175 if options.header: 176 upload.uploadHeaders() 177 return 0 178 179 ret = upload.packages() 180 if ret != 0: 181 return 1 182 return 0

185 # pylint: disable=E1101,W0201,W0632 186

187 - def __init__(self, options, files=None):

188 uploadLib.UploadClass.__init__(self, options, files) 189 self.url_v2 = None

190

191 - def setURL(self):

192 server = sstr(idn_ascii_to_puny(self.options.server)) 193 if server is None: 194 self.die(1, "Required parameter --server not supplied") 195 scheme, netloc, path, params, query, fragment = tupleify_urlparse( 196 urlparse.urlparse(server)) 197 if not netloc: 198 # No schema - trying to patch it up ourselves? 199 server = "http://%s" % server 200 scheme, netloc, path, params, query, fragment = tupleify_urlparse( 201 urlparse.urlparse(server)) 202 203 if not netloc: 204 self.die(2, "Invalid URL %s" % server) 205 if path == '': 206 path = '/APP' 207 if scheme.lower() not in ('http', 'https'): 208 self.die(3, "Unknown URL scheme %s" % scheme) 209 self.url = urlparse.urlunparse((scheme, netloc, path, params, query, 210 fragment)) 211 self.url_v2 = urlparse.urlunparse((scheme, netloc, "/PACKAGE-PUSH", 212 params, query, fragment))

213

214 - def setOrg(self):

215 if self.options.nullorg: 216 if self.options.force: 217 self.die(1, "ERROR: You cannot force a package to a nullorg channel.") 218 else: 219 # They push things to the None org id 220 self.orgId = '' 221 else: 222 self.orgId = self.options.orgid or -1

223

224 - def setForce(self):

225 if self.options.force: 226 self.force = 4 227 else: 228 self.force = None

229

230 - def setRelativeDir(self):

231 self.relativeDir = self.options.reldir

232

233 - def setChannels(self):

234 self.channels = self.options.channel or []

235 236 # pylint: disable=W0702

237 - def _test_force(self):

238 test_force_str = "Setting force flag: %s" 239 test_force = "Passed" 240 try: 241 self.setForce() 242 except: 243 test_force = "Failed" 244 print(test_force_str % test_force)

245

246 - def _test_set_org(self):

247 test_set_org_str = "Setting the org: %s" 248 test_set_org = "Passed" 249 try: 250 self.setOrg() 251 except: 252 test_set_org = "Failed" 253 print(test_set_org_str % test_set_org)

254

255 - def _test_set_url(self):

256 test_set_url_str = "Setting the URL: %s" 257 test_set_url = "Passed" 258 try: 259 self.setURL() 260 except: 261 test_set_url = "Failed" 262 print(test_set_url_str % test_set_url)

263

264 - def _test_set_channels(self):

265 test_set_channels_str = "Setting the channels: %s" 266 test_set_channels = "Passed" 267 try: 268 self.setChannels() 269 except: 270 test_set_channels = "Failed" 271 print(test_set_channels_str % test_set_channels)

272

273 - def _test_username_password(self):

274 test_user_pass_str = "Setting the username and password: %s" 275 test_user_pass = "Passed" 276 try: 277 self.setUsernamePassword() 278 except: 279 test_user_pass = "Failed" 280 print(test_user_pass_str % test_user_pass)

281

282 - def _test_set_server(self):

283 test_set_server_str = "Setting the server: %s" 284 test_set_server = "Passed" 285 try: 286 self.setServer() 287 except: 288 test_set_server = "Failed" 289 print(test_set_server_str % test_set_server)

290

291 - def _test_connect(self):

292 auth_ret = uploadLib.call(self.server.packages.test_login, 293 self.username, self.password) 294 if auth_ret == 1: 295 test_auth = "Passed" 296 else: 297 test_auth = "Failed" 298 print("Testing connection and authentication: %s" % test_auth)

299

300 - def _test_access(self):

301 access_ret = callable(self.server.packages.channelPackageSubscriptionBySession) 302 303 if access_ret == 1: 304 test_access = "Passed" 305 else: 306 test_access = "Failed" 307 print("Testing access to upload functionality on server: %s" % test_access)

308 309 # 12/22/05 wregglej 173287 Added a this funtion to test the new session authentication stuff. 310 # It still needs work.

311 - def _test_authenticate(self):

312 self.authenticate()

313

314 - def extended_test(self):

315 self._test_force() 316 self._test_set_org() 317 self._test_set_url() 318 self._test_set_channels() 319 self._test_username_password() 320 self._test_set_server() 321 self._test_connect() 322 self._test_access() 323 print("The files that would have been pushed:") 324 self.test()

325

326 - def packages(self):

327 self.setForce() 328 # set the org 329 self.setOrg() 330 # set the URL 331 self.setURL() 332 # set the channels 333 self.setChannels() 334 # set the server 335 self.setServer() 336 # 12/22/05 wregglej 173287 authenticate the session. 337 self.authenticate() 338 339 # Do we have the new-style handler available? 340 341 # ping the server for status 342 self.warn(2, "url is", self.url_v2) 343 ping = rhnpush_v2.PingPackageUpload(self.url_v2, self.options.proxy) 344 ping_status, errmsg, headerinfo = ping.ping() 345 self.warn(2, "Result codes:", ping_status, errmsg) 346 347 # move patch clusters to the end because all the patches in the cluster 348 # have to be pushed before the cluster itself 349 files1 = [] 350 files2 = [] 351 for filename in self.files: 352 if filename.startswith('patch-cluster-'): 353 files2.append(filename) 354 else: 355 files1.append(filename) 356 357 self.files = files1 + files2 358 359 channel_packages = [] 360 361 # a little fault tolarence is in order 362 random.seed() 363 tries = 3 364 365 # satellites < 4.1.0 are no more supported 366 if sys.version_info[0] == 3: 367 pack_exist_check = headerinfo.get('X-RHN-Check-Package-Exists') 368 else: 369 pack_exist_check = headerinfo.getheader('X-RHN-Check-Package-Exists') 370 if not pack_exist_check: 371 self.die(-1, "Pushing to Satellite < 4.1.0 is not supported.") 372 373 (server_digest_hash, pkgs_info, digest_hash) = self.check_package_exists() 374 375 for pkg in self.files: 376 ret = None # pkilambi:errors off as not initialized.this fixes it. 377 378 # temporary fix for picking pkgs instead of full paths 379 pkg_key = (pkg.strip()).split('/')[-1] 380 381 if pkg_key not in server_digest_hash: 382 continue 383 384 checksum_type, checksum = digest = digest_hash[pkg_key] 385 server_digest = tuple(server_digest_hash[pkg_key]) 386 387 # compare checksums for existance check 388 if server_digest == digest and not self.options.force: 389 channel_packages.append(pkgs_info[pkg_key]) 390 self.warn(1, "Package %s already exists on the RHN Server-- Skipping Upload...." % pkg) 391 continue 392 393 if server_digest == (): 394 self.warn(1, "Package %s Not Found on RHN Server -- Uploading" % pkg) 395 396 elif server_digest == "on-disk" and not self.options.force: 397 channel_packages.append(pkgs_info[pkg_key]) 398 self.warn(0, "Package on disk but not on db -- Skipping Upload " % pkg) 399 continue 400 401 elif server_digest != digest: 402 if self.options.force: 403 self.warn(1, "Package checksum %s mismatch -- Forcing Upload" % pkg) 404 else: 405 msg = "Error: Package %s already exists on the server with" \ 406 " a different checksum. Skipping upload to prevent" \ 407 " overwriting existing package. (You may use rhnpush with" \ 408 " the --force option to force this upload if the" \ 409 " force_upload option is enabled on your server.)\n" % pkg 410 if not self.options.tolerant: 411 self.die(-1, msg) 412 self.warn(0, msg) 413 continue 414 415 for _t in range(0, tries): 416 try: 417 ret = self.package(pkg, checksum_type, checksum) 418 if ret is None: 419 raise uploadLib.UploadError() 420 421 # TODO: Revisit this. We throw this error all over the place, 422 # but doing so will cause us to skip the --tolerant logic 423 # below. I don't think we really want this behavior. 424 # There are some cases where we don't want to retry 3 425 # times, but not at the expense of disabling the tolerant 426 # flag, IMHO. This loop needs some lovin'. -- pav 427 428 # FIX: it checks for tolerant flag and aborts only if the flag is 429 #not specified 430 except uploadLib.UploadError: 431 ue = sys.exc_info()[1] 432 if not self.options.tolerant: 433 self.die(1, ue) 434 self.warn(2, ue) 435 except AuthenticationRequired: 436 # session expired so we re-authenticate for the process to complete 437 # this uses the username and password from memory if available 438 # else it prompts for one. 439 self.authenticate() 440 except: 441 self.warn(2, sys.exc_info()[1]) 442 wait = random.randint(1, 5) 443 self.warn(0, "Waiting %d seconds and trying again..." % wait) 444 time.sleep(wait) 445 # The else clause gets executed in the stuff in the try-except block *succeeds*. 446 else: 447 break 448 449 # if the preceeding for-loop exits without a call to break, then this else clause gets called. 450 # What's kind of weird is that if the preceeding for-loop doesn't call break then an error occurred 451 # and all of retry attempts failed. If the for-loop *does* call break then everything is hunky-dory. 452 # In short, this else clause only get's called if something is F.U.B.A.R and the retry attempts don't 453 # fix anything. 454 else: 455 if not self.options.tolerant: 456 # pkilambi:bug#176358:this exits with a error code of 1 457 self.die(1, "Giving up after %d attempts" % tries) 458 else: 459 print("Giving up after %d attempts and continuing on..." % (tries,)) 460 461 # 5/13/05 wregglej - 154248 ?? we still want to add the packages if they're source. 462 if ret and self.channels: # and ret['arch'] != 'src': 463 # Don't bother to add the package if 464 # no channel was specified or a source rpm was passed 465 channel_packages.append(ret) 466 467 # self.channels is never None, it always has at least one entry with an empty string. 468 if len(self.channels) == 1 and self.channels[0] == '': 469 return 0 470 info = { 471 'packages': channel_packages, 472 'channels': self.channels 473 } 474 if self.orgId == '' or self.orgId > 0: 475 info['orgId'] = self.orgId 476 477 # 2/3/06 wregglej 173287 Added check to see if we can use session tokens. 478 if channel_packages: 479 self.authenticate() 480 uploadLib.call(self.server.packages.channelPackageSubscriptionBySession, 481 self.session.getSessionString(), info) 482 return 0

483 484 # does an existance check of the packages to be uploaded and returns their checksum and other info

485 - def check_package_exists(self):

486 self.warn(2, "Computing checksum and package info. This may take some time ...") 487 pkg_hash = {} 488 digest_hash = {} 489 490 for pkg in self.files: 491 pkg_info = {} 492 pkg_key = (pkg.strip()).split('/')[-1] 493 494 if not os.access(pkg, os.R_OK): 495 if not self.options.tolerant: 496 self.die(-1, "Could not read file %s" % pkg) 497 self.warn(-1, "Could not read file %s" % pkg) 498 continue 499 try: 500 a_pkg = package_from_filename(pkg) 501 a_pkg.read_header() 502 a_pkg.payload_checksum() 503 except InvalidPackageError: 504 if not self.options.tolerant: 505 self.die(-1, "ERROR: %s: This file doesn't appear to be a package" % pkg) 506 self.warn(2, "ERROR: %s: This file doesn't appear to be a package" % pkg) 507 continue 508 except IOError: 509 if not self.options.tolerant: 510 self.die(-1, "ERROR: %s: No such file or directory available" % pkg) 511 self.warn(2, "ERROR: %s: No such file or directory available" % pkg) 512 continue 513 514 digest_hash[pkg_key] = (a_pkg.checksum_type, a_pkg.checksum) 515 a_pkg.input_stream.close() 516 517 for tag in ('name', 'version', 'release', 'epoch', 'arch'): 518 val = a_pkg.header[tag] 519 if val is None: 520 val = '' 521 pkg_info[tag] = val 522 # b195903:the arch for srpms should be obtained by is_source check 523 # instead of checking arch in header 524 if a_pkg.header.is_source: 525 if not self.options.source: 526 self.die(-1, "ERROR: Trying to Push src rpm, Please re-try with --source.") 527 if RPMTAG_NOSOURCE in a_pkg.header.keys(): 528 pkg_info['arch'] = 'nosrc' 529 else: 530 pkg_info['arch'] = 'src' 531 pkg_info['checksum_type'] = a_pkg.checksum_type 532 pkg_info['checksum'] = a_pkg.checksum 533 pkg_hash[pkg_key] = pkg_info 534 535 if self.options.nullorg: 536 # to satisfy xmlrpc from None values. 537 orgid = 'null' 538 else: 539 orgid = '' 540 541 info = { 542 'packages': pkg_hash, 543 'channels': self.channels, 544 'org_id': orgid, 545 'force': self.options.force or 0 546 } 547 # rpc call to get checksum info for all the packages to be uploaded 548 if not self.options.source: 549 # computing checksum and other info is expensive process and session 550 # could have expired.Make sure its re-authenticated. 551 self.authenticate() 552 if uploadLib.exists_getPackageChecksumBySession(self.server): 553 checksum_data = uploadLib.getPackageChecksumBySession(self.server, 554 self.session.getSessionString(), info) 555 else: 556 # old server only md5 capable 557 checksum_data = uploadLib.getPackageMD5sumBySession(self.server, 558 self.session.getSessionString(), info) 559 else: 560 # computing checksum and other info is expensive process and session 561 # could have expired.Make sure its re-authenticated. 562 self.authenticate() 563 if uploadLib.exists_getPackageChecksumBySession(self.server): 564 checksum_data = uploadLib.getSourcePackageChecksumBySession(self.server, 565 self.session.getSessionString(), info) 566 else: 567 # old server only md5 capable 568 checksum_data = uploadLib.getSourcePackageMD5sumBySession(self.server, 569 self.session.getSessionString(), info) 570 571 return (checksum_data, pkg_hash, digest_hash)

572

573 - def package(self, package, fileChecksumType, fileChecksum):

574 self.warn(1, "Uploading package %s" % package) 575 if not os.access(package, os.R_OK): 576 self.die(-1, "Could not read file %s" % package) 577 578 try: 579 h = uploadLib.get_header(package, source=self.options.source) 580 except uploadLib.UploadError: 581 e = sys.exc_info()[1] 582 # GS: MALFORMED PACKAGE 583 print("Unable to load package", package, ":", e) 584 return None 585 586 if hasattr(h, 'packaging'): 587 packaging = h.packaging 588 else: 589 packaging = 'rpm' 590 591 if packaging == 'rpm' and self.options.nosig is None and not h.is_signed(): 592 # pkilambi:bug#173886:force exit to check for sig if --nosig 593 raise uploadLib.UploadError("ERROR: %s: unsigned rpm (use --nosig to force)" % package) 594 595 try: 596 ret = self._push_package_v2(package, fileChecksumType, fileChecksum) 597 except uploadLib.UploadError: 598 e = sys.exc_info()[1] 599 ret, diff_level, pdict = e.args[:3] 600 severities = { 601 1: 'path changed', 602 2: 'package resigned', 603 3: 'differing build times or hosts', 604 4: 'package recompiled', 605 } 606 if diff_level in severities: 607 strmsg = \ 608 "Error: Package with same name already exists on " + \ 609 "server but contents differ (" + \ 610 severities[diff_level] + \ 611 "). Use --force or remove old package before " + \ 612 "uploading the newer version." 613 else: 614 strmsg = "Error: severity %s" % diff_level 615 self.warn(-1, "Uploading failed for %s\n%s\n\tDiff: %s" % 616 (package, strmsg, pdict['diff']['diff'])) 617 if diff_level != 1: 618 # This will prevent us from annoyingly retrying when there is 619 # no reason to. 620 raise uploadLib.UploadError() 621 return ret 622 623 return ret

624

625 - def _push_package_v2(self, package, fileChecksumType, fileChecksum):

626 self.warn(1, "Using POST request") 627 pu = rhnpush_v2.PackageUpload(self.url_v2, self.options.proxy) 628 629 pu.set_session(self.session.getSessionString()) 630 pu.set_force(self.options.force) 631 pu.set_null_org(self.options.nullorg) 632 pu.set_timeout(self.options.timeout) 633 634 status, msgstr = pu.upload(package, fileChecksumType, fileChecksum) 635 636 ret = {} 637 for tag in ('name', 'version', 'release', 'epoch', 'arch'): 638 val = getattr(pu, "package_%s" % tag) 639 if val is None: 640 val = '' 641 ret[tag] = val 642 643 ret['checksum_type'] = fileChecksumType 644 ret['checksum'] = fileChecksum 645 if status == 400: 646 # Bad request - something bad happened 647 try: 648 data = rpclib.xmlrpclib.loads(msgstr) 649 except: 650 # Raise the exception instead of silently dying 651 raise_with_tb(uploadLib.UploadError("Error pushing %s: %s (%s)" % 652 (package, msgstr, status)), sys.exc_info()[2]) 653 (diff_dict, ), methodname = data 654 del methodname 655 diff_level = diff_dict['level'] 656 pdict = diff_dict['diff'] 657 raise uploadLib.UploadError(ret, diff_level, pdict) 658 659 if status == 403: 660 # auth expired raise an exception to grab one 661 raise AuthenticationRequired() 662 663 if status != 200: 664 self.die(1, "Error pushing %s: %s (%s)" % (package, msgstr, status)) 665 666 return ret

Source Code for Module rhnpush.rhnpush_main