backend.server.repomd.mapper

1 # 2 # Copyright (c) 2008--2018 Red Hat, Inc. 3 # 4 # This software is licensed to you under the GNU General Public License, 5 # version 2 (GPLv2). There is NO WARRANTY for this software, express or 6 # implied, including the implied warranties of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 8 # along with this software; if not, see 9 # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. 10 # 11 # Red Hat trademarks are not licensed under GPLv2. No permission is 12 # granted to use or replicate Red Hat trademarks that are incorporated 13 # in this software or its documentation. 14 # 15 # 16 # Classes for mapping domain objects to the rhn db. 17 # 18 19 import os.path 20 import re 21 import time 22 23 from spacewalk.common import rhnCache 24 from spacewalk.common.rhnConfig import CFG 25 from spacewalk.common.usix import UnicodeType 26 from spacewalk.server import rhnSQL 27 28 import domain 29 30 31 CACHE_PREFIX = "/var/cache/rhn/" 32 33

34 -class ChannelMapper:

35 36 """ Data Mapper for Channels to the RHN db. """ 37

38 - def __init__(self, pkg_mapper, erratum_mapper, repomd_mapper):

39 self.pkg_mapper = pkg_mapper 40 self.erratum_mapper = erratum_mapper 41 self.repomd_mapper = repomd_mapper 42 43 self.channel_details_sql = rhnSQL.prepare(""" 44 select 45 c.label, 46 c.name, 47 ct.label checksum_type 48 from 49 rhnChannel c, 50 rhnChecksumType ct 51 where c.id = :channel_id 52 and c.checksum_type_id = ct.id 53 """) 54 55 self.channel_sql = rhnSQL.prepare(""" 56 select 57 package_id 58 from 59 rhnChannelPackage 60 where 61 channel_id = :channel_id 62 """) 63 64 self.last_modified_sql = rhnSQL.prepare(""" 65 select 66 to_char(last_modified, 'YYYYMMDDHH24MISS') as last_modified 67 from 68 rhnChannel 69 where id = :channel_id 70 """) 71 72 self.errata_id_sql = rhnSQL.prepare(""" 73 select 74 e.id 75 from 76 rhnChannelErrata ce, 77 rhnErrata e 78 where 79 ce.channel_id = :channel_id 80 and e.id = ce.errata_id 81 """) 82 83 self.comps_id_sql = rhnSQL.prepare(""" 84 select 85 id 86 from 87 rhnChannelComps 88 where 89 channel_id = :channel_id 90 and comps_type_id = 1 91 order by id desc 92 """) 93 94 self.modules_id_sql = rhnSQL.prepare(""" 95 select 96 id 97 from 98 rhnChannelComps 99 where 100 channel_id = :channel_id 101 and comps_type_id = 2 102 order by id desc 103 """) 104 105 self.cloned_from_id_sql = rhnSQL.prepare(""" 106 select 107 original_id id 108 from 109 rhnChannelCloned 110 where 111 id = :channel_id 112 """)

113

114 - def last_modified(self, channel_id):

115 """ Get the last_modified field for the provided channel_id. """ 116 self.last_modified_sql.execute(channel_id=channel_id) 117 return self.last_modified_sql.fetchone()[0]

118

119 - def get_channel(self, channel_id):

120 """ Load the channel with id channel_id and its packages. """ 121 122 self.channel_details_sql.execute(channel_id=channel_id) 123 details = self.channel_details_sql.fetchone() 124 125 channel = domain.Channel(channel_id) 126 127 channel.label = details[0] 128 channel.name = details[1] 129 channel.checksum_type = details[2] 130 131 self.channel_sql.execute(channel_id=channel_id) 132 package_ids = self.channel_sql.fetchall() 133 134 channel.num_packages = len(package_ids) 135 channel.packages = self._package_generator(package_ids) 136 137 channel.errata = self._erratum_generator(channel_id) 138 139 self.comps_id_sql.execute(channel_id=channel_id) 140 comps_id = self.comps_id_sql.fetchone() 141 self.modules_id_sql.execute(channel_id=channel_id) 142 modules_id = self.modules_id_sql.fetchone() 143 144 if comps_id: 145 channel.comps = self.repomd_mapper.get_repomd(comps_id[0]) 146 if modules_id: 147 channel.modules = self.repomd_mapper.get_repomd(modules_id[0]) 148 149 self.cloned_from_id_sql.execute(channel_id=channel_id) 150 cloned_row = self.cloned_from_id_sql.fetchone() 151 if cloned_row is not None: 152 channel.cloned_from_id = cloned_row[0] 153 else: 154 channel.cloned_from_id = None 155 156 return channel

157

158 - def _package_generator(self, package_ids):

159 for package_id in package_ids: 160 pkg = self.pkg_mapper.get_package(package_id[0]) 161 yield pkg

162

163 - def _erratum_generator(self, channel_id):

164 self.errata_id_sql.execute(channel_id=channel_id) 165 erratum_ids = self.errata_id_sql.fetchall() 166 167 for erratum_id in erratum_ids: 168 erratum = self.erratum_mapper.get_erratum(erratum_id[0]) 169 yield erratum

170 171

172 -class CachedPackageMapper:

173 174 """ Data Mapper for Packages to an on-disc cache. """ 175

176 - def __init__(self, mapper):

177 cache = rhnCache.Cache() 178 179 # For more speed, we won't compress. 180 # cache = rhnCache.CompressedCache(cache) 181 182 cache = rhnCache.ObjectCache(cache) 183 self.cache = rhnCache.NullCache(cache) 184 self.mapper = mapper

185

186 - def get_package(self, package_id):

187 """ 188 Load the package with id package_id. 189 190 Load from the cache, if it is new enough. If not, fall back to the 191 provided mapper. 192 """ 193 package_id = str(package_id) 194 195 last_modified = str(self.mapper.last_modified(package_id)) 196 last_modified = last_modified.replace(" ", "") 197 last_modified = last_modified.replace(":", "") 198 last_modified = last_modified.replace("-", "") 199 200 cache_key = "repomd-packages/" + package_id 201 if self.cache.has_key(cache_key, last_modified): 202 package = self.cache.get(cache_key) 203 else: 204 package = self.mapper.get_package(package_id) 205 self.cache.set(cache_key, package, last_modified) 206 207 return package

208 209

210 -class SqlPackageMapper:

211 212 """ Data Mapper for Packages to the RHN db. """ 213

214 - def __init__(self):

215 self.details_sql = rhnSQL.prepare(""" 216 select 217 pn.name, 218 pevr.version, 219 pevr.release, 220 pevr.epoch, 221 pa.label arch, 222 c.checksum checksum, 223 p.summary, 224 p.description, 225 p.vendor, 226 p.build_time, 227 p.package_size, 228 p.payload_size, 229 p.installed_size, 230 p.header_start, 231 p.header_end, 232 pg.name package_group, 233 p.build_host, 234 p.copyright, 235 p.path, 236 sr.name source_rpm, 237 p.last_modified, 238 c.checksum_type 239 from 240 rhnPackage p, 241 rhnPackageName pn, 242 rhnPackageEVR pevr, 243 rhnPackageArch pa, 244 rhnPackageGroup pg, 245 rhnSourceRPM sr, 246 rhnChecksumView c 247 where 248 p.id = :package_id 249 and p.name_id = pn.id 250 and p.evr_id = pevr.id 251 and p.package_arch_id = pa.id 252 and p.package_group = pg.id 253 and p.source_rpm_id = sr.id 254 and p.checksum_id = c.id 255 """) 256 257 self.filelist_sql = rhnSQL.prepare(""" 258 select 259 pc.name 260 from 261 rhnPackageCapability pc, 262 rhnPackageFile pf 263 where 264 pf.package_id = :package_id 265 and pf.capability_id = pc.id 266 """) 267 268 self.prco_sql = rhnSQL.prepare(""" 269 select 270 'provides', 271 pp.sense, 272 pc.name, 273 pc.version 274 from 275 rhnPackageCapability pc, 276 rhnPackageProvides pp 277 where 278 pp.package_id = :package_id 279 and pp.capability_id = pc.id 280 union all 281 select 282 'requires', 283 pr.sense, 284 pc.name, 285 pc.version 286 from 287 rhnPackageCapability pc, 288 rhnPackageRequires pr 289 where 290 pr.package_id = :package_id 291 and pr.capability_id = pc.id 292 union all 293 select 294 'recommends', 295 prec.sense, 296 pc.name, 297 pc.version 298 from 299 rhnPackageCapability pc, 300 rhnPackageRecommends prec 301 where 302 prec.package_id = :package_id 303 and prec.capability_id = pc.id 304 union all 305 select 306 'supplements', 307 supp.sense, 308 pc.name, 309 pc.version 310 from 311 rhnPackageCapability pc, 312 rhnPackageSupplements supp 313 where 314 supp.package_id = :package_id 315 and supp.capability_id = pc.id 316 union all 317 select 318 'enhances', 319 enh.sense, 320 pc.name, 321 pc.version 322 from 323 rhnPackageCapability pc, 324 rhnPackageEnhances enh 325 where 326 enh.package_id = :package_id 327 and enh.capability_id = pc.id 328 union all 329 select 330 'suggests', 331 sugg.sense, 332 pc.name, 333 pc.version 334 from 335 rhnPackageCapability pc, 336 rhnPackageSuggests sugg 337 where 338 sugg.package_id = :package_id 339 and sugg.capability_id = pc.id 340 union all 341 select 342 'conflicts', 343 pcon.sense, 344 pc.name, 345 pc.version 346 from 347 rhnPackageCapability pc, 348 rhnPackageConflicts pcon 349 where 350 pcon.package_id = :package_id 351 and pcon.capability_id = pc.id 352 union all 353 select 354 'obsoletes', 355 po.sense, 356 pc.name, 357 pc.version 358 from 359 rhnPackageCapability pc, 360 rhnPackageObsoletes po 361 where 362 po.package_id = :package_id 363 and po.capability_id = pc.id 364 union all 365 select 366 'breaks', 367 brks.sense, 368 pc.name, 369 pc.version 370 from 371 rhnPackageCapability pc, 372 rhnPackageBreaks brks 373 where 374 brks.package_id = :package_id 375 and brks.capability_id = pc.id 376 union all 377 select 378 'predepends', 379 pdep.sense, 380 pc.name, 381 pc.version 382 from 383 rhnPackageCapability pc, 384 rhnPackagePredepends pdep 385 where 386 pdep.package_id = :package_id 387 and pdep.capability_id = pc.id 388 """) 389 390 self.last_modified_sql = rhnSQL.prepare(""" 391 select 392 to_char(last_modified, 'YYYYMMDDHH24MISS') as last_modified 393 from 394 rhnPackage 395 where id = :package_id 396 """) 397 398 self.other_sql = rhnSQL.prepare(""" 399 select 400 name, 401 text, 402 time 403 from 404 rhnPackageChangelog 405 where package_id = :package_id 406 """)

407

408 - def last_modified(self, package_id):

409 """ Get the last_modified date on the package with id package_id. """ 410 self.last_modified_sql.execute(package_id=package_id) 411 return self.last_modified_sql.fetchone()[0]

412

413 - def get_package(self, package_id):

414 """ Get the package with id package_id from the RHN db. """ 415 package = domain.Package(package_id) 416 self._fill_package_details(package) 417 self._fill_package_prco(package) 418 self._fill_package_filelist(package) 419 self._fill_package_other(package) 420 return package

421

422 - def _get_package_filename(self, pkg):

423 if pkg[18]: 424 path = pkg[18] 425 return os.path.basename(path) 426 else: 427 name = pkg[0] 428 version = pkg[1] 429 release = pkg[2] 430 arch = pkg[4] 431 432 return "%s-%s-%s.%s.rpm" % (name, version, release, arch)

433

434 - def _fill_package_details(self, package):

435 """ Load the packages basic details (summary, description, etc). """ 436 self.details_sql.execute(package_id=package.id) 437 pkg = self.details_sql.fetchone() 438 439 package.name = pkg[0] 440 package.version = pkg[1] 441 package.release = pkg[2] 442 if pkg[3] is not None: 443 package.epoch = pkg[3] 444 package.arch = pkg[4] 445 446 package.checksum_type = pkg[21] 447 package.checksum = pkg[5] 448 package.summary = string_to_unicode(pkg[6]) 449 package.description = string_to_unicode(pkg[7]) 450 package.vendor = string_to_unicode(pkg[8]) 451 452 package.build_time = oratimestamp_to_sinceepoch(pkg[9]) 453 454 package.package_size = pkg[10] 455 package.payload_size = pkg[11] 456 package.installed_size = pkg[12] 457 package.header_start = pkg[13] 458 package.header_end = pkg[14] 459 package.package_group = pkg[15] 460 package.build_host = pkg[16] 461 package.copyright = string_to_unicode(pkg[17]) 462 package.filename = self._get_package_filename(pkg) 463 package.source_rpm = pkg[19]

464

465 - def _fill_package_prco(self, package):

466 """ Load the package's provides, requires, conflicts, obsoletes. """ 467 self.prco_sql.execute(package_id=package.id) 468 deps = self.prco_sql.fetchall() or [] 469 470 for item in deps: 471 version = item[3] or "" 472 relation = "" 473 release = None 474 epoch = 0 475 if version: 476 sense = item[1] or 0 477 relation = SqlPackageMapper.__get_relation(sense) 478 479 vertup = version.split('-') 480 if len(vertup) > 1: 481 version = vertup[0] 482 release = vertup[1] 483 484 vertup = version.split(':') 485 if len(vertup) > 1: 486 epoch = vertup[0] 487 version = vertup[1] 488 489 dep = {'name': string_to_unicode(item[2]), 'flag': relation, 490 'version': version, 'release': release, 'epoch': epoch} 491 492 if item[0] == "provides": 493 package.provides.append(dep) 494 elif item[0] == "requires": 495 package.requires.append(dep) 496 elif item[0] == "conflicts": 497 package.conflicts.append(dep) 498 elif item[0] == "obsoletes": 499 package.obsoletes.append(dep) 500 elif item[0] == "recommends": 501 package.recommends.append(dep) 502 elif item[0] == "supplements": 503 package.supplements.append(dep) 504 elif item[0] == "enhances": 505 package.enhances.append(dep) 506 elif item[0] == "suggests": 507 package.suggests.append(dep) 508 elif item[0] == "breaks": 509 package.breaks.append(dep) 510 elif item[0] == "predepends": 511 package.predepends.append(dep) 512 else: 513 assert False, "Unknown PRCO type: %s" % item[0]

514 515 # @staticmethod

516 - def __get_relation(sense):

517 """ Convert the binary sense into a string. """ 518 519 # Flip the bits for easy comparison 520 sense = sense & 0xf 521 522 if sense == 2: 523 relation = "LT" 524 elif sense == 4: 525 relation = "GT" 526 elif sense == 8: 527 relation = "EQ" 528 elif sense == 10: 529 relation = "LE" 530 elif sense == 12: 531 relation = "GE" 532 else: 533 assert False, "Unknown relation sense: %s" % sense 534 535 return relation

536 537 __get_relation = staticmethod(__get_relation) 538

539 - def _fill_package_filelist(self, package):

540 """ Load the package's list of files. """ 541 self.filelist_sql.execute(package_id=package.id) 542 files = self.filelist_sql.fetchall() or [] 543 544 for file_dict in files: 545 package.files.append(string_to_unicode(file_dict[0]))

546

547 - def _fill_package_other(self, package):

548 """ Load the package's changelog info. """ 549 550 self.other_sql.execute(package_id=package.id) 551 log_data = self.other_sql.fetchall() or [] 552 553 for data in log_data: 554 555 date = oratimestamp_to_sinceepoch(data[2]) 556 557 chglog = {'author': string_to_unicode(data[0]), 'date': date, 558 'text': string_to_unicode(data[1])} 559 package.changelog.append(chglog)

560 561

562 -class CachedErratumMapper:

563 564 """ Data Mapper for Errata to an on-disc cache. """ 565

566 - def __init__(self, mapper, package_mapper):

567 self.package_mapper = package_mapper 568 569 cache = rhnCache.Cache() 570 cache = rhnCache.ObjectCache(cache) 571 self.cache = rhnCache.NullCache(cache) 572 self.mapper = mapper

573

574 - def get_erratum(self, erratum_id):

575 """ 576 Load the erratum with id erratum_id. 577 578 Load from the cache, if it is new enough. If not, fall back to the 579 provided mapper. 580 """ 581 erratum_id = str(erratum_id) 582 583 last_modified = str(self.mapper.last_modified(erratum_id)) 584 last_modified = re.sub(" ", "", last_modified) 585 last_modified = re.sub(":", "", last_modified) 586 last_modified = re.sub("-", "", last_modified) 587 588 cache_key = "repomd-errata/" + erratum_id 589 if self.cache.has_key(cache_key, last_modified): 590 erratum = self.cache.get(cache_key) 591 for package_id in erratum.package_ids: 592 package = self.package_mapper.get_package(package_id) 593 erratum.packages.append(package) 594 else: 595 erratum = self.mapper.get_erratum(erratum_id) 596 597 tmp_packages = erratum.packages 598 erratum.packages = [] 599 self.cache.set(cache_key, erratum, last_modified) 600 erratum.packages = tmp_packages 601 602 return erratum

603 604

605 -class SqlErratumMapper:

606

607 - def __init__(self, package_mapper):

608 self.package_mapper = package_mapper 609 610 self.last_modified_sql = rhnSQL.prepare(""" 611 select 612 to_char(last_modified, 'YYYYMMDDHH24MISS') as last_modified 613 from 614 rhnErrata 615 where id = :erratum_id 616 """) 617 618 self.erratum_details_sql = rhnSQL.prepare(""" 619 select 620 advisory, 621 advisory_name, 622 advisory_type, 623 advisory_rel, 624 description, 625 synopsis, 626 TO_CHAR(issue_date, 'YYYY-MM-DD HH24:MI:SS') AS issue_date, 627 TO_CHAR(update_date, 'YYYY-MM-DD HH24:MI:SS') AS update_date 628 from 629 rhnErrata 630 where 631 id = :erratum_id 632 """) 633 634 self.erratum_cves_sql = rhnSQL.prepare(""" 635 select 636 cve.name as cve_name 637 from 638 rhnCVE cve, 639 rhnErrataCVE ec 640 where 641 ec.errata_id = :erratum_id 642 and ec.cve_id = cve.id 643 """) 644 645 self.erratum_bzs_sql = rhnSQL.prepare(""" 646 select 647 bug_id, 648 summary, 649 href 650 from 651 rhnErrataBuglist 652 where 653 errata_id = :erratum_id 654 """) 655 656 self.erratum_packages_sql = rhnSQL.prepare(""" 657 select 658 package_id 659 from 660 rhnErrataPackage 661 where 662 errata_id = :erratum_id 663 """)

664

665 - def last_modified(self, erratum_id):

666 """ Get the last_modified field for the provided erratum_id. """ 667 self.last_modified_sql.execute(erratum_id=erratum_id) 668 return self.last_modified_sql.fetchone()[0]

669

670 - def get_erratum(self, erratum_id):

671 """ Get the package with id package_id from the RHN db. """ 672 erratum = domain.Erratum(erratum_id) 673 self._fill_erratum_details(erratum) 674 675 # TODO: These two don't work on satellites. 676 # We must not install the tables there 677 self._fill_erratum_bz_references(erratum) 678 self._fill_erratum_cve_references(erratum) 679 680 self._fill_erratum_packages(erratum) 681 return erratum

682

683 - def _fill_erratum_details(self, erratum):

684 self.erratum_details_sql.execute(erratum_id=erratum.id) 685 ertm = self.erratum_details_sql.fetchone() 686 687 erratum.readable_id = ertm[0] 688 erratum.title = ertm[1] 689 690 if ertm[2] == 'Security Advisory': 691 erratum.advisory_type = 'security' 692 elif ertm[2] == 'Bug Fix Advisory': 693 erratum.advisory_type = 'bugfix' 694 elif ertm[2] == 'Product Enhancement Advisory': 695 erratum.advisory_type = 'enhancement' 696 else: 697 erratum.advisory_type = 'errata' 698 699 erratum.version = ertm[3] 700 erratum.description = ertm[4] 701 erratum.synopsis = ertm[5] 702 erratum.issued = ertm[6] 703 erratum.updated = ertm[7]

704

705 - def _fill_erratum_bz_references(self, erratum):

706 self.erratum_bzs_sql.execute(erratum_id=erratum.id) 707 bz_refs = self.erratum_bzs_sql.fetchall_dict() 708 709 if bz_refs: 710 erratum.bz_references = bz_refs

711

712 - def _fill_erratum_cve_references(self, erratum):

713 self.erratum_cves_sql.execute(erratum_id=erratum.id) 714 cve_refs = self.erratum_cves_sql.fetchall() 715 716 for cve_ref in cve_refs: 717 erratum.cve_references.append(cve_ref[0])

718

719 - def _fill_erratum_packages(self, erratum):

720 self.erratum_packages_sql.execute(erratum_id=erratum.id) 721 pkgs = self.erratum_packages_sql.fetchall() 722 723 for pkg in pkgs: 724 package = self.package_mapper.get_package(pkg[0]) 725 erratum.packages.append(package) 726 erratum.package_ids.append(pkg[0])

727 728

729 -class SqlRepoMDMapper:

730

731 - def __init__(self):

732 self.repomd_sql = rhnSQL.prepare(""" 733 select 734 relative_filename 735 from 736 rhnChannelComps 737 where 738 id = :repomd_id 739 """)

740

741 - def get_repomd(self, repomd_id):

742 self.repomd_sql.execute(repomd_id=repomd_id) 743 repomd_row = self.repomd_sql.fetchone() 744 filename = os.path.join(CFG.mount_point, repomd_row[0]) 745 return domain.RepoMD(repomd_id, filename)

746 747

748 -def get_channel_mapper():

749 """ Factory Method-ish function to load a Channel Mapper. """ 750 package_mapper = get_package_mapper() 751 erratum_mapper = get_erratum_mapper(package_mapper) 752 repomd_mapper = SqlRepoMDMapper() 753 channel_mapper = ChannelMapper(package_mapper, erratum_mapper, repomd_mapper) 754 755 return channel_mapper

756 757

758 -def get_package_mapper():

759 """ Factory Method-ish function to load a Package Mapper. """ 760 package_mapper = SqlPackageMapper() 761 package_mapper = CachedPackageMapper(package_mapper) 762 763 return package_mapper

764 765

766 -def get_erratum_mapper(package_mapper):

767 """ Factory Method-ish function to load an Erratum Mapper. """ 768 erratum_mapper = SqlErratumMapper(package_mapper) 769 erratum_mapper = CachedErratumMapper(erratum_mapper, package_mapper) 770 771 return erratum_mapper

772 773

774 -def oratimestamp_to_sinceepoch(ts):

775 return time.mktime((ts.year, ts.month, ts.day, ts.hour, ts.minute, 776 ts.second, 0, 0, -1))

777 778

779 -def string_to_unicode(text):

780 if text is None: 781 return '' 782 if isinstance(text, UnicodeType): 783 return text 784 785 # First try a bunch of encodings in strict mode 786 encodings = ['ascii', 'iso-8859-1', 'iso-8859-15', 'iso-8859-2'] 787 for encoding in encodings: 788 try: 789 dec = text.decode(encoding) 790 enc = dec.encode('utf-8') 791 return enc 792 except UnicodeError: 793 continue 794 795 # None of those worked, just do ascii with replace 796 dec = text.decode(encoding, 'replace') 797 enc = dec.encode('utf-8', 'replace') 798 return enc

799

Source Code for Module backend.server.repomd.mapper