backend.server.repomd.repository

1 # 2 # Copyright (c) 2008--2018 Red Hat, Inc. 3 # 4 # This software is licensed to you under the GNU General Public License, 5 # version 2 (GPLv2). There is NO WARRANTY for this software, express or 6 # implied, including the implied warranties of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 8 # along with this software; if not, see 9 # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. 10 # 11 # Red Hat trademarks are not licensed under GPLv2. No permission is 12 # granted to use or replicate Red Hat trademarks that are incorporated 13 # in this software or its documentation. 14 # 15 # 16 # Classes for generating repository metadata from RHN info. 17 # 18 19 import time 20 try: 21 # python 2 22 import StringIO 23 except ImportError: 24 # python3 25 import io as StringIO 26 import shutil 27 import os.path 28 29 from gzip import GzipFile 30 from gzip import write32u 31 32 from spacewalk.common.usix import LongType 33 from spacewalk.common import checksum 34 from spacewalk.common import rhnCache 35 from spacewalk.common.rhnLog import log_debug 36 from spacewalk.common.rhnConfig import CFG 37 38 import mapper 39 import view 40 from domain import RepoMD 41 from spacewalk.server import rhnChannel 42 43 # One meg 44 CHUNK_SIZE = 1048576 45 46 comps_mapping = { 47 'rhel-x86_64-client-5': 'rhn/kickstart/ks-rhel-x86_64-client-5/Client/repodata/comps-rhel5-client-core.xml', 48 'rhel-x86_64-client-vt-5': 'rhn/kickstart/ks-rhel-x86_64-client-5/VT/repodata/comps-rhel5-vt.xml', 49 'rhel-x86_64-client-workstation-5': 'rhn/kickstart/ks-rhel-x86_64-client-5/Workstation/repodata/comps-rhel5-client-workstation.xml', 50 'rhel-x86_64-server-5': 'rhn/kickstart/ks-rhel-x86_64-server-5/Server/repodata/comps-rhel5-server-core.xml', 51 'rhel-x86_64-server-vt-5': 'rhn/kickstart/ks-rhel-x86_64-server-5/VT/repodata/comps-rhel5-vt.xml', 52 'rhel-x86_64-server-cluster-5': 'rhn/kickstart/ks-rhel-x86_64-server-5/Cluster/repodata/comps-rhel5-cluster.xml', 53 'rhel-x86_64-server-cluster-storage-5': 'rhn/kickstart/ks-rhel-x86_64-server-5/ClusterStorage/repodata/comps-rhel5-cluster-st.xml', 54 } 55 for k in comps_mapping.keys(): 56 for arch in ('i386', 'ia64', 's390x', 'ppc'): 57 comps_mapping[k.replace('x86_64', arch)] = comps_mapping[k].replace('x86_64', arch) 58 59

60 -class Repository(object):

61 62 """ 63 Representation of RHN channels as repository metadata. 64 65 This class can generate primary.xml, filelists.xml, and other.xml 66 """ 67

68 - def __init__(self, channel):

69 self.channel_id = channel['id'] 70 self.last_modified = channel['last_modified'] 71 72 self.primary_prefix = "repomd_primary.xml" 73 self.other_prefix = "repomd_other.xml" 74 self.filelists_prefix = "repomd_filelists.xml" 75 self.updateinfo_prefix = "repomd_updateinfo.xml" 76 77 self._channel = None 78 79 cache = rhnCache.Cache() 80 self.cache = rhnCache.NullCache(cache)

81

82 - def get_primary_xml_file(self):

83 """ Return a file-like object of the primarl.xml for this channel. """ 84 ret = self.get_primary_cache() 85 86 if not ret: 87 viewobj = self.get_primary_view() 88 89 self.generate_files([viewobj]) 90 ret = self.get_primary_cache() 91 92 return ret

93

94 - def get_other_xml_file(self):

95 """ Return a file-like object of the other.xml for this channel. """ 96 ret = self.get_other_cache() 97 98 if not ret: 99 viewobj = self.get_other_view() 100 101 self.generate_files([viewobj]) 102 ret = self.get_other_cache() 103 104 return ret

105

106 - def get_filelists_xml_file(self):

107 """ Return a file-like object of the filelists.xml for this channel. """ 108 ret = self.get_filelists_cache() 109 110 if not ret: 111 viewobj = self.get_filelists_view() 112 113 self.generate_files([viewobj]) 114 ret = self.get_filelists_cache() 115 116 return ret

117

118 - def get_updateinfo_xml_file(self):

119 """ Return a file-like object of the updateinfo.xml for the channel. """ 120 ret = self.get_cache_file(self.updateinfo_prefix) 121 122 if not ret: 123 viewobj = self.get_cache_view(self.updateinfo_prefix, 124 view.UpdateinfoView) 125 126 viewobj.write_updateinfo() 127 viewobj.fileobj.close() 128 ret = self.get_cache_file(self.updateinfo_prefix) 129 130 return ret

131

132 - def get_cache_entry_name(self, cache_prefix):

133 return "%s-%s" % (cache_prefix, self.channel_id)

134

135 - def get_cache_file(self, cache_prefix):

136 cache_entry = self.get_cache_entry_name(cache_prefix) 137 ret = self.cache.get_file(cache_entry, self.last_modified) 138 return ret

139

140 - def get_cache_view(self, cache_prefix, view_class):

141 cache_entry = self.get_cache_entry_name(cache_prefix) 142 ret = self.cache.set_file(cache_entry, self.last_modified) 143 viewobj = view_class(self.channel, ret) 144 return viewobj

145

146 - def get_primary_cache(self):

147 return self.get_cache_file(self.primary_prefix)

148

149 - def get_other_cache(self):

150 return self.get_cache_file(self.other_prefix)

151

152 - def get_filelists_cache(self):

153 return self.get_cache_file(self.filelists_prefix)

154

155 - def get_primary_view(self):

156 return self.get_cache_view(self.primary_prefix, view.PrimaryView)

157

158 - def get_other_view(self):

159 return self.get_cache_view(self.other_prefix, view.OtherView)

160

161 - def get_filelists_view(self):

162 return self.get_cache_view(self.filelists_prefix, view.FilelistsView)

163

164 - def get_repomd_file(self, repomd_obj, func_name):

165 """ Return a file-like object of the comps.xml/modules.yaml for the channel. """ 166 if repomd_obj: 167 repomd_view = view.RepoMDView(repomd_obj) 168 return repomd_view.get_file() 169 elif func_name == 'get_comps_file' and self.channel.label in comps_mapping: 170 comps_view = view.RepoMDView(RepoMD(None, 171 os.path.join(CFG.mount_point, comps_mapping[self.channel.label]))) 172 return comps_view.get_file() 173 else: 174 if self.channel.cloned_from_id is not None: 175 log_debug(1, "No comps/modules and no comps_mapping for [%s] cloned from [%s] trying to get comps from the original one." 176 % (self.channel.id, self.channel.cloned_from_id)) 177 cloned_from_channel = rhnChannel.Channel().load_by_id(self.channel.cloned_from_id) 178 cloned_from_channel_label = cloned_from_channel._row['label'] 179 func = getattr(Repository(rhnChannel.channel_info(cloned_from_channel_label)), func_name) 180 return func() 181 return None

182

183 - def get_comps_file(self):

184 return self.get_repomd_file(self.channel.comps, 'get_comps_file')

185

186 - def get_modules_file(self):

187 return self.get_repomd_file(self.channel.modules, 'get_modules_file')

188

189 - def generate_files(self, views):

190 for view in views: 191 view.write_start() 192 193 for package in self.channel.packages: 194 for view in views: 195 view.write_package(package) 196 197 for view in views: 198 view.write_end() 199 view.fileobj.close()

200

201 - def __get_channel(self):

202 """ Late binding for the channel. """ 203 if self._channel is None: 204 channel_mapper = mapper.get_channel_mapper() 205 self._channel = channel_mapper.get_channel(self.channel_id) 206 return self._channel

207 208 channel = property(__get_channel)

209 210

211 -class CompressedRepository:

212 213 """ Decorator for Repositories adding gzip compression of the output. """ 214

215 - def __init__(self, repository):

216 self.repository = repository 217 218 self.primary_prefix = self.repository.primary_prefix + ".gz" 219 self.other_prefix = self.repository.other_prefix + ".gz" 220 self.filelists_prefix = self.repository.filelists_prefix + ".gz" 221 self.updateinfo_prefix = self.repository.updateinfo_prefix + ".gz"

222

223 - def get_primary_xml_file(self):

224 xml_file = self.repository.get_primary_xml_file() 225 return self.__get_compressed_file(xml_file)

226

227 - def get_other_xml_file(self):

228 """ Return gzipped other.xml file """ 229 xml_file = self.repository.get_other_xml_file() 230 return self.__get_compressed_file(xml_file)

231

232 - def get_filelists_xml_file(self):

233 """ Return gzipped filelists.xml file """ 234 xml_file = self.repository.get_filelists_xml_file() 235 return self.__get_compressed_file(xml_file)

236

237 - def get_updateinfo_xml_file(self):

238 """ Return gzipped updateinfo.xml file """ 239 xml_file = self.repository.get_updateinfo_xml_file() 240 return self.__get_compressed_file(xml_file)

241

242 - def __getattr__(self, x):

243 return getattr(self.repository, x)

244

245 - def __get_compressed_file(self, uncompressed_file):

246 string_file = StringIO.StringIO() 247 gzip_file = NoTimeStampGzipFile(mode="wb", fileobj=string_file) 248 249 shutil.copyfileobj(uncompressed_file, gzip_file) 250 251 gzip_file.close() 252 253 string_file.seek(0, 0) 254 255 return string_file

256 257

258 -class CachedRepository:

259 260 """ Decorator for Repositories adding caching. """ 261

262 - def __init__(self, repository):

263 self.repository = repository 264 265 cache = rhnCache.Cache() 266 self.cache = rhnCache.NullCache(cache)

267

268 - def get_primary_xml_file(self):

269 """ Return the cached primary metadata file, if it exists. """ 270 return self._cached(self.primary_prefix, 271 self.repository.get_primary_xml_file)

272

273 - def get_other_xml_file(self):

274 return self._cached(self.other_prefix, 275 self.repository.get_other_xml_file)

276

277 - def get_filelists_xml_file(self):

278 return self._cached(self.filelists_prefix, 279 self.repository.get_filelists_xml_file)

280

281 - def get_updateinfo_xml_file(self):

282 return self._cached(self.updateinfo_prefix, 283 self.repository.get_updateinfo_xml_file)

284

285 - def _cached(self, cache_prefix, fallback_method):

286 """ 287 Return the cached results if they are new enough, else get new results. 288 289 cache_prefix is a unique string that will identify the cached data. 290 fallback_method is the method to call if the cached data doesn't exist 291 or isn't new enough. 292 """ 293 cache_entry = "%s-%s" % (cache_prefix, self.channel_id) 294 ret = self.cache.get_file(cache_entry, self.last_modified) 295 if ret: 296 log_debug(4, "Scored cache hit", self.channel_id) 297 else: 298 ret = fallback_method() 299 cache_file = self.cache.set_file(cache_entry, self.last_modified) 300 301 shutil.copyfileobj(ret, cache_file) 302 303 ret.close 304 cache_file.close() 305 ret = self.cache.get_file(cache_entry, self.last_modified) 306 return ret

307

308 - def __getattr__(self, x):

309 return getattr(self.repository, x)

310 311

312 -class MetadataRepository:

313 314 """ 315 A repository that can provide repomd data. 316 317 A Metadata Repository is composed of a repository and a 318 CompressedRepository, as both are required to generate the repomd file. 319 """ 320

321 - def __init__(self, repository, compressed_repository):

322 self.repository = repository 323 self.compressed_repository = compressed_repository 324 325 self.repomd_prefix = "repomd.xml"

326

327 - def get_repomd_file(self):

328 """ Return uncompressed repomd.xml file """ 329 330 cache_entry = "%s-%s" % (self.repomd_prefix, self.channel_id) 331 ret = self.cache.get_file(cache_entry, self.last_modified) 332 333 if not ret: 334 # We need the time in seconds since the epoch for the xml file. 335 timestamp = int(time.mktime(time.strptime(self.last_modified, 336 "%Y%m%d%H%M%S"))) 337 338 to_generate = [] 339 340 if not self.repository.get_primary_cache(): 341 to_generate.append(self.repository.get_primary_view()) 342 if not self.repository.get_other_cache(): 343 to_generate.append(self.repository.get_other_view()) 344 if not self.repository.get_filelists_cache(): 345 to_generate.append(self.repository.get_filelists_view()) 346 347 self.repository.generate_files(to_generate) 348 349 primary = self.__compute_checksums(timestamp, 350 self.repository.get_primary_xml_file(), 351 self.compressed_repository.get_primary_xml_file()) 352 353 filelists = self.__compute_checksums(timestamp, 354 self.repository.get_filelists_xml_file(), 355 self.compressed_repository.get_filelists_xml_file()) 356 357 other = self.__compute_checksums(timestamp, 358 self.repository.get_other_xml_file(), 359 self.compressed_repository.get_other_xml_file()) 360 361 updateinfo = self.__compute_checksums(timestamp, 362 self.repository.get_updateinfo_xml_file(), 363 self.compressed_repository.get_updateinfo_xml_file()) 364 365 # Comps and modules might not exist on disc 366 comps = None 367 comps_file = None 368 modules = None 369 modules_file = None 370 try: 371 comps_file = self.repository.get_comps_file() 372 except IOError: 373 pass 374 try: 375 modules_file = self.repository.get_modules_file() 376 except IOError: 377 pass 378 if comps_file: 379 comps = self.__compute_open_checksum(timestamp, comps_file) 380 if modules_file: 381 modules = self.__compute_checksums(timestamp, modules_file) 382 383 ret = self.cache.set_file(cache_entry, self.last_modified) 384 repomd_view = view.RepoView(primary, filelists, other, updateinfo, 385 comps, modules, ret, self.__get_checksumtype()) 386 387 repomd_view.write_repomd() 388 ret.close() 389 ret = self.cache.get_file(cache_entry, self.last_modified) 390 391 return ret

392

393 - def __get_file_checksum(self, xml_file):

394 hash_computer = checksum.getHashlibInstance(self.__get_checksumtype(), False) 395 396 chunk = xml_file.read(CHUNK_SIZE) 397 while chunk: 398 hash_computer.update(chunk) 399 chunk = xml_file.read(CHUNK_SIZE) 400 401 return hash_computer.hexdigest()

402

403 - def __compute_open_checksum(self, timestamp, xml_file):

404 template_hash = {} 405 406 template_hash['open_checksum'] = self.__get_file_checksum(xml_file) 407 template_hash['timestamp'] = timestamp 408 409 return template_hash

410

411 - def __compute_checksums(self, timestamp, xml_file, xml_gz_file):

412 template_hash = self.__compute_open_checksum(timestamp, xml_file) 413 414 template_hash['gzip_checksum'] = self.__get_file_checksum(xml_gz_file) 415 416 return template_hash

417

418 - def __get_checksumtype(self):

419 return self.repository.channel.checksum_type

420

421 - def __getattr__(self, x):

422 return getattr(self.compressed_repository, x)

423 424

425 -def get_repository(channel):

426 """ Factory Method-ish function to create a repository from a channel. """ 427 repository = Repository(channel) 428 429 compressed_repository = CompressedRepository(repository) 430 compressed_repository = CachedRepository(compressed_repository) 431 432 meta_repository = MetadataRepository(repository, compressed_repository) 433 434 return meta_repository

435 436

437 -class NoTimeStampGzipFile(GzipFile):

438

439 - def _write_gzip_header(self):

440 self.fileobj.write('\037\213') 441 self.fileobj.write('\010') 442 # no flags 443 self.fileobj.write('\x00') 444 write32u(self.fileobj, LongType(0)) 445 self.fileobj.write('\002') 446 self.fileobj.write('\377')

447

Source Code for Module backend.server.repomd.repository