backend.common.rhnCache

1 # Copyright (c) 2008--2016 Red Hat, Inc. 2 # 3 # This software is licensed to you under the GNU General Public License, 4 # version 2 (GPLv2). There is NO WARRANTY for this software, express or 5 # implied, including the implied warranties of MERCHANTABILITY or FITNESS 6 # FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 7 # along with this software; if not, see 8 # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. 9 # 10 # Red Hat trademarks are not licensed under GPLv2. No permission is 11 # granted to use or replicate Red Hat trademarks that are incorporated 12 # in this software or its documentation. 13 # 14 15 # This module implements a simple object caching system using shelves 16 # stored in files on the file system 17 # 18 19 import os 20 import gzip 21 try: 22 # python 2 23 import cPickle 24 except ImportError: 25 # python3 26 import pickle as cPickle 27 import fcntl 28 import sys 29 from stat import ST_MTIME 30 from errno import EEXIST 31 32 from spacewalk.common.rhnLib import timestamp 33 34 from spacewalk.common.usix import raise_with_tb 35 from spacewalk.common.fileutils import makedirs, setPermsPath 36 37 # this is a constant I'm not too happy about but one way or another we have 38 # to reserve our own shared memory space. 39 CACHEDIR = "/var/cache/rhn"

40 41 42 -def cleanupPath(path):

43 """take ~taw/../some/path/$MOUNT_POINT/blah and make it sensible.""" 44 if path is None: 45 return None 46 return os.path.normpath( 47 os.path.expanduser( 48 os.path.expandvars(path)))

49

50 # build a filename for storing the key - eventually this is going to get 51 # more compelx as we observe issues 52 53 54 -def _fname(name):

55 fname = "%s/%s" % (CACHEDIR, name) 56 return cleanupPath(fname)

57

58 59 -def _unlock(fd):

60 try: 61 fcntl.lockf(fd, fcntl.LOCK_UN) 62 except IOError: 63 # If LOCK is not relinquished try flock, 64 # its usually more forgiving. 65 fcntl.flock(fd, fcntl.LOCK_UN)

66

67 # The following functions expose this module as a dictionary 68 69 70 -def get(name, modified=None, raw=None, compressed=None, missing_is_null=1):

71 cache = __get_cache(raw, compressed) 72 73 if missing_is_null: 74 cache = NullCache(cache) 75 76 return cache.get(name, modified)

77

78 79 -def set(name, value, modified=None, raw=None, compressed=None, 80 user='root', group='root', mode=int('0755', 8)):

81 # pylint: disable=W0622 82 cache = __get_cache(raw, compressed) 83 84 cache.set(name, value, modified, user, group, mode)

85

86 87 -def has_key(name, modified=None):

88 cache = Cache() 89 return cache.has_key(name, modified)

90

91 92 -def delete(name):

93 cache = Cache() 94 cache.delete(name)

95

96 97 -def __get_cache(raw, compressed):

98 cache = Cache() 99 if compressed: 100 cache = CompressedCache(cache) 101 if not raw: 102 cache = ObjectCache(cache) 103 104 return cache

105

106 107 -class UnreadableFileError(Exception):

108 pass

109

110 111 -def _safe_create(fname, user, group, mode):

112 """ This function returns a file descriptor for the open file fname 113 If the file is already there, it is truncated 114 otherwise, all the directories up to it are created and the file is created 115 as well. 116 """ 117 118 # There can be race conditions between the moment we check for the file 119 # existence and when we actually create it, so retry if something fails 120 tries = 5 121 while tries: 122 tries = tries - 1 123 # we're really picky about what can we do 124 if os.access(fname, os.F_OK): # file exists 125 if not os.access(fname, os.R_OK | os.W_OK): 126 raise UnreadableFileError() 127 128 fd = os.open(fname, os.O_WRONLY | os.O_TRUNC) 129 # We're done 130 return fd 131 132 # If directory does not exist, attempt to create it 133 dirname = os.path.dirname(fname) 134 if not os.path.isdir(dirname): 135 try: 136 #os.makedirs(dirname, 0755) 137 makedirs(dirname, mode, user, group) 138 except OSError: 139 e = sys.exc_info()[1] 140 # There is a window between the moment we check the disk and 141 # the one we try to create the directory 142 # We double-check the file existance here 143 if not (e.errno == EEXIST and os.path.isdir(dirname)): 144 # If the exception was thrown on a parent dir 145 # check the subdirectory to go through next loop. 146 if os.path.isdir(e.filename): 147 continue 148 # Pass exception through 149 raise 150 except: 151 # Pass exception through 152 raise 153 # If we got here, it means the directory exists 154 155 # file does not exist, attempt to create it 156 # we pass most of the exceptions through 157 try: 158 fd = os.open(fname, os.O_WRONLY | os.O_CREAT | os.O_EXCL, int('0644', 8)) 159 except OSError: 160 e = sys.exc_info()[1] 161 # The file may be already there 162 if e.errno == EEXIST and os.access(fname, os.F_OK): 163 # Retry 164 continue 165 # Pass exception through 166 raise 167 # If we got here, the file is created, so break out of the loop 168 setPermsPath(fname, user, group, mode) 169 return fd 170 171 # Ran out of tries; something is fishy 172 # (if we manage to create or truncate the file, we've returned from the 173 # function already) 174 raise RuntimeError("Attempt to create file %s failed" % fname)

175

176 177 -class LockedFile(object):

178

179 - def __init__(self, name, modified=None, user='root', group='root', 180 mode=int('0755', 8)):

181 if modified: 182 self.modified = timestamp(modified) 183 else: 184 self.modified = None 185 186 self.fname = _fname(name) 187 self.fd = self.get_fd(name, user, group, mode) 188 189 self.closed = False

190

191 - def close(self):

192 if not self.closed: 193 self.close_fd() 194 195 _unlock(self.fd.fileno()) 196 self.fd.close() 197 self.closed = True

198

199 - def get_fd(self, name, user, group, mode):

200 raise NotImplementedError

201

202 - def close_fd(self):

203 raise NotImplementedError

204

205 - def __getattr__(self, x):

206 return getattr(self.fd, x)

207

208 209 -class ReadLockedFile(LockedFile):

210

211 - def get_fd(self, name, _user, _group, _mode):

212 if not os.access(self.fname, os.R_OK): 213 raise KeyError(name) 214 fd = open(self.fname, "r") 215 216 fcntl.lockf(fd.fileno(), fcntl.LOCK_SH) 217 218 if self.modified: 219 if os.fstat(fd.fileno())[ST_MTIME] != self.modified: 220 fd.close() 221 raise KeyError(name) 222 223 return fd

224

225 - def close_fd(self):

226 pass

227

228 229 -class WriteLockedFile(LockedFile):

230

231 - def get_fd(self, name, user, group, mode):

232 try: 233 fd = _safe_create(self.fname, user, group, mode) 234 except UnreadableFileError: 235 raise_with_tb(OSError("cache entry exists, but is not accessible: %s" % \ 236 name), sys.exc_info()[2]) 237 238 # now we have the fd open, lock it 239 fcntl.lockf(fd, fcntl.LOCK_EX) 240 return os.fdopen(fd, 'w')

241

242 - def close_fd(self):

243 # Set the file's mtime if necessary 244 self.flush() 245 if self.modified: 246 os.utime(self.fname, (self.modified, self.modified))

247

248 249 -class Cache:

250

251 - def __init__(self):

252 pass

253

254 - def get(self, name, modified=None):

255 fd = self.get_file(name, modified) 256 257 s = fd.read() 258 fd.close() 259 260 return s

261

262 - def set(self, name, value, modified=None, user='root', group='root', 263 mode=int('0755', 8)):

264 fd = self.set_file(name, modified, user, group, mode) 265 266 fd.write(value) 267 fd.close()

268 269 @staticmethod

270 - def has_key(name, modified=None):

271 fname = _fname(name) 272 if modified is not None: 273 modified = timestamp(modified) 274 if not os.access(fname, os.R_OK): 275 return False 276 # the file exists, so os.stat should not raise an exception 277 statinfo = os.stat(fname) 278 if modified is not None and statinfo[ST_MTIME] != modified: 279 return False 280 return True

281 282 @staticmethod

283 - def delete(name):

284 fname = _fname(name) 285 # test for valid entry 286 if not os.access(fname, os.R_OK): 287 raise KeyError("Invalid cache key for delete: %s" % name) 288 # now can we delete it? 289 if not os.access(fname, os.W_OK): 290 raise OSError("Read-Only access for cache entry: %s" % name) 291 os.unlink(fname)

292 293 @staticmethod

294 - def get_file(name, modified=None):

295 fd = ReadLockedFile(name, modified) 296 return fd

297 298 @staticmethod

299 - def set_file(name, modified=None, user='root', group='root', 300 mode=int('0755', 8)):

301 fd = WriteLockedFile(name, modified, user, group, mode) 302 return fd

303

304 305 -class ClosingZipFile(object):

306 307 """ Like a GzipFile, but close closes both files. """ 308

309 - def __init__(self, mode, io):

310 self.zipfile = gzip.GzipFile(None, mode, 5, io) 311 self.rawfile = io

312

313 - def close(self):

314 self.zipfile.close() 315 self.rawfile.close()

316

317 - def __getattr__(self, x):

318 return getattr(self.zipfile, x)

319

320 321 -class CompressedCache:

322

323 - def __init__(self, cache):

324 self.cache = cache

325

326 - def get(self, name, modified=None):

327 fd = self.get_file(name, modified) 328 try: 329 value = fd.read() 330 except (ValueError, IOError, gzip.zlib.error): 331 # Some gzip error 332 # poking at gzip.zlib may not be such a good idea 333 fd.close() 334 raise_with_tb(KeyError(name), sys.exc_info()[2]) 335 fd.close() 336 337 return value

338

339 - def set(self, name, value, modified=None, user='root', group='root', 340 mode=int('0755', 8)):

341 # Since most of the data is kept in memory anyway, don't bother to 342 # write it to a temp file at this point 343 f = self.set_file(name, modified, user, group, mode) 344 f.write(value) 345 f.close()

346

347 - def has_key(self, name, modified=None):

348 return self.cache.has_key(name, modified)

349

350 - def delete(self, name):

351 self.cache.delete(name)

352

353 - def get_file(self, name, modified=None):

354 compressed_file = self.cache.get_file(name, modified) 355 return ClosingZipFile('r', compressed_file)

356

357 - def set_file(self, name, modified=None, user='root', group='root', 358 mode=int('0755', 8)):

359 io = self.cache.set_file(name, modified, user, group, mode) 360 361 f = ClosingZipFile('w', io) 362 return f

363

364 365 -class ObjectCache:

366

367 - def __init__(self, cache):

368 self.cache = cache

369

370 - def get(self, name, modified=None):

371 pickled = self.cache.get(name, modified) 372 373 try: 374 return cPickle.loads(pickled) 375 except cPickle.UnpicklingError: 376 raise_with_tb(KeyError(name), sys.exc_info()[2])

377

378 - def set(self, name, value, modified=None, user='root', group='root', 379 mode=int('0755', 8)):

380 pickled = cPickle.dumps(value, -1) 381 self.cache.set(name, pickled, modified, user, group, mode)

382

383 - def has_key(self, name, modified=None):

384 return self.cache.has_key(name, modified)

385

386 - def delete(self, name):

387 self.cache.delete(name)

388 389 @staticmethod

390 - def get_file(*_args):

391 raise RuntimeError("Getting a file descriptor for an object makes no sense.")

392

393 394 -class NullCache:

395 396 """ A cache that returns None rather than raises a KeyError. """ 397

398 - def __init__(self, cache):

399 self.cache = cache

400

401 - def get(self, name, modified=None):

402 try: 403 return self.cache.get(name, modified) 404 except KeyError: 405 return None

406

407 - def set(self, name, value, modified=None, user='root', group='root', 408 mode=int('0755', 8)):

409 self.cache.set(name, value, modified, user, group, mode)

410

411 - def has_key(self, name, modified=None):

412 return self.cache.has_key(name, modified)

413

414 - def delete(self, name):

415 self.cache.delete(name)

416

417 - def get_file(self, name, modified=None):

418 try: 419 return self.cache.get_file(name, modified) 420 except KeyError: 421 return None

422

423 - def set_file(self, name, modified=None, user='root', group='root', 424 mode=int('0755', 8)):

425 return self.cache.set_file(name, modified, user, group, mode)

426

Source Code for Module backend.common.rhnCache