backend.common.fileutils

1 # 2 # Copyright (c) 2008--2016 Red Hat, Inc. 3 # 4 # This software is licensed to you under the GNU General Public License, 5 # version 2 (GPLv2). There is NO WARRANTY for this software, express or 6 # implied, including the implied warranties of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 8 # along with this software; if not, see 9 # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. 10 # 11 # Red Hat trademarks are not licensed under GPLv2. No permission is 12 # granted to use or replicate Red Hat trademarks that are incorporated 13 # in this software or its documentation. 14 # 15 16 import os 17 import sys 18 import bz2 19 import gzip 20 import pwd 21 import grp 22 import shutil 23 import subprocess 24 import select 25 import stat 26 import tempfile 27 from spacewalk.common.checksum import getFileChecksum 28 from spacewalk.common.rhnLib import isSUSE 29 from spacewalk.common.usix import ListType, TupleType, MaxInt

30 31 -def cleanupAbsPath(path):

32 """ take ~taw/../some/path/$MOUNT_POINT/blah and make it sensible. 33 34 Path returned is absolute. 35 NOTE: python 2.2 fixes a number of bugs with this and eliminates 36 the need for os.path.expanduser 37 """ 38 39 if path is None: 40 return None 41 return os.path.abspath( 42 os.path.expanduser( 43 os.path.expandvars(path)))

44

45 46 -def cleanupNormPath(path, dotYN=0):

47 """ take ~taw/../some/path/$MOUNT_POINT/blah and make it sensible. 48 49 Returned path may be relative. 50 NOTE: python 2.2 fixes a number of bugs with this and eliminates 51 the need for os.path.expanduser 52 """ 53 if path is None: 54 return None 55 path = os.path.normpath( 56 os.path.expanduser( 57 os.path.expandvars(path))) 58 if dotYN and not (path and path[0] == '/'): 59 dirs = path.split('/') 60 if dirs[:1] not in (['.'], ['..']): 61 dirs = ['.'] + dirs 62 path = '/'.join(dirs) 63 return path

64

65 66 -def rotateFile(filepath, depth=5, suffix='.', verbosity=0):

67 """ backup/rotate a file 68 depth (-1==no limit) refers to num. of backups (rotations) to keep. 69 70 Behavior: 71 (1) 72 x.txt (current) 73 x.txt.1 (old) 74 x.txt.2 (older) 75 x.txt.3 (oldest) 76 (2) 77 all file stats preserved. Doesn't blow away original file. 78 (3) 79 if x.txt and x.txt.1 are identical (size or checksum), None is 80 returned 81 """ 82 83 # check argument sanity (should really be down outside of this function) 84 if not filepath or not isinstance(filepath, type('')): 85 raise ValueError("filepath '%s' is not a valid arguement" % filepath) 86 if not isinstance(depth, type(0)) or depth < -1 \ 87 or depth > MaxInt - 1 or depth == 0: 88 raise ValueError("depth must fall within range " 89 "[-1, 1...%s]" % (MaxInt - 1)) 90 91 # force verbosity to be a numeric value 92 verbosity = verbosity or 0 93 if not isinstance(verbosity, type(0)) or verbosity < -1 \ 94 or verbosity > MaxInt - 1: 95 raise ValueError('invalid verbosity value: %s' % (verbosity)) 96 97 filepath = cleanupAbsPath(filepath) 98 if not os.path.isfile(filepath): 99 raise ValueError("filepath '%s' does not lead to a file" % filepath) 100 101 pathNSuffix = filepath + suffix 102 pathNSuffix1 = pathNSuffix + '1' 103 104 if verbosity > 1: 105 sys.stderr.write("Working dir: %s\n" 106 % os.path.dirname(pathNSuffix)) 107 108 # is there anything to do? (existence, then size, then checksum) 109 checksum_type = 'sha1' 110 if os.path.exists(pathNSuffix1) and os.path.isfile(pathNSuffix1) \ 111 and os.stat(filepath)[6] == os.stat(pathNSuffix1)[6] \ 112 and getFileChecksum(checksum_type, filepath) == \ 113 getFileChecksum(checksum_type, pathNSuffix1): 114 # nothing to do 115 if verbosity: 116 sys.stderr.write("File '%s' is identical to its rotation. " 117 "Nothing to do.\n" % os.path.basename(filepath)) 118 return None 119 120 # find last in series (of rotations): 121 last = 0 122 while os.path.exists('%s%d' % (pathNSuffix, last + 1)): 123 last = last + 1 124 125 # percolate renames: 126 for i in range(last, 0, -1): 127 os.rename('%s%d' % (pathNSuffix, i), '%s%d' % (pathNSuffix, i + 1)) 128 if verbosity > 1: 129 filename = os.path.basename(pathNSuffix) 130 sys.stderr.write("Moving file: %s%d --> %s%d\n" % (filename, i, 131 filename, i + 1)) 132 133 # blow away excess rotations: 134 if depth != -1: 135 last = last + 1 136 for i in range(depth + 1, last + 1): 137 path = '%s%d' % (pathNSuffix, i) 138 os.unlink(path) 139 if verbosity: 140 sys.stderr.write("Rotated out: '%s'\n" % ( 141 os.path.basename(path))) 142 143 # do the actual rotation 144 shutil.copy2(filepath, pathNSuffix1) 145 if os.path.exists(pathNSuffix1) and verbosity: 146 sys.stderr.write("Backup made: '%s' --> '%s'\n" 147 % (os.path.basename(filepath), 148 os.path.basename(pathNSuffix1))) 149 150 # return the full filepath of the backed up file 151 return pathNSuffix1

152

153 154 -def rhn_popen(cmd, progressCallback=None, bufferSize=16384, outputLog=None):

155 """ popen-like function, that accepts execvp-style arguments too (i.e. an 156 array of params, thus making shell escaping unnecessary) 157 158 cmd can be either a string (like "ls -l /dev"), or an array of 159 arguments ["ls", "-l", "/dev"] 160 161 Returns the command's error code, a stream with stdout's contents 162 and a stream with stderr's contents 163 164 progressCallback --> progress bar twiddler 165 outputLog --> optional log file file object write method 166 """ 167 168 cmd_is_list = isinstance(cmd, (ListType, TupleType)) 169 if cmd_is_list: 170 cmd = list(map(str, cmd)) 171 # pylint: disable=E1101 172 c = subprocess.Popen(cmd, bufsize=0, stdin=subprocess.PIPE, 173 stdout=subprocess.PIPE, stderr=subprocess.PIPE, 174 close_fds=True, shell=(not cmd_is_list)) 175 176 # We don't write to the child process 177 c.stdin.close() 178 179 # Create two temporary streams to hold the info from stdout and stderr 180 child_out = tempfile.TemporaryFile(prefix='/tmp/my-popen-', mode='r+b') 181 child_err = tempfile.TemporaryFile(prefix='/tmp/my-popen-', mode='r+b') 182 183 # Map the input file descriptor with the temporary (output) one 184 fd_mappings = [(c.stdout, child_out), (c.stderr, child_err)] 185 exitcode = None 186 count = 1 187 188 while 1: 189 # Is the child process done? 190 status = c.poll() 191 if status is not None: 192 if status >= 0: 193 # Save the exit code, we still have to read from the pipes 194 exitcode = status 195 else: 196 # Some signal sent to this process 197 if outputLog is not None: 198 outputLog("rhn_popen: Signal %s received\n" % (-status)) 199 exitcode = status 200 break 201 202 fd_set = [x[0] for x in fd_mappings] 203 readfds = select.select(fd_set, [], [])[0] 204 205 for in_fd, out_fd in fd_mappings: 206 if in_fd in readfds: 207 # There was activity on this file descriptor 208 output = os.read(in_fd.fileno(), bufferSize) 209 if output: 210 # show progress 211 if progressCallback: 212 count = count + len(output) 213 progressCallback(count) 214 215 if outputLog is not None: 216 outputLog(output) 217 218 # write to the output buffer(s) 219 out_fd.write(output) 220 out_fd.flush() 221 222 if exitcode is not None: 223 # Child process is done 224 break 225 226 for f_in, f_out in fd_mappings: 227 f_in.close() 228 f_out.seek(0, 0) 229 230 return exitcode, child_out, child_err

231

232 233 -def makedirs(path, mode=int('0755', 8), user=None, group=None):

234 "makedirs function that also changes the owners" 235 236 dirs_to_create = [] 237 dirname = path 238 239 uid, gid = getUidGid(user, group) 240 241 while 1: 242 if os.path.isdir(dirname): 243 # We're done with this step 244 break 245 # We have to create this directory 246 dirs_to_create.append(dirname) 247 dirname, last = os.path.split(dirname) 248 if not last: 249 # We reached the top directory 250 break 251 252 # Now create the directories 253 while dirs_to_create: 254 dirname = dirs_to_create.pop() 255 try: 256 os.mkdir(dirname, mode) 257 except OSError: 258 e = sys.exc_info()[1] 259 if e.errno != 17: # File exists 260 raise 261 # Ignore the error 262 try: 263 os.chown(dirname, uid, gid) 264 except OSError: 265 # Changing permissions failed; ignore the error 266 sys.stderr.write("Changing owner for %s failed\n" % dirname)

267

268 -def createPath(path, user=None, group=None, chmod=int('0755', 8)):

269 """advanced makedirs 270 271 Will create the path if necessary. 272 Will chmod, and chown that path properly. 273 Defaults for user/group to the apache user 274 275 Uses the above makedirs() function. 276 """ 277 if isSUSE(): 278 if user is None: 279 user = 'wwwrun' 280 if group is None: 281 group = 'www' 282 else: 283 if user is None: 284 user = 'apache' 285 if group is None: 286 group = 'apache' 287 288 path = cleanupAbsPath(path) 289 if not os.path.exists(path): 290 makedirs(path, mode=chmod, user=user, group=group) 291 elif not os.path.isdir(path): 292 raise ValueError("ERROR: createPath('%s'): path doesn't lead to a directory" % str(path)) 293 else: 294 os.chmod(path, chmod) 295 uid, gid = getUidGid(user, group) 296 try: 297 os.chown(path, uid, gid) 298 except OSError: 299 # Changing permissions failed; ignore the error 300 sys.stderr.write("Changing owner for %s failed\n" % path)

301

302 303 -def setPermsPath(path, user=None, group='root', chmod=int('0750', 8)):

304 """chown user.group and set permissions to chmod""" 305 if isSUSE() and user is None: 306 user = 'wwwrun' 307 elif user is None: 308 user = 'apache' 309 310 if not os.path.exists(path): 311 raise OSError("*** ERROR: Path doesn't exist (can't set permissions): %s" % path) 312 313 # If non-root, don't bother to change owners 314 if os.getuid() != 0: 315 return 316 317 gc = GecosCache() 318 uid = gc.getuid(user) 319 if uid is None: 320 raise OSError("*** ERROR: user '%s' doesn't exist. Cannot set permissions properly." % user) 321 322 gid = gc.getgid(group) 323 if gid is None: 324 raise OSError("*** ERROR: group '%s' doesn't exist. Cannot set permissions properly." % group) 325 326 uid_, gid_ = os.stat(path)[4:6] 327 if uid_ != uid or gid_ != gid: 328 os.chown(path, uid, gid) 329 os.chmod(path, chmod)

330

331 332 -class GecosCache:

333 334 "Cache getpwnam() and getgrnam() calls" 335 __shared_data = {} 336

337 - def __init__(self):

338 self.__dict__ = self.__shared_data 339 if not list(self.__shared_data.keys()): 340 # Not initialized 341 self._users = {} 342 self._groups = {}

343

344 - def getuid(self, name):

345 "Return the UID of the user by name" 346 if name in self._users: 347 return self._users[name] 348 try: 349 uid = pwd.getpwnam(name)[2] 350 except KeyError: 351 # XXX misa: gripe? taw: I think we need to do something! 352 sys.stderr.write("XXX: User %s does not exist\n" % name) 353 return None 354 self._users[name] = uid 355 return uid

356

357 - def getgid(self, name):

358 "Return the GID of the group by name" 359 if name in self._groups: 360 return self._groups[name] 361 try: 362 gid = grp.getgrnam(name)[2] 363 except KeyError: 364 # XXX misa: gripe? 365 sys.stderr.write("XXX: Group %s does not exist\n" % name) 366 return None 367 self._groups[name] = gid 368 return gid

369

370 - def reset(self):

371 self.__shared_data.clear() 372 self.__init__()

373

374 375 -def getUidGid(user=None, group=None):

376 "return uid, gid given user and group" 377 378 gc = GecosCache() 379 uid = os.getuid() 380 if uid != 0: 381 # Don't bother to change the owner, it will fail anyway 382 # group ownership may work though 383 user = None 384 else: 385 uid = gc.getuid(user) 386 387 if group: 388 gid = gc.getgid(group) 389 else: 390 gid = None 391 392 if gid is None: 393 gid = os.getgid() 394 return uid, gid

395 396 # Duplicated in client/tools/rhncfg/config_common/file_utils.py to remove dependency 397 # requirement. If making changes make them there too. 398 FILETYPE2CHAR = { 399 'file': '-', 400 'directory': 'd', 401 'symlink': 'l', 402 'chardev': 'c', 403 'blockdev': 'b', 404 }

405 406 # Duplicated in client/tools/rhncfg/config_common/file_utils.py to remove dependency 407 # requirement. If making changes make them there too. 408 409 410 -def _ifelse(cond, thenval, elseval):

411 if cond: 412 return thenval 413 else: 414 return elseval

415

416 # Duplicated in client/tools/rhncfg/config_common/file_utils.py to remove dependency 417 # requirement. If making changes make them there too. 418 419 420 -def ostr_to_sym(octstr, ftype):

421 """ Convert filemode in octets (like '644') to string like "ls -l" ("-rwxrw-rw-") 422 ftype is one of: file, directory, symlink, chardev, blockdev. 423 """ 424 mode = int(str(octstr), 8) 425 426 symstr = FILETYPE2CHAR.get(ftype, '?') 427 428 symstr += _ifelse(mode & stat.S_IRUSR, 'r', '-') 429 symstr += _ifelse(mode & stat.S_IWUSR, 'w', '-') 430 symstr += _ifelse(mode & stat.S_IXUSR, 431 _ifelse(mode & stat.S_ISUID, 's', 'x'), 432 _ifelse(mode & stat.S_ISUID, 'S', '-')) 433 symstr += _ifelse(mode & stat.S_IRGRP, 'r', '-') 434 symstr += _ifelse(mode & stat.S_IWGRP, 'w', '-') 435 symstr += _ifelse(mode & stat.S_IXGRP, 436 _ifelse(mode & stat.S_ISGID, 's', 'x'), 437 _ifelse(mode & stat.S_ISGID, 'S', '-')) 438 symstr += _ifelse(mode & stat.S_IROTH, 'r', '-') 439 symstr += _ifelse(mode & stat.S_IWOTH, 'w', '-') 440 symstr += _ifelse(mode & stat.S_IXOTH, 441 _ifelse(mode & stat.S_ISVTX, 't', 'x'), 442 _ifelse(mode & stat.S_ISVTX, 'T', '-')) 443 return symstr

444

445 # Duplicated in client/tools/rhncfg/config_common/file_utils.py to remove dependency 446 # requirement. If making changes make them there too. 447 448 449 -def f_date(dbiDate):

450 return "%04d-%02d-%02d %02d:%02d:%02d" % (dbiDate.year, dbiDate.month, 451 dbiDate.day, dbiDate.hour, dbiDate.minute, dbiDate.second)

452

453 454 -class payload:

455 456 """ this class implements simple file like object usable for reading payload 457 from rpm, mpm, etc. 458 it skips first 'skip' bytes of header 459 """ 460

461 - def __init__(self, filename, skip=0):

462 self.fileobj = open(filename, 'r') 463 self.skip = skip 464 self.seek(0)

465

466 - def seek(self, offset, whence=0):

467 if whence == 0: 468 offset += self.skip 469 return self.fileobj.seek(offset, whence)

470

471 - def tell(self):

472 return self.fileobj.tell() - self.skip

473 474 @staticmethod

475 - def truncate(size=-1):

476 # pylint: disable=W0613 477 raise AttributeError("'Payload' object do not implement this method")

478 479 @staticmethod

480 - def write(_s):

481 raise AttributeError("'Payload' object do not implement this method")

482 483 @staticmethod

484 - def writelines(_seq):

485 raise AttributeError("'Payload' object do not implement this method")

486

487 - def __getattr__(self, x):

488 return getattr(self.fileobj, x)

489

490 491 -def decompress_open(filename, mode='r'):

492 file_obj = None 493 if filename.endswith('.gz'): 494 file_obj = gzip.open(filename, mode) 495 elif filename.endswith('.bz2'): 496 file_obj = bz2.BZ2File(filename, mode) 497 elif filename.endswith('.xz'): 498 try: 499 # pylint: disable=F0401,E1101 500 import lzma 501 file_obj = lzma.LZMAFile(filename, mode) 502 except ImportError: # No LZMA lib - be sad 503 # xz uncompresses foo.xml.xz to foo.xml 504 # uncompress, keep both, return uncompressed file 505 subprocess.call(['xz', '-d', '-k', filename]) 506 uncompressed_path = filename.rsplit('.', 1)[0] 507 file_obj = open(uncompressed_path, mode) 508 else: 509 file_obj = open(filename, mode) 510 return file_obj

511

Source Code for Module backend.common.fileutils