backend.satellite_tools.syncLib

1 # 2 # Copyright (c) 2008--2017 Red Hat, Inc. 3 # 4 # This software is licensed to you under the GNU General Public License, 5 # version 2 (GPLv2). There is NO WARRANTY for this software, express or 6 # implied, including the implied warranties of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 8 # along with this software; if not, see 9 # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. 10 # 11 # Red Hat trademarks are not licensed under GPLv2. No permission is 12 # granted to use or replicate Red Hat trademarks that are incorporated 13 # in this software or its documentation. 14 # 15 16 # system imports: 17 import os 18 import sys 19 import time 20 21 try: 22 # python 2 23 from StringIO import StringIO 24 except ImportError: 25 # python3 26 from io import StringIO 27 28 # rhn imports: 29 from spacewalk.common.usix import raise_with_tb 30 from spacewalk.common import rhnLib 31 from spacewalk.common.rhnConfig import CFG 32 from spacewalk.common.rhnLog import log_time, log_clean 33 from spacewalk.common.fileutils import createPath, setPermsPath 34 35 import messages 36 37 EMAIL_LOG = None 38 39

40 -def initEMAIL_LOG(reinit=0):

41 global EMAIL_LOG 42 if EMAIL_LOG is None or reinit: 43 EMAIL_LOG = StringIO()

44 45

46 -def dumpEMAIL_LOG():

47 if EMAIL_LOG is not None: 48 return EMAIL_LOG.getvalue() 49 return None

50 51

52 -class RhnSyncException(Exception):

53 54 """General exception handler for all sync activity.""" 55 pass

56 57

58 -class ReprocessingNeeded(Exception):

59 60 """Exception raised when a contition has been hit that would require a new 61 run of the sync process""" 62 pass

63 64 # logging functions: 65 # log levels rule of thumb: 66 # 0 - no logging, yet no feedback either 67 # 1 - minimal logging/feedback 68 # 2 - normal level of logging/feedback 69 # 3 - a bit much 70 ## 4+ - excessive 71 72

73 -def _timeString1():

74 """time string as: "2002/11/18 12:56:34" """ 75 return log_time()

76 77

78 -def _timeString2():

79 """time string as: "12:56:34" """ 80 return time.strftime("%H:%M:%S", time.localtime(time.time()))

81 82

83 -def _prepLogMsg(msg, cleanYN=0, notimeYN=0, shortYN=0):

84 """prepare formating of message for logging. 85 86 cleanYN - no extra info, period. 87 notimeYN - spaced as if there were a time-stamp. 88 shortYN - no date (used for stdout/stderr really) 89 """ 90 if not cleanYN: 91 if shortYN: 92 if notimeYN: 93 msg = '%s %s' % (' ' * len(_timeString2()), msg) 94 else: 95 msg = '%s %s' % (_timeString2(), msg) 96 else: 97 if notimeYN: 98 msg = '%s %s' % (' ' * len(_timeString1()), msg) 99 else: 100 msg = '%s %s' % (_timeString1(), msg) 101 return msg

102 103

104 -def log2disk(level, msg, cleanYN=0, notimeYN=0):

105 """Log to a log file. 106 Arguments: see def _prepLogMsg(...) above. 107 """ 108 if not isinstance(msg, type([])): 109 msg = [msg] 110 for m in msg: 111 try: 112 log_clean(level=level, msg=_prepLogMsg(m, cleanYN, notimeYN)) 113 except (KeyboardInterrupt, SystemExit): 114 raise 115 except Exception: # pylint: disable=E0012, W0703 116 e = sys.exc_info()[1] 117 sys.stderr.write('ERROR: upon attempt to write to log file: %s' % e)

118 119

120 -def log2stream(level, msg, cleanYN, notimeYN, stream):

121 """Log to a specified stream. 122 Arguments: see def _prepLogMsg(...) above. 123 """ 124 if not isinstance(msg, type([])): 125 msg = [msg] 126 if CFG.DEBUG >= level: 127 for m in msg: 128 stream.write(_prepLogMsg(m, cleanYN, notimeYN, shortYN=1) + '\n') 129 stream.flush()

130 131

132 -def log2email(level, msg, cleanYN=0, notimeYN=0):

133 """ Log to the email log. 134 Arguments: see def _prepLogMsg(...) above. 135 """ 136 if EMAIL_LOG is not None: 137 log2stream(level, msg, cleanYN, notimeYN, EMAIL_LOG)

138 139

140 -def log2background(level, msg, cleanYN=0, notimeYN=0):

141 """Log to email and disk 142 Arguments: see def _prepLogMsg(...) above. 143 """ 144 log2email(level, msg, cleanYN, notimeYN) 145 log2disk(level, msg, cleanYN, notimeYN)

146 147

148 -def log2stderr(level, msg, cleanYN=0, notimeYN=0):

149 """Log to standard error 150 Arguments: see def _prepLogMsg(...) above. 151 """ 152 log2email(level, msg, cleanYN, notimeYN) 153 log2stream(level, msg, cleanYN, notimeYN, sys.stderr)

154 155

156 -def log2stdout(level, msg, cleanYN=0, notimeYN=0):

157 """Log to standard out 158 Arguments: see def _prepLogMsg(...) above. 159 """ 160 log2email(level, msg, cleanYN, notimeYN) 161 log2stream(level, msg, cleanYN, notimeYN, sys.stdout)

162 163

164 -def log2(levelDisk, levelStream, msg, cleanYN=0, notimeYN=0, stream=sys.stdout):

165 """Log to disk and some stream --- differing log levels. 166 Arguments: see def _prepLogMsg(...) above. 167 """ 168 log2disk(levelDisk, msg, cleanYN, notimeYN) 169 if stream is sys.stdout: 170 log2stdout(levelStream, msg, cleanYN, notimeYN) 171 elif stream is sys.stderr: 172 log2stderr(levelStream, msg, cleanYN, notimeYN) 173 else: 174 log2stream(levelStream, msg, cleanYN, notimeYN, stream=stream)

175 176

177 -def log(level, msg, cleanYN=0, notimeYN=0, stream=sys.stdout):

178 """Log to disk and some stream --- share same log level. 179 Arguments: see def _prepLogMsg(...) above. 180 """ 181 log2(level, level, msg, cleanYN, notimeYN, stream=stream)

182 183

184 -class FileCreationError(Exception):

185 pass

186 187

188 -class FileManip:

189 190 "Generic file manipulation class" 191

192 - def __init__(self, relative_path, timestamp, file_size):

193 self.relative_path = relative_path 194 self.timestamp = rhnLib.timestamp(timestamp) 195 self.file_size = file_size 196 self.full_path = os.path.join(CFG.MOUNT_POINT, self.relative_path) 197 self.buffer_size = CFG.BUFFER_SIZE

198

199 - def write_file(self, stream_in):

200 """Writes the contents of stream_in to the filesystem 201 Returns the file size(success) or raises FileCreationError""" 202 dirname = os.path.dirname(self.full_path) 203 createPath(dirname) 204 stat = os.statvfs(dirname) 205 206 f_bsize = stat[0] # file system block size 207 # misa: it's kind of icky whether to use f_bfree (free blocks) or 208 # f_bavail (free blocks for non-root). f_bavail is more correct, since 209 # you don't want to have the system out of disk space because of 210 # satsync; but people would get confused when looking at the output of 211 # df 212 f_bavail = stat[4] # free blocks 213 freespace = f_bsize * float(f_bavail) 214 if self.file_size is not None and self.file_size > freespace: 215 msg = messages.not_enough_diskspace % (freespace / 1024) 216 log(-1, msg, stream=sys.stderr) 217 # pkilambi: As the metadata download does'nt check for unfetched rpms 218 # abort the sync when it runs out of disc space 219 sys.exit(-1) 220 #raise FileCreationError(msg) 221 if freespace < 5000 * 1024: # arbitrary 222 msg = messages.not_enough_diskspace % (freespace / 1024) 223 log(-1, msg, stream=sys.stderr) 224 # pkilambi: As the metadata download does'nt check for unfetched rpms 225 # abort the sync when it runs out of disc space 226 sys.exit(-1) 227 #raise FileCreationError(msg) 228 229 fout = open(self.full_path, 'wb') 230 # setting file permissions; NOTE: rhnpush uses apache to write to disk, 231 # hence the 6 setting. 232 if rhnLib.isSUSE(): 233 setPermsPath(self.full_path, user='wwwrun', group='www', chmod=int('0644', 8)) 234 else: 235 setPermsPath(self.full_path, user='apache', group='apache', chmod=int('0644', 8)) 236 size = 0 237 try: 238 while 1: 239 buf = stream_in.read(self.buffer_size) 240 if not buf: 241 break 242 buf_len = len(buf) 243 fout.write(buf) 244 size = size + buf_len 245 except IOError: 246 e = sys.exc_info()[1] 247 msg = "IOError: %s" % e 248 log(-1, msg, stream=sys.stderr) 249 # Try not to leave garbage around 250 try: 251 os.unlink(self.full_path) 252 except (OSError, IOError): 253 pass 254 raise_with_tb(FileCreationError(msg), sys.exc_info()[2]) 255 l_file_size = fout.tell() 256 fout.close() 257 258 if self.file_size is not None and self.file_size != l_file_size: 259 # Something bad happened 260 msg = "Error: file %s has wrong size. Expected %s bytes, got %s bytes" % ( 261 self.full_path, self.file_size, l_file_size) 262 log(-1, msg, stream=sys.stderr) 263 # Try not to leave garbage around 264 try: 265 os.unlink(self.full_path) 266 except (OSError, IOError): 267 pass 268 raise FileCreationError(msg) 269 270 os.utime(self.full_path, (self.timestamp, self.timestamp)) 271 return l_file_size

272 273

274 -class RpmManip(FileManip):

275 276 """General [S]RPM manipulation class. 277 278 o Check checksums for mismatches 279 o Write RPMs to the filesystem 280 o get NVRE and NVREA 281 """ 282

283 - def __init__(self, pdict, path):

284 FileManip.__init__(self, relative_path=path, 285 timestamp=pdict['last_modified'], file_size=pdict['package_size']) 286 self.pdict = pdict

287

288 - def nvrea(self):

289 return tuple([self.pdict[x] for x in 290 ['name', 'version', 'release', 'epoch', 'arch']])

291

Source Code for Module backend.satellite_tools.syncLib