backend.server.importlib.backendLib

1 # 2 # Copyright (c) 2008--2016 Red Hat, Inc. 3 # 4 # This software is licensed to you under the GNU General Public License, 5 # version 2 (GPLv2). There is NO WARRANTY for this software, express or 6 # implied, including the implied warranties of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 8 # along with this software; if not, see 9 # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. 10 # 11 # Red Hat trademarks are not licensed under GPLv2. No permission is 12 # granted to use or replicate Red Hat trademarks that are incorporated 13 # in this software or its documentation. 14 # 15 # 16 # Generic DB backend data structures 17 # 18 19 import time 20 import string 21 22 from UserDict import UserDict 23 from spacewalk.common.usix import ListType, StringType, DictType, IntType, UnicodeType 24 25 # A function that formats a UNIX timestamp to the session's format 26 27

28 -def gmtime(timestamp):

29 return _format_time(time.gmtime(float(timestamp)))

30 31

32 -def localtime(timestamp):

33 return _format_time(time.localtime(float(timestamp)))

34 35

36 -def _format_time(time_tuple):

37 return time.strftime("%Y-%m-%d %H:%M:%S", time_tuple)

38 39 # Database datatypes 40 41

42 -class DBtype:

43 pass

44 45

46 -class DBint(DBtype):

47 pass

48 49

50 -class DBstring(DBtype):

51

52 - def __init__(self, limit):

53 self.limit = limit

54 55

56 -class DBblob(DBtype):

57 pass

58 59

60 -class DBdate(DBtype):

61 pass

62 63

64 -class DBdateTime(DBtype):

65 pass

66 67 # Database objects 68 69

70 -class Table:

71 # A list of supported keywords 72 keywords = { 73 'fields': DictType, 74 'pk': ListType, 75 'attribute': StringType, 76 'map': DictType, 77 'nullable': ListType, # Will become a hash eventually 78 'severityHash': DictType, 79 'defaultSeverity': IntType, 80 'sequenceColumn': StringType, 81 } 82

83 - def __init__(self, name, **kwargs):

84 self.name = name 85 for k in kwargs.keys(): 86 if k not in self.keywords: 87 raise TypeError("Unknown keyword attribute '%s'" % k) 88 # Initialize stuff 89 # Fields 90 self.fields = {} 91 # Primary keys 92 self.pk = [] 93 # Mapping from database fields to generic attribute names 94 self.map = {} 95 # Name of the attribute this table links back to 96 self.attribute = None 97 # Nullable columns; will become a hash 98 self.nullable = [] 99 # Compute the diff 100 self.severityHash = {} 101 self.defaultSeverity = 4 102 # Sequence column - a column that is populated off a sequence 103 self.sequenceColumn = None 104 105 for k, v in kwargs.items(): 106 datatype = self.keywords[k] 107 if not isinstance(v, datatype): 108 raise TypeError("%s expected to be %s; got %s" % ( 109 k, datatype, type(v))) 110 setattr(self, k, v) 111 112 # Fix nullable 113 nullable = self.nullable 114 self.nullable = {} 115 if nullable: 116 for field in nullable: 117 if field not in self.fields: 118 raise TypeError("Unknown nullable field %s in table %s" % ( 119 field, name)) 120 self.nullable[field] = None 121 122 # Now analyze pk 123 for field in self.pk: 124 if field not in self.fields: 125 raise TypeError("Unknown primary key field %s" % field)

126

127 - def __str__(self):

128 return "Instance of class %s.%s: PK: %s, Fields: %s" % (self.__class__.__module__, 129 self.__class__.__name__, self.pk, self.fields)

130 __repr__ = __str__ 131

132 - def isNullable(self, field):

133 if field not in self.fields: 134 raise TypeError("Unknown field %s" % field) 135 return field in self.nullable

136

137 - def getPK(self):

138 return self.pk

139

140 - def getFields(self):

141 return self.fields

142

143 - def getAttribute(self):

144 return self.attribute

145

146 - def getObjectAttribute(self, attribute):

147 if attribute in self.map: 148 return self.map[attribute] 149 return attribute

150

151 - def getSeverityHash(self):

152 for field in self.fields.keys(): 153 if field not in self.severityHash: 154 self.severityHash[field] = self.defaultSeverity 155 return self.severityHash

156 157 # A collection of tables 158 159

160 -class TableCollection(UserDict):

161

162 - def __init__(self, *list):

163 UserDict.__init__(self) 164 # Verify if the list's items are the right format 165 for table in list: 166 if not isinstance(table, Table): 167 raise TypeError("Expected a Table instance; got %s" % 168 type(table)) 169 # Now initialize the collection 170 for table in list: 171 self.__setitem__(table.name, table)

172 173 # Lookup class 174 # The problem stems from the different way we're supposed to build a query if 175 # the value is nullable 176 177

178 -class BaseTableLookup:

179

180 - def __init__(self, table, dbmodule):

181 # Generates a bunch of queries that look up data based on the primary 182 # keys of this table 183 self.dbmodule = dbmodule 184 self.table = table 185 self.pks = self.table.getPK() 186 self.whereclauses = {} 187 self.queries = {} 188 self._buildWhereClauses()

189

190 - def _buildWhereClauses(self):

191 # Keys is a list of lists of 0/1, 0 if the column is not nullable 192 keys = [[]] 193 # The corresponding query arguments 194 queries = [[]] 195 for col in self.pks: 196 k = [] 197 q = [] 198 for i in range(len(keys)): 199 key = keys[i] 200 query = queries[i] 201 k.append(key + [0]) 202 q.append(query + ["%s = :%s" % (col, col)]) 203 if self.table.isNullable(col): 204 k.append(key + [1]) 205 q.append(query + ["%s is null" % col]) 206 keys = k 207 queries = q 208 # Now put the queries in self.sqlqueries, keyed on the list of 0/1 209 for i in range(len(keys)): 210 key = tuple(keys[i]) 211 query = string.join(queries[i], ' and ') 212 self.whereclauses[key] = query

213

214 - def _selectQueryKey(self, value):

215 # Determine which query should we use 216 # Build the key first 217 hash = {} 218 key = [] 219 for col in self.pks: 220 if self.table.isNullable(col) and value[col] in [None, '']: 221 key.append(1) 222 else: 223 key.append(0) 224 hash[col] = value[col] 225 key = tuple(key) 226 return key, hash

227

228 - def _buildQuery(self, key):

229 # Stub 230 return None

231

232 - def _getCachedQuery(self, key, blob_map=None):

233 if key in self.queries: 234 # Serve it from the pool 235 return self.queries[key] 236 237 statement = self.dbmodule.prepare(self._buildQuery(key), blob_map=blob_map) 238 # And save it to the cached queries pool 239 self.queries[key] = statement 240 return statement

241

242 - def query(self, values):

243 key, values = self._selectQueryKey(values) 244 statement = self._getCachedQuery(key) 245 statement.execute(**values) 246 return statement

247 248

249 -class TableLookup(BaseTableLookup):

250

251 - def __init__(self, table, dbmodule):

252 BaseTableLookup.__init__(self, table, dbmodule) 253 self.queryTemplate = "select * from %s where %s"

254

255 - def _buildQuery(self, key):

256 return self.queryTemplate % (self.table.name, self.whereclauses[key])

257 258

259 -class TableUpdate(BaseTableLookup):

260

261 - def __init__(self, table, dbmodule):

262 BaseTableLookup.__init__(self, table, dbmodule) 263 self.queryTemplate = "update %s set %s where %s" 264 self.fields = list(self.table.getFields().keys()) 265 self.count = 1000 266 # Fields minus pks 267 self.otherfields = [] 268 # BLOBs cannot be PKs, and have to be updated differently 269 self.blob_fields = [] 270 for field in self.fields: 271 if field in self.pks: 272 continue 273 datatype = self.table.fields[field] 274 if isinstance(datatype, DBblob): 275 self.blob_fields.append(field) 276 else: 277 self.otherfields.append(field) 278 self.updateclause = string.join( 279 ["%s = :%s" % (x, x) for x in self.otherfields], ', ') 280 # key 281 self.firstkey = None 282 for pk in self.pks: 283 if not self.table.isNullable(pk): 284 # This is it 285 self.firstkey = pk 286 break

287

288 - def _buildQuery(self, key):

289 return self.queryTemplate % (self.table.name, self.updateclause, 290 self.whereclauses[key])

291

292 - def _split_blob_values(self, values, blob_only=0):

293 # Splits values that have to be inserted 294 # Blobs will be in a separate hash 295 valuesHash = {} 296 # blobValuesHash is a hash keyed on the primary key fields 297 # should only have one element if the primary key has no nullable 298 # fields 299 blobValuesHash = {} 300 for key in self.whereclauses.keys(): 301 hash = {} 302 for i in range(len(key)): 303 pk = self.pks[i] 304 # Only add the PK if it's non-null 305 if not key[i]: 306 hash[pk] = [] 307 # And then add everything else 308 for k in self.otherfields: 309 hash[k] = [] 310 valuesHash[key] = hash 311 blobValuesHash[key] = [] 312 313 # Split the query values on key components 314 for i in range(len(values[self.firstkey])): 315 # Build the value 316 pk_val = {} 317 val = {} 318 for k in self.pks: 319 pk_val[k] = val[k] = values[k][i] 320 key, val = self._selectQueryKey(val) 321 322 if not blob_only: 323 # Add the rest of the values 324 for k in self.otherfields: 325 val[k] = values[k][i] 326 addHash(valuesHash[key], val) 327 328 if not self.blob_fields: 329 # Nothing else to do 330 continue 331 val = {} 332 for k in self.blob_fields: 333 val[k] = values[k][i] 334 blobValuesHash[key].append((pk_val, val)) 335 336 return valuesHash, blobValuesHash

337

338 - def query(self, values):

339 valuesHash, blobValuesHash = self._split_blob_values(values, blob_only=0) 340 # And now do the actual update for non-blobs 341 if self.otherfields: 342 for key, val in valuesHash.items(): 343 if not val[self.firstkey]: 344 # Nothing to do 345 continue 346 statement = self._getCachedQuery(key) 347 executeStatement(statement, val, self.count) 348 349 if not self.blob_fields: 350 return 351 352 self._update_blobs(blobValuesHash)

353

354 - def _update_blobs(self, blobValuesHash):

355 # Now update BLOB fields 356 template = "select %s from %s where %s for update" 357 blob_fields_string = string.join(self.blob_fields, ", ") 358 for key, val in blobValuesHash.items(): 359 statement = template % (blob_fields_string, self.table.name, 360 self.whereclauses[key]) 361 h = self.dbmodule.prepare(statement) 362 for lookup_hash, blob_hash in val: 363 h.execute(**lookup_hash) 364 # Should have exactly one row here 365 row = h.fetchone_dict() 366 if not row: 367 # XXX This should normally not happen 368 raise ValueError("BLOB query did not retrieve a value") 369 for k, v in blob_hash.items(): 370 blob = row[k] 371 len_v = len(v) 372 # If new value is shorter than old value, we have to trim 373 # the blob 374 if blob.size() > len_v: 375 blob.trim(len_v) 376 # blobs don't like to write the empty string 377 if len_v: 378 blob.write(v) 379 # Is this the only row? 380 row = h.fetchone_dict() 381 if row is not None: 382 # XXX This should not happen, the primary key was not 383 # unique 384 raise ValueError("Primary key not unique", 385 self.table.name, lookup_hash)

386 387

388 -class TableDelete(TableLookup):

389

390 - def __init__(self, table, dbmodule):

391 TableLookup.__init__(self, table, dbmodule) 392 self.queryTemplate = "delete from %s where %s" 393 self.count = 1000

394

395 - def query(self, values):

396 # Build the values hash 397 valuesHash = {} 398 for key in self.whereclauses.keys(): 399 hash = {} 400 for i in range(len(key)): 401 pk = self.pks[i] 402 # Only add the PK if it's non-null 403 if not key[i]: 404 hash[pk] = [] 405 valuesHash[key] = hash 406 407 # Split the query values on key components 408 firstkey = self.pks[0] 409 for i in range(len(values[firstkey])): 410 # Build the value 411 val = {} 412 for k in self.pks: 413 val[k] = values[k][i] 414 key, val = self._selectQueryKey(val) 415 addHash(valuesHash[key], val) 416 417 # And now do the actual delete 418 for key, val in valuesHash.items(): 419 firstkey = val.keys()[0] 420 if not val[firstkey]: 421 # Nothing to do 422 continue 423 statement = self._getCachedQuery(key) 424 executeStatement(statement, val, self.count)

425 426

427 -class TableInsert(TableUpdate):

428

429 - def __init__(self, table, dbmodule):

430 TableUpdate.__init__(self, table, dbmodule) 431 self.queryTemplate = "insert into %s (%s) values (%s)" 432 self.count = 1000 433 434 self.insert_fields = self.pks + self.otherfields + self.blob_fields 435 self.insert_values = [':%s' % x for x in self.pks + self.otherfields + self.blob_fields]

436

437 - def _buildQuery(self, key):

438 q = self.queryTemplate % (self.table.name, 439 string.join(self.insert_fields, ', '), 440 string.join(self.insert_values, ', ')) 441 return q

442

443 - def query(self, values):

444 if self.blob_fields: 445 chunksize = 1 446 blob_map = {} 447 for f in self.blob_fields: 448 blob_map[f] = f 449 else: 450 chunksize = self.count 451 blob_map = None 452 453 # Do the insert 454 statement = self._getCachedQuery(None, blob_map=blob_map) 455 executeStatement(statement, values, chunksize)

456 457

458 -def executeStatement(statement, valuesHash, chunksize):

459 # Executes a statement with chunksize values at the time 460 if not valuesHash: 461 # Empty hash 462 return 463 count = 0 464 while 1: 465 tempdict = {} 466 for k, vals in valuesHash.items(): 467 if not vals: 468 # Empty 469 break 470 if chunksize > 1: 471 tempdict[k] = vals[:chunksize] 472 else: 473 tempdict[k] = vals[0] 474 del vals[:chunksize] 475 if not tempdict: 476 # Empty dictionary: we're done 477 break 478 # Now execute it 479 if chunksize > 1: 480 count += statement.executemany(**tempdict) 481 else: 482 count += statement.execute(**tempdict) 483 return count

484 485

486 -def sanitizeValue(value, datatype):

487 if isinstance(datatype, DBstring): 488 if value is None or value == '': 489 return None # we really want to preserve Nones 490 # and not depend on Oracle converting 491 # empty strings to NULLs -- PostgreSQL 492 # does not do this 493 elif isinstance(value, UnicodeType): 494 value = UnicodeType.encode(value, 'utf-8') 495 if len(value) > datatype.limit: 496 value = value[:datatype.limit] 497 # ignore incomplete characters created after truncating 498 value = value.decode('utf-8', 'ignore') 499 value = value.encode('utf-8') 500 return value 501 if isinstance(datatype, DBblob): 502 if value is None: 503 value = '' 504 if isinstance(value, UnicodeType): 505 value = UnicodeType.encode(value, 'utf-8') 506 return str(value) 507 if value in [None, '']: 508 return None 509 if isinstance(datatype, DBdateTime): 510 s = str(value) 511 if len(s) == 10: 512 # Pad it to be a real datetime 513 s = s + " 00:00:00" 514 return s 515 if isinstance(datatype, DBdate): 516 return str(value)[:10] 517 if isinstance(datatype, DBint): 518 return int(value) 519 return value

520 521

522 -def addHash(hasharray, hash):

523 # hasharray is a hash of arrays 524 # add hash's values to hasharray 525 for k, v in hash.items(): 526 if k in hasharray: 527 hasharray[k].append(v)

528

Source Code for Module backend.server.importlib.backendLib