Source Code for Module backend.server.importlib.errataImport

  1  # 
  2  # Copyright (c) 2008--2016 Red Hat, Inc. 
  3  # 
  4  # This software is licensed to you under the GNU General Public License, 
  5  # version 2 (GPLv2). There is NO WARRANTY for this software, express or 
  6  # implied, including the implied warranties of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 
  8  # along with this software; if not, see 
  9  # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. 
 10  # 
 11  # Red Hat trademarks are not licensed under GPLv2. No permission is 
 12  # granted to use or replicate Red Hat trademarks that are incorporated 
 13  # in this software or its documentation. 
 14  # 
 15  # 
 16  # Errata import process 
 17  # 
 18   
 19  from spacewalk.common.rhnException import rhnFault 
 20  from importLib import GenericPackageImport 
 21  from spacewalk.satellite_tools.syncLib import log 
 22   
 23   
 24 -class ErrataImport(GenericPackageImport): 
 25   
 26 -    def __init__(self, batch, backend, queue_timeout=600): 
 27          GenericPackageImport.__init__(self, batch, backend) 
 28          # A composite key of the name, evr, arch plus org_id 
 29          self.packages = {} 
 30          self.ignoreMissing = 0 
 31          self.cve = {} 
 32          self.queue_timeout = queue_timeout 
 33          self.file_types = {} 
 34   
 35 -    def preprocess(self): 
 36          # Processes the package batch to a form more suitable for database 
 37          # operations 
 38   
 39          # We use this to avoid having the same erratum pushed multiple times 
 40          advisories = {} 
 41          errata_hash = {} 
 42   
 43          for errata in self.batch: 
 44              advisory = errata['advisory_name'] 
 45              release = errata['advisory_rel'] 
 46              errata_hash["%s%s" % (advisory, release)] = errata 
 47              if advisory in advisories: 
 48                  if long(release) < long(advisories[advisory]): 
 49                      # Seen a newer one already 
 50                      errata.ignored = 1 
 51                      continue 
 52                  else: 
 53                      # if this release is higher 
 54                      # we have to ignore the older one! 
 55                      errata_hash["%s%s" % (advisory, advisories[advisory])].ignored = 1 
 56              advisories[advisory] = release 
 57              self._preprocessErratum(errata) 
 58              self._preprocessErratumCVE(errata) 
 59              self._preprocessErratumFiles(errata) 
 60              self._preprocessErratumFileChannels(errata) 
 61   
 62 -    def _preprocessErratum(self, errata): 
 63          # Process packages 
 64          for package in errata['packages']: 
 65              self._processPackage(package) 
 66   
 67          # Process channels 
 68          channelHash = {} 
 69          for channel in errata['channels']: 
 70              channelName = channel['label'] 
 71              channelHash[channelName] = channel 
 72              self.channels[channelName] = None 
 73          # Replace the channel list with the unique one 
 74          errata['channels'] = list(channelHash.values()) 
 75   
 76 -    def _preprocessErratumCVE(self, erratum): 
 77          # Build the CVE dictionary 
 78          # FIXME: this if decision is here to deal with missing cve data. 
 79          #        fix later. 
 80          if not erratum['cve']: 
 81              erratum['cve'] = [] 
 82          for cve in erratum['cve']: 
 83              self.cve[cve] = None 
 84   
 85 -    def _preprocessErratumFiles(self, erratum): 
 86          for f in (erratum['files'] or []): 
 87              checksumTuple = (f['checksum_type'], f['checksum']) 
 88              if checksumTuple not in self.checksums: 
 89                  self.checksums[checksumTuple] = None 
 90   
 91              if f['file_type'] == 'RPM': 
 92                  package = f.get('pkgobj') 
 93                  if package: 
 94                      self._processPackage(package) 
 95                      nevrao = tuple(get_nevrao(package)) 
 96                      self.packages[nevrao] = package 
 97              # Oval errata files need to be removed from the import for now. 
 98              # This is to make sure non-oval capable satellites won't be importing 
 99              # the oval data from an oval-enabled dump. 
100              elif f['file_type'] == 'OVAL': 
101                  erratum['files'].remove(f) 
102              # elif f['file_type'] == 'SRPM': 
103              #    # XXX misa: do something here 
104              #    pass 
105   
106 -    def _preprocessErratumFileChannels(self, erratum): 
107          for f in (erratum['files'] or []): 
108              for channel_name in (f.get('channel_list') or []): 
109                  self.channels[channel_name] = None 
110   
111 -    def fix(self): 
112          self.backend.lookupChannels(self.channels) 
113          self.backend.lookupErrataFileTypes(self.file_types) 
114          for erratum in self.batch: 
115              for ef in erratum['files']: 
116                  eft = ef['file_type'] 
117                  if eft not in self.file_types: 
118                      raise Exception("Unknown file type %s" % eft) 
119                  ef['type'] = self.file_types[eft] 
120   
121          self._fixCVE() 
122   
123          self.backend.lookupPackageNames(self.names) 
124          self.backend.lookupEVRs(self.evrs) 
125          self.backend.lookupChecksums(self.checksums) 
126          self.backend.lookupPackageArches(self.package_arches) 
127   
128          for erratum in self.batch: 
129              if erratum.ignored: 
130                  # Skip it 
131                  continue 
132              self._fix_erratum_channels(erratum) 
133              self._fix_erratum_packages_lookup(erratum) 
134              self._fix_erratum_file_packages(erratum) 
135              # fix severity stuff 
136              self._fix_erratum_severity(erratum) 
137              # fix oval info to populate the relevant dbtables 
138              self._fix_erratum_oval_info(erratum) 
139   
140          self.backend.lookupPackages(list(self.packages.values()), self.checksums, self.ignoreMissing) 
141          for erratum in self.batch: 
142              if erratum.ignored: 
143                  # Skip it 
144                  continue 
145              self._fix_erratum_packages(erratum) 
146              self._fix_erratum_file_channels(erratum) 
147   
148          # remove erratas that have been ignored 
149          ignored_erratas = list(filter(lambda x: x.ignored, self.batch)) 
150          if len(ignored_erratas) > 0: 
151              log(0, "Ignoring %d old, superseded erratas" % len(ignored_erratas)) 
152              self.batch = list(filter(lambda x: not x.ignored, self.batch)) 
153   
154 -    def _fixCVE(self): 
155          # Look up and insert the missing CVE's 
156          self.backend.processCVEs(self.cve) 
157          # Fix the CVE stuff 
158          for erratum in self.batch: 
159              if erratum.ignored: 
160                  continue 
161              cves = [] 
162              for cve in erratum['cve']: 
163                  entry = { 
164                      'cve_id': self.cve[cve], 
165                  } 
166                  cves.append(entry) 
167              erratum['cve'] = cves 
168   
169 -    def _fix_files(self): 
170          rpm_files = [] 
171          srpm_files = [] 
172          oval_files = [] 
173          channel_files = [] 
174          for erratum in self.batch: 
175              if erratum.ignored: 
176                  continue 
177              for file in erratum['files']: 
178                  file_type = file['file_type'] 
179                  file_id = file['id'] 
180                  package_id = file.get('package_id') 
181                  if package_id is not None: 
182                      pkg = { 
183                          'errata_file_id': file_id, 
184                          'package_id': package_id, 
185                      } 
186                      if file_type == 'RPM': 
187                          rpm_files.append(pkg) 
188                      elif file_type == 'SRPM': 
189                          srpm_files.append(pkg) 
190                      elif file_type == 'OVAL': 
191                          pkg = { 
192                              'errata_id': file_id, 
193                              'filename': file['filename'], 
194                          } 
195                          oval_files.append(pkg) 
196                  for channel_id in file['channels']: 
197                      channel_files.append({ 
198                          'errata_file_id': file_id, 
199                          'channel_id': channel_id, 
200                      }) 
201          self.backend._do_diff(rpm_files, 'rhnErrataFilePackage', 
202                                ['errata_file_id', 'package_id'], []) 
203          self.backend._do_diff(srpm_files, 'rhnErrataFilePackageSource', 
204                                ['errata_file_id', 'package_id'], []) 
205          self.backend._do_diff(channel_files, 'rhnErrataFileChannel', 
206                                ['errata_file_id', 'channel_id'], []) 
207          self.backend._do_diff(oval_files, 'rhnErrataFile', 
208                                ['errata_id', 'filename'], []) 
209   
210 -    def submit(self): 
211          try: 
212              dml = self.backend.processErrata(self.batch) 
213              self.backend.update_channels_affected_by_errata(dml) 
214              self._fix_files() 
215              self.backend.queue_errata(self.batch, self.queue_timeout) 
216          except: 
217              self.backend.rollback() 
218              raise 
219          self.backend.commit() 
220   
221 -    def _fix_erratum_channels(self, erratum): 
222          # Fix the erratum's channels 
223          channels = {} 
224          for ch in erratum['channels']: 
225              label = ch['label'] 
226              channel = self.channels[label] 
227              if not channel: 
228                  # Invalid channel 
229                  if self.ignoreMissing: 
230                      # Ignore missing channel 
231                      continue 
232                  # XXX Raising an exception here; it may be too harsh though 
233                  erratum.ignored = 1 
234                  raise Exception("XXX Invalid channel %s" % label) 
235   
236              channels[channel['id']] = None 
237   
238          erratum['channels'] = [{'channel_id': x} for x in channels.keys()] 
239   
240 -    def _fix_erratum_packages_lookup(self, erratum): 
241          # To make the packages unique 
242          packageHash = {} 
243          for package in erratum['packages']: 
244              if package.ignored: 
245                  # Skip it 
246                  continue 
247   
248              self._postprocessPackageNEVRA(package) 
249   
250              # Check the uniqueness 
251              nevrao = tuple(get_nevrao(package)) 
252   
253              if nevrao in packageHash: 
254                  # Been there already 
255                  package.ignored = 1 
256                  continue 
257   
258              package['nevrao'] = nevrao 
259   
260              # And put this package both in the local and in the global hash 
261              packageHash[nevrao] = package 
262              self.packages[nevrao] = package 
263   
264          erratum['packages'] = packageHash 
265   
266 -    def _fix_erratum_file_packages(self, erratum): 
267          for ef in erratum['files']: 
268              ef['checksum_id'] = self.checksums[(ef['checksum_type'], ef['checksum'])] 
269              if ef['file_type'] == 'RPM': 
270                  package = ef.get('pkgobj') 
271                  if not package: 
272                      continue 
273                  self._postprocessPackageNEVRA(package) 
274              # XXX fix source rpms 
275   
276 -    def _fix_erratum_packages(self, erratum): 
277          pkgs = [] 
278          # This is a workaround; It would be much straightforward to use 
279          # 'package in erratum['packages'].values()' here. But for (to me) unknown 
280          # reason it sometimes has package.id == None which makes whole import fail. 
281          # And self.packages[nevrao].id contains always right value. 
282          for nevrao in erratum['packages'].keys(): 
283              package = self.packages[nevrao] 
284              if package.ignored: 
285                  # Ignore this package 
286                  continue 
287              pkgs.append({'package_id': package.id}) 
288   
289          erratum['packages'] = pkgs 
290   
291          for ef in (erratum['files'] or []): 
292              if ef['file_type'] == 'RPM': 
293                  package = ef.get('pkgobj') 
294                  if package: 
295                      ef['package_id'] = package.id 
296   
297 -    def _fix_erratum_file_channels(self, erratum): 
298          for f in (erratum['files'] or []): 
299              channels = [] 
300              for c in (f.get('channel_list') or []): 
301                  if not self.channels[c]: 
302                      # Unsupported channel 
303                      # XXX misa: should we gripe loudly? 
304                      continue 
305                  channels.append(self.channels[c]['id']) 
306              f['channels'] = channels 
307   
308 -    def _fix_erratum_severity(self, erratum): 
309          """sets the severity-id to insert into rhnErrata 
310          """ 
311          # Re-check for severity, it could be a RHBA or RHEA 
312          # If RHBA/RHEA severity is irrelevant and posibly 
313          # not included or it could not be hosted 
314          if 'security_impact' in erratum: 
315              erratum['severity_id'] = self.backend.lookupErrataSeverityId(erratum) 
316   
317 -    def _fix_erratum_oval_info(self, erratum): 
318          """ 
319          manipulate oval package info to populate in the 
320          appropriate fields in the db tables. 
321   
322          """ 
323          import os 
324   
325          if 'oval_info' not in erratum: 
326              return 
327   
328          for oval_file in erratum['oval_info']: 
329              if has_suffix(oval_file['filename'], '.xml'): 
330                  eft = oval_file['file_type'] = 'OVAL' 
331                  if eft not in self.file_types: 
332                      raise Exception("Unknown file type %s" % eft) 
333                  oval_file['type'] = self.file_types[eft] 
334   
335              # XXX: stubs incase we need to associate them to channels/packages 
336              oval_file['channel_list'] = [] 
337              oval_file['channels'] = [] 
338              oval_file['package_id'] = None 
339   
340              if not os.path.isfile(oval_file['filename']): 
341                  # Don't bother to copy the package 
342                  raise rhnFault(47, 
343                                 "Oval file %s not found on the server. " % oval_file['filename'], 
344                                 explain=0) 
345   
346              # add the oval info into the files field to get 
347              # populated into db 
348              erratum['files'].append(oval_file) 
349   
350   
351 -def get_nevrao(package): 
352      return list(map(lambda x, d=package: d[x], 
353                 ['name', 'epoch', 'version', 'release', 'arch', 'org_id', 'checksum_type', 'checksum'])) 
354   
355   
356 -def has_suffix(s, suffix): 
357      return s[-len(suffix):] == suffix 
358