Source Code for Module backend.server.importlib.channelImport

  1  # 
  2  # Copyright (c) 2008--2017 Red Hat, Inc. 
  3  # 
  4  # This software is licensed to you under the GNU General Public License, 
  5  # version 2 (GPLv2). There is NO WARRANTY for this software, express or 
  6  # implied, including the implied warranties of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 
  8  # along with this software; if not, see 
  9  # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. 
 10  # 
 11  # Red Hat trademarks are not licensed under GPLv2. No permission is 
 12  # granted to use or replicate Red Hat trademarks that are incorporated 
 13  # in this software or its documentation. 
 14  # 
 15  # Channel import process 
 16  # 
 17   
 18  from importLib import Import, InvalidArchError, \ 
 19      InvalidChannelError, InvalidChannelFamilyError, MissingParentChannelError 
 20  from spacewalk.satellite_tools.syncLib import log 
 21   
 22   
 23 -class ChannelImport(Import): 
 24   
 25 -    def __init__(self, batch, backend): 
 26          Import.__init__(self, batch, backend) 
 27          self.arches = {} 
 28          self.families = {} 
 29          self.dists = {} 
 30          self.will_commit = 1 
 31          self.releases = {} 
 32          self.channels = {} 
 33          self.checksum_types = {} 
 34   
 35 -    def preprocess(self): 
 36          # Processes the batch to a form more suitable for database 
 37          # operations 
 38          for channel in self.batch: 
 39              self.__processChannel(channel) 
 40   
 41 -    def __processChannel(self, channel): 
 42          # Processes a package 
 43          arch = channel['channel_arch'] 
 44          if arch not in self.arches: 
 45              self.arches[arch] = None 
 46          for family in channel['families']: 
 47              self.families[family['label']] = None 
 48          # Dists 
 49          if 'dists' in channel and channel['dists'] is not None: 
 50              for dist in channel['dists']: 
 51                  self.arches[dist['channel_arch']] = None 
 52          # Product Names 
 53          if 'release' in channel and channel['release'] is not None: 
 54              for release in channel['release']: 
 55                  self.arches[release['channel_arch']] = None 
 56          if 'receiving_updates' not in channel or channel['receiving_updates'] is None: 
 57              channel['receiving_updates'] = 'N' 
 58          # Yum repo checksum type 
 59          if (channel['checksum_type'] 
 60                  and channel['checksum_type'] not in self.checksum_types): 
 61              self.checksum_types[channel['checksum_type']] = None 
 62   
 63          # bug #528227 
 64          # Print a warning in case the sync would move the channel between orgs 
 65          if 'org_id' in channel and channel['org_id']: 
 66              org_id = self.backend.lookupChannelOrg(channel['label']) 
 67   
 68              if org_id and int(channel['org_id']) != org_id['org_id']: 
 69                  log(1, "WARNING: Channel %s is already present in orgid %s." % 
 70                      (channel['label'], org_id['org_id'])) 
 71                  log(1, "         Running synchronization will move the channel to orgid %s." % 
 72                      channel['org_id']) 
 73                  log(1, '') 
 74   
 75 -    def fix(self): 
 76          self.backend.lookupChannelArches(self.arches) 
 77          self.backend.lookupChannelFamilies(self.families) 
 78          self.backend.lookupChecksumTypes(self.checksum_types) 
 79          # Fix 
 80          for channel in self.batch: 
 81              self.__postprocessChannel(channel) 
 82   
 83 -    def __postprocessChannel(self, channel): 
 84          if channel.ignored: 
 85              return 
 86          arch = channel['channel_arch'] 
 87          if self.arches[arch] is None: 
 88              # Mark it as ignored 
 89              channel.ignored = 1 
 90              raise InvalidArchError(arch, "Unsupported channel arch %s" % arch) 
 91          channel['channel_arch_id'] = self.arches[arch] 
 92          if channel['checksum_type']: 
 93              channel['checksum_type_id'] = self.checksum_types[channel['checksum_type']] 
 94          else: 
 95              channel['checksum_type_id'] = None 
 96   
 97          if 'product_name' in channel: 
 98              channel['product_name_id'] = self.backend.lookupProductNames( 
 99                  channel['product_name']) 
100          families = [] 
101          for family in channel['families']: 
102              # Link back the channel to families 
103              channel_family_id = self.families[family['label']] 
104   
105              if channel_family_id is None: 
106                  # Still cant get the id, Unknown channel family 
107                  raise InvalidChannelFamilyError(family['label']) 
108   
109              families.append({ 
110                  'channel_family_id': self.families[family['label']] 
111              }) 
112          channel['families'] = families 
113          # Dists 
114          self.__postprocessChannelMaps(channel, 'dists') 
115          # release 
116          self.__postprocessChannelMaps(channel, 'release') 
117   
118 -    def __postprocessChannelMaps(self, channel, map): 
119          if map in channel and channel[map] is not None: 
120              for dict in channel[map]: 
121                  arch = dict['channel_arch'] 
122                  if self.arches[arch] is None: 
123                      # Mark it as ignored 
124                      channel.ignored = 1 
125                      raise InvalidArchError(arch, "Unsupported channel arch %s" % arch) 
126                  dict['channel_arch_id'] = self.arches[arch] 
127   
128 -    def submit(self): 
129          parentChannels = {} 
130          # Split the batch into null and non-null parent channels 
131          nullParentBatch = [] 
132          nonNullParentBatch = [] 
133          channel_trusts = [] 
134          for channel in self.batch: 
135              if channel.ignored: 
136                  continue 
137              if 'trust_list' in channel and channel['trust_list']: 
138                  self.backend.clearChannelTrusts(channel['label']) 
139                  for trust in channel['trust_list']: 
140                      if ('org_id' in channel and channel['org_id'] 
141                              and self.backend.orgTrustExists( 
142                              channel['org_id'], trust['org_trust_id'])): 
143                          channel_trusts.append( 
144                              {'channel-label': channel['label'], 
145                               'org-id': trust['org_trust_id']}) 
146              parent = channel['parent_channel'] 
147              if not parent: 
148                  nullParentBatch.append(channel) 
149                  continue 
150              nonNullParentBatch.append(channel) 
151              # And save the parent channel's label in a hash too 
152              parentChannels[parent] = None 
153          # Process the easy case of null parent channels 
154          try: 
155              self.backend.processChannels(nullParentBatch, True) 
156          except: 
157              self.backend.rollback() 
158              raise 
159   
160          # Find the parent channels ids 
161          for channel in nullParentBatch: 
162              if channel.ignored: 
163                  continue 
164              label = channel['label'] 
165              if label not in parentChannels: 
166                  # This channel is not a parent channel to anybody 
167                  continue 
168              parentChannels[label] = channel.id 
169   
170          # Build an extra hash for the channels with unknown ids 
171          unknownChannels = {} 
172          for k, v in parentChannels.items(): 
173              if v is None: 
174                  unknownChannels[k] = None 
175   
176          # And look them up 
177          self.backend.lookupChannels(unknownChannels) 
178   
179          # Copy the ids back into parentChannels, to make life easier 
180          missingParents = [] 
181          for k, v in unknownChannels.items(): 
182              if v is None: 
183                  missingParents.append(k) 
184              else: 
185                  parentChannels[k] = v['id'] 
186          if missingParents: 
187              raise MissingParentChannelError( 
188                  missingParents, "Invalid import, this parents need to be imported: %s" % str(", ".join(missingParents))) 
189   
190          # Fix up the parent channels 
191          for channel in nonNullParentBatch: 
192              parent = channel['parent_channel'] 
193              if parent not in parentChannels: 
194                  # Unknown parent channel 
195                  channel.ignored = 1 
196                  continue 
197              # Replace the label with the id 
198              channel['parent_channel'] = parentChannels[parent] 
199   
200          # And process these channels too 
201          try: 
202              self.backend.processChannels(nonNullParentBatch, False) 
203          except: 
204              self.backend.rollback() 
205              raise 
206   
207          # Process the channel trusts 
208          if len(channel_trusts) > 0: 
209              self.backend.processChannelTrusts(channel_trusts) 
210   
211          # Finally go back and add the products and content sources, if any 
212          for channel in self.batch: 
213              if channel.ignored: 
214                  continue 
215   
216              if ('channel_product' in channel and channel['channel_product']) \ 
217                      or ('product_name' in channel and channel['product_name']): 
218                  self.backend.processChannelProduct(channel) 
219   
220              self.backend.processChannelContentSources(channel) 
221   
222          # Sometimes we may want to turn commits off 
223          if self.will_commit: 
224              self.backend.commit() 
225   
226   
227 -class ChannelFamilyImport(Import): 
228   
229 -    def preprocess(self): 
230          self.__filterCustomChannelFamilies() 
231          # We have to look up the channels for this channel family first 
232          self.channels = {} 
233          for cf in self.batch: 
234              for c in cf['channels']: 
235                  self.channels[c] = None 
236   
237 -    def fix(self): 
238          self.backend.lookupChannels(self.channels) 
239          for cf in self.batch: 
240              channel_ids = cf['channel_ids'] = [] 
241              for c in cf['channels']: 
242                  chash = self.channels[c] 
243                  if chash is None: 
244                      # Skip 
245                      continue 
246                  cid = chash['id'] 
247                  channel_ids.append(cid) 
248   
249 -    def submit(self): 
250          try: 
251              self.backend.processChannelFamilies(self.batch) 
252              self.backend.processChannelFamilyMembers(self.batch) 
253              self.backend.processChannelFamilyPermissions(self.batch) 
254          except: 
255              self.backend.rollback() 
256              raise 
257          self.backend.commit() 
258   
259 -    def __filterCustomChannelFamilies(self): 
260          """Filter out private channel families from ISS syncs. WebUI 
261             creates these for us at the org creation time. 
262          """ 
263          new_batch = [] 
264          for cf in self.batch: 
265              if not cf['label'].startswith("private-channel-family"): 
266                  new_batch.append(cf) 
267          self.batch = new_batch 
268   
269   
270 -class DistChannelMapImport(Import): 
271   
272 -    def __init__(self, batch, backend): 
273          Import.__init__(self, batch, backend) 
274          self.arches = {} 
275          self.channels = {} 
276   
277 -    def preprocess(self): 
278          # Processes the batch to a form more suitable for database 
279          # operations 
280          for dcm in self.batch: 
281              self.arches[dcm['arch']] = None 
282              self.channels[dcm['channel']] = None 
283   
284 -    def fix(self): 
285          # Look up arches and channels 
286          self.backend.lookupChannelArches(self.arches) 
287          self.backend.lookupChannels(self.channels) 
288          for dcm in self.batch: 
289              arch = self.arches[dcm['arch']] 
290              if arch is None: 
291                  # Invalid arch 
292                  dcm.ignored = 1 
293                  raise InvalidArchError(dcm['arch'], 
294                                         "Invalid dist_channel_map arch %s" % dcm['arch']) 
295              channel = self.channels[dcm['channel']] 
296              if channel is None: 
297                  dcm.ignored = 1 
298                  raise InvalidChannelError(dcm['channel'], 
299                                            "Invalid dist_channel_map channel %s" % dcm['channel']) 
300              dcm['arch'] = arch 
301              dcm['channel_id'] = channel['id'] 
302              dcm['org_id'] = None 
303   
304 -    def submit(self): 
305          try: 
306              self.backend.processDistChannelMap(self.batch) 
307          except: 
308              self.backend.rollback() 
309              raise 
310          self.backend.commit() 
311   
312   
313  # for testing only 
314  if __name__ == '__main__': 
315      import sys 
316      from spacewalk.server import rhnSQL 
317      from backendOracle import OracleBackend 
318      from importLib import Collection, ChannelFamily, DistChannelMap 
319      backend = OracleBackend() 
320      if 1: 
321          batch = Collection() 
322          dcms = [ 
323              { 
324                  'os': 'Red Hat Linux', 
325                  'release': '7.2', 
326                  'arch': 'i386', 
327                  'channel': 'redhat-linux-i386-7.2', 
328              }, 
329              { 
330                  'os': 'Red Hat Linux', 
331                  'release': '6.2', 
332                  'arch': 'i386', 
333                  'channel': 'redhat-linux-i386-6.2', 
334              }, 
335          ] 
336   
337          for dcm in dcms: 
338              x = DistChannelMap() 
339              x.populate(dcm) 
340              batch.append(x) 
341          rhnSQL.initDB() 
342          backend.init() 
343          dcmimp = DistChannelMapImport(batch, backend) 
344          dcmimp.run() 
345          sys.exit(0) 
346      if 0: 
347          batch = Collection() 
348          families = [ 
349              { 
350                  'name': 'Cisco Linux', 
351                  'label': 'cisco', 
352                  'product_url': 'http://www.redhat.com/products/ADSFASDFASDF', 
353              }, 
354              { 
355                  'name': 'Misa Linux', 
356                  'label': 'misa', 
357                  'product_url': 'http://people.redhat.com/misa/ASDFASDFASDF', 
358              }, 
359          ] 
360          for fam in families: 
361              cf = ChannelFamily() 
362              cf.populate(fam) 
363              batch.append(cf) 
364          rhnSQL.initDB() 
365          backend.init() 
366          cfimp = ChannelFamilyImport(batch, backend) 
367          cfimp.run() 
368          sys.exit(0) 
369