1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 import sys
20 import os
21 import shutil
22
23 from optparse import Option, OptionParser
24 from spacewalk.common.rhnLog import initLOG, rhnLog
25 from spacewalk.common.rhnConfig import CFG, initCFG
26 from spacewalk.common import rhn_rpm
27 from spacewalk.server.rhnLib import parseRPMFilename, get_package_path
28 from spacewalk.server import rhnSQL, rhnPackageUpload
29 from spacewalk.server.rhnServer import server_packages
30 from spacewalk.satellite_tools.progress_bar import ProgressBar
31 from spacewalk.common.checksum import getFileChecksum
32 from spacewalk.server.importlib import mpmSource
33
34 initCFG('server.satellite')
35 initLOG(CFG.LOG_FILE, CFG.DEBUG)
36
37 OPTIONS = None
38 debug = 0
39 verbose = 0
40
41 options_table = [
42 Option("--update-package-files", action="store_true",
43 help="Update package files (bugs #659348, #652852)"),
44 Option("--update-sha256", action="store_true",
45 help="Update SHA-256 capable packages"),
46 Option("--update-filer", action="store_true",
47 help="Convert filer structure"),
48 Option("--update-kstrees", action="store_true",
49 help="Fix kickstart trees permissions"),
50 Option("--update-changelog", action="store_true",
51 help="Fix incorrectly encoded package changelog data"),
52 Option("-v", "--verbose", action="count",
53 help="Increase verbosity"),
54 Option("--debug", action="store_true",
55 help="Log the debug information to a log file"),
56 ]
57
58
60 global debug, verbose
61 parser = OptionParser(option_list=options_table)
62
63 (options, args) = parser.parse_args()
64
65 if args:
66 for arg in args:
67 sys.stderr.write("Not a valid option ('%s'), try --help\n" % arg)
68 sys.exit(-1)
69
70 if options.verbose:
71 initLOG("stdout", options.verbose or 0)
72 verbose = 1
73
74 if options.debug:
75 initLOG(CFG.LOG_FILE, options.debug or 0)
76 debug = 1
77
78 rhnSQL.initDB()
79
80 if options.update_filer:
81 process_package_data()
82
83 if options.update_sha256:
84 process_sha256_packages()
85
86 if options.update_kstrees:
87 process_kickstart_trees()
88
89 if options.update_package_files:
90 process_package_files()
91
92 if options.update_changelog:
93 process_changelog()
94
95 _get_path_query = """
96 select id, checksum_type, checksum, path, epoch, new_path
97 from (
98 select rhnPackage.id,
99 rhnChecksumView.checksum_type,
100 rhnChecksumView.checksum,
101 rhnPackage.path,
102 rhnPackageEvr.epoch,
103 case when rhnPackage.org_id is null then 'NULL'
104 else rhnPackage.org_id || '' end
105 || '/' || substr(rhnChecksumView.checksum, 1, 3)
106 || '/' || rhnPackageName.name
107 || '/' || case when rhnPackageEvr.epoch is null then ''
108 else rhnPackageEvr.epoch || ':' end
109 || rhnPackageEvr.version || '-' || rhnPackageEvr.release
110 || '/' || rhnPackageArch.label
111 || '/' || rhnChecksumView.checksum
112 || substr(rhnPackage.path, instr(rhnPackage.path, '/', -1))
113 as new_path
114 from rhnPackage, rhnPackagename, rhnPackageEvr, rhnPackageArch, rhnChecksumView
115 where rhnPackage.name_id = rhnPackageName.id
116 and rhnPackage.evr_id = rhnPackageEvr.id
117 and rhnPackage.package_arch_id = rhnPackageArch.id
118 and rhnPackage.checksum_id = rhnChecksumView.id
119 ) X
120 where '/' || new_path <> nvl(substr(path, -length(new_path) - 1), 'x')
121 """
122
123 _update_pkg_path_query = """
124 update rhnPackage
125 set path = :new_path
126 where id = :the_id
127 """
128
129
131 if debug:
132 log = rhnLog('/var/log/rhn/update-packages.log', 5)
133
134 _get_path_sql = rhnSQL.prepare(_get_path_query)
135 _update_package_path = rhnSQL.prepare(_update_pkg_path_query)
136
137 _get_path_sql.execute()
138 paths = _get_path_sql.fetchall_dict()
139
140 if not paths:
141
142 return
143 if verbose:
144 print("Processing %s packages" % len(paths))
145 pb = ProgressBar(prompt='standby: ', endTag=' - Complete!',
146 finalSize=len(paths), finalBarLength=40, stream=sys.stdout)
147 pb.printAll(1)
148 skip_list = []
149 new_ok_list = []
150 i = 0
151 for path in paths:
152 pb.addTo(1)
153 pb.printIncrement()
154 old_path_nvrea = path['path'].split('/')
155 org_id = old_path_nvrea[1]
156
157 try:
158 nevra = parseRPMFilename(old_path_nvrea[-1])
159 if nevra[1] in [None, '']:
160 nevra[1] = path['epoch']
161 except Exception:
162
163 if debug:
164 log.writeMessage("Skipping: %s Not a valid rpm"
165 % old_path_nvrea[-1])
166 continue
167 old_abs_path = os.path.join(CFG.MOUNT_POINT, path['path'])
168
169 checksum_type = path['checksum_type']
170 checksum = path['checksum']
171 new_path = get_package_path(nevra, org_id, prepend=old_path_nvrea[0],
172 checksum=checksum)
173 new_abs_path = os.path.join(CFG.MOUNT_POINT, new_path)
174
175 bad_abs_path = os.path.join(CFG.MOUNT_POINT,
176 get_package_path(nevra, org_id, prepend=old_path_nvrea[0],
177 omit_epoch=True, checksum=checksum))
178
179 if not os.path.exists(old_abs_path):
180 if os.path.exists(new_abs_path):
181 new_ok_list.append(new_abs_path)
182 if debug:
183 log.writeMessage("File %s already on final path %s" % (path['path'], new_abs_path))
184 old_abs_path = new_abs_path
185 elif os.path.exists(bad_abs_path):
186 log.writeMessage("File %s found on %s" % (path['path'], bad_abs_path))
187 old_abs_path = bad_abs_path
188 else:
189 skip_list.append(old_abs_path)
190 if debug:
191 log.writeMessage("Missing path %s for package %d" % (old_abs_path, path['id']))
192 continue
193
194
195 try:
196 hdr = rhn_rpm.get_package_header(filename=old_abs_path)
197 except Exception:
198 e = sys.exc_info()[1]
199 msg = "Exception occurred when reading package header %s: %s" % \
200 (old_abs_path, str(e))
201 print(msg)
202 if debug:
203 log.writeMessage(msg)
204 rhnSQL.commit()
205 sys.exit(1)
206
207 if old_abs_path != new_abs_path:
208 new_abs_dir = os.path.dirname(new_abs_path)
209
210 if debug:
211 log.writeMessage("Relocating %s to %s on filer"
212 % (old_abs_path, new_abs_path))
213 if not os.path.isdir(new_abs_dir):
214 os.makedirs(new_abs_dir)
215 shutil.move(old_abs_path, new_abs_path)
216
217 os.removedirs(os.path.dirname(old_abs_path))
218
219 os.chmod(new_abs_path, int('0644', 8))
220
221
222 _update_package_path.execute(the_id=path['id'],
223 new_path=new_path)
224 if debug:
225 log.writeMessage("query Executed: update rhnPackage %d to %s"
226 % (path['id'], new_path))
227
228 server_packages.processPackageKeyAssociations(hdr, checksum_type, checksum)
229 if debug:
230 log.writeMessage("gpg key info updated from %s" % new_abs_path)
231 i = i + 1
232
233 if i % 1000 == 0:
234 rhnSQL.commit()
235 pb.printComplete()
236
237 rhnSQL.commit()
238 sys.stderr.write("Transaction Committed! \n")
239 if verbose:
240 print(" Skipping %s packages, paths not found" % len(skip_list))
241 if new_ok_list and verbose:
242 print(" There were %s packages found in the correct location" % len(new_ok_list))
243 return
244
245
247 for root, _dirs, files in os.walk(CFG.MOUNT_POINT + "/rhn/"):
248 for name in files:
249 os.chmod(root + '/' + name, int('0644', 8))
250
251 _get_sha256_packages_query = """
252 select p.id, p.path
253 from rhnPackage p,
254 rhnPackageRequires pr,
255 rhnPackageCapability pc,
256 rhnChecksumView cv
257 where pr.package_id = p.id and
258 pr.capability_id = pc.id and
259 pc.name = 'rpmlib(FileDigests)' and
260 pc.version = '4.6.0-1' and
261 cv.id = p.checksum_id and
262 cv.checksum_type = 'md5'
263 """
264
265 _update_sha256_package = """
266 update rhnPackage
267 set checksum_id = lookup_checksum(:ctype, :csum),
268 path = :path
269 where id = :id
270 """
271
272 _select_checksum_type_id = """
273 select id from rhnChecksumType where label = :ctype
274 """
275
276 _update_package_files = """
277 declare
278 checksum_id number;
279 begin
280 begin
281 insert into rhnChecksum values (
282 sequence_nextval('rhnChecksum_seq'),
283 :ctype_id,
284 :csum ) returning id into checksum_id;
285 exception when dup_val_on_index then
286 select c.id
287 into checksum_id
288 from rhnChecksum c
289 where c.checksum = :csum and
290 c.checksum_type_id = :ctype_id;
291 end;
292
293 update rhnPackageFile p
294 set p.checksum_id = checksum_id
295 where p.capability_id = (
296 select c.id
297 from rhnPackageCapability c
298 where p.package_id = :pid and
299 c.name = :filename
300 ) and p.package_id = :pid;
301 end;
302 """
303
304
306 if debug:
307 log = rhnLog('/var/log/rhn/update-packages.log', 5)
308
309 _get_sha256_packages_sql = rhnSQL.prepare(_get_sha256_packages_query)
310 _get_sha256_packages_sql.execute()
311 packages = _get_sha256_packages_sql.fetchall_dict()
312
313 if not packages:
314 print("No SHA256 capable packages to process.")
315 if debug:
316 log.writeMessage("No SHA256 capable packages to process.")
317
318 return
319
320 if verbose:
321 print("Processing %s SHA256 capable packages" % len(packages))
322
323 pb = ProgressBar(prompt='standby: ', endTag=' - Complete!',
324 finalSize=len(packages), finalBarLength=40, stream=sys.stdout)
325 pb.printAll(1)
326
327 _update_sha256_package_sql = rhnSQL.prepare(_update_sha256_package)
328 _update_package_files_sql = rhnSQL.prepare(_update_package_files)
329
330 for package in packages:
331 pb.addTo(1)
332 pb.printIncrement()
333
334 old_abs_path = os.path.join(CFG.MOUNT_POINT, package['path'])
335 if debug and verbose:
336 log.writeMessage("Processing package: %s" % old_abs_path)
337 temp_file = open(old_abs_path, 'rb')
338 header, _payload_stream, _header_start, _header_end = \
339 rhnPackageUpload.load_package(temp_file)
340 checksum_type = header.checksum_type()
341 checksum = getFileChecksum(checksum_type, file_obj=temp_file)
342
343 old_path = package['path'].split('/')
344 nevra = parseRPMFilename(old_path[-1])
345 org_id = old_path[1]
346 new_path = get_package_path(nevra, org_id, prepend=old_path[0], checksum=checksum)
347 new_abs_path = os.path.join(CFG.MOUNT_POINT, new_path)
348
349
350 try:
351 if old_abs_path != new_abs_path:
352 if debug:
353 log.writeMessage("Relocating %s to %s on filer" % (old_abs_path, new_abs_path))
354
355 new_abs_dir = os.path.dirname(new_abs_path)
356 if not os.path.isdir(new_abs_dir):
357 os.makedirs(new_abs_dir)
358
359
360 if not os.path.exists(new_abs_path):
361 os.link(old_abs_path, new_abs_path)
362 elif debug:
363 log.writeMessage("File %s already exists" % new_abs_path)
364
365
366 os.chmod(new_abs_path, int('0644', 8))
367 except OSError:
368 e = sys.exc_info()[1]
369 message = "Error when relocating %s to %s on filer: %s" % \
370 (old_abs_path, new_abs_path, str(e))
371 print(message)
372 if debug:
373 log.writeMessage(message)
374 sys.exit(1)
375
376
377 _update_sha256_package_sql.execute(ctype=checksum_type, csum=checksum,
378 path=new_path, id=package['id'])
379
380 _select_checksum_type_id_sql = rhnSQL.prepare(_select_checksum_type_id)
381 _select_checksum_type_id_sql.execute(ctype=checksum_type)
382 checksum_type_id = _select_checksum_type_id_sql.fetchone()[0]
383
384
385 for i, f in enumerate(header['filenames']):
386 csum = header['filemd5s'][i]
387
388
389 if not csum:
390 continue
391
392 _update_package_files_sql.execute(ctype_id=checksum_type_id, csum=csum,
393 pid=package['id'], filename=f)
394
395 rhnSQL.commit()
396
397 try:
398 if os.path.exists(old_abs_path):
399 os.unlink(old_abs_path)
400 if os.path.exists(os.path.dirname(old_abs_path)):
401 os.removedirs(os.path.dirname(old_abs_path))
402 except OSError:
403 e = sys.exc_info()[1]
404 message = "Error when removing %s: %s" % (old_abs_path, str(e))
405 print(message)
406 if debug:
407 log.writeMessage(message)
408
409 sys.exit(1)
410
411 pb.printComplete()
412
413 package_query = """
414 select p.id as id,
415 p.path as path,
416 count(pf.capability_id) as filecount,
417 count(pf.checksum_id) as nonnullcsums
418 from rhnPackage p left outer join rhnPackageFile pf
419 on p.id = pf.package_id
420 where path is not null
421 group by id, path
422 """
423
424 package_capabilities = """
425 select PC.name,
426 PF.package_id,
427 PF.capability_id,
428 C.checksum,
429 C.checksum_type
430 from rhnPackageFile PF left outer join rhnChecksumView C
431 on PF.checksum_id = C.id,
432 rhnPackageCapability PC
433 where PC.id = PF.capability_id and
434 PF.package_id = :pid
435 """
436
437 update_packagefile_checksum = """
438 update rhnPackageFile
439 set checksum_id = lookup_checksum(:ctype, :csum)
440 where package_id = :pid and
441 capability_id = :cid
442 """
443
444 insert_packagefile = """
445 insert into rhnPackageFile (
446 package_id, capability_id, device, inode, file_mode, username,
447 groupname, rdev, file_size, mtime, linkto, flags, verifyflags,
448 lang, checksum_id
449 )
450 values (
451 :pid, lookup_package_capability(:name, null),
452 :device, :inode, :file_mode, :username, :groupname,
453 :rdev, :file_size, to_timestamp(:mtime, 'YYYY-MM-DD HH24:MI:SS'), :linkto,
454 :flags, :verifyflags, :lang, lookup_checksum(:ctype, :csum)
455 )
456 """
457
458 package_name_query = """
459 select pn.name as name,
460 evr_t_as_vre_simple(pevr.evr) as vre,
461 pa.label as arch
462 from rhnPackage p,
463 rhnPackageName pn,
464 rhnPackageEVR pevr,
465 rhnPackageArch pa
466 where p.id = :pid and
467 p.name_id = pn.id and
468 p.evr_id = pevr.id and
469 p.package_arch_id = pa.id
470 """
471
472 package_repodata_delete = """
473 delete
474 from rhnPackageRepoData
475 where package_id = :pid
476 """
477
478
485
486 package_name_h = rhnSQL.prepare(package_name_query)
487
488 def package_name(pid):
489 package_name_h.execute(pid=pid)
490 r = package_name_h.fetchall_dict()[0]
491 return "%s-%s.%s" % (r['name'], r['vre'], r['arch'])
492
493 package_repodata_h = rhnSQL.prepare(package_repodata_delete)
494
495 def delete_package_repodata(pid):
496 package_repodata_h.execute(pid=pid)
497
498 log = rhnLog('/var/log/rhn/update-packages.log', 5)
499
500 package_query_h = rhnSQL.prepare(package_query)
501 package_query_h.execute()
502
503 package_capabilities_h = rhnSQL.prepare(package_capabilities)
504 update_packagefile_checksum_h = rhnSQL.prepare(update_packagefile_checksum)
505 insert_packagefile_h = rhnSQL.prepare(insert_packagefile)
506
507 while (True):
508 row = package_query_h.fetchone_dict()
509 if not row:
510 break
511
512 package_path = os.path.join(CFG.MOUNT_POINT, row['path'])
513
514 if not os.path.exists(package_path):
515 if debug:
516 log.writeMessage("Package path '%s' does not exist." % package_path)
517 continue
518
519
520 try:
521 hdr = rhn_rpm.get_package_header(filename=package_path)
522 except Exception:
523 e = sys.exc_info()[1]
524 message = "Error when reading package %s header: %s" % (package_path, e)
525 if debug:
526 log.writeMessage(message)
527 continue
528
529 pkg_updates = 0
530 if row['filecount'] != len(hdr['filenames']):
531
532
533 package_capabilities_h.execute(pid=row['id'])
534 pkg_caps = {}
535 for cap in package_capabilities_h.fetchall_dict() or []:
536 pkg_caps[cap['name']] = cap
537
538 for f in parse_header(hdr)['files']:
539 if f['name'] in pkg_caps:
540 continue
541
542
543 insert_packagefile_h.execute(pid=row['id'], name=f['name'],
544 ctype=f['checksum_type'], csum=f['checksum'], device=f['device'],
545 inode=f['inode'], file_mode=f['file_mode'], username=f['username'],
546 groupname=f['groupname'], rdev=f['rdev'], file_size=f['file_size'],
547 mtime=f['mtime'], linkto=f['linkto'], flags=f['flags'],
548 verifyflags=f['verifyflags'], lang=f['lang'])
549 pkg_updates += 1
550
551 if debug and pkg_updates:
552 log.writeMessage("Package id: %s, name: %s, %s files inserted" %
553 (row['id'], package_name(row['id']), pkg_updates))
554 elif row['nonnullcsums'] == 0:
555
556 package_capabilities_h.execute(pid=row['id'])
557 pkg_caps = {}
558 for cap in package_capabilities_h.fetchall_dict() or []:
559 pkg_caps[cap['name']] = cap
560
561 for f in parse_header(hdr)['files']:
562 if f['checksum'] == '':
563 f['checksum'] = None
564
565 caps = pkg_caps[f['name']]
566
567 if not caps['checksum'] == f['checksum']:
568
569 update_packagefile_checksum_h.execute(ctype=f['checksum_type'], csum=f['checksum'],
570 pid=caps['package_id'], cid=caps['capability_id'])
571 pkg_updates += 1
572
573 if debug and pkg_updates:
574 log.writeMessage("Package id: %s, name: %s, %s checksums updated" %
575 (row['id'], package_name(row['id']), pkg_updates))
576
577 if pkg_updates:
578 log.writeMessage("Package id: %s, purging rhnPackageRepoData" % row['id'])
579 delete_package_repodata(row['id'])
580
581 rhnSQL.commit()
582
583
585 def convert(u):
586 last = ''
587 while u != last:
588 last = u
589 try:
590 u = last.encode('iso8859-1').decode('utf8')
591 except (UnicodeDecodeError, UnicodeEncodeError):
592 e = sys.exc_info()[1]
593 if e.reason == 'unexpected end of data':
594 u = u[:-1]
595 continue
596 else:
597 break
598 return u
599
600 if CFG.db_backend == 'postgresql':
601 lengthb = "octet_length(%s)"
602 else:
603 lengthb = "lengthb(%s)"
604 _non_ascii_changelog_data_count = """select count(*) as cnt from rhnpackagechangelogdata
605 where length(name) <> %s
606 or length(text) <> %s
607 """ % (lengthb % 'name', lengthb % 'text')
608 _non_ascii_changelog_data = """select * from rhnpackagechangelogdata
609 where length(name) <> %s
610 or length(text) <> %s
611 """ % (lengthb % 'name', lengthb % 'text')
612 _update_changelog_data_name = """update rhnpackagechangelogdata set name = :name
613 where id = :id"""
614 _update_changelog_data_text = """update rhnpackagechangelogdata set text = :text
615 where id = :id"""
616 if debug:
617 log = rhnLog('/var/log/rhn/update-packages.log', 5)
618
619 query_count = rhnSQL.prepare(_non_ascii_changelog_data_count)
620 query_count.execute()
621 nrows = query_count.fetchall_dict()[0]['cnt']
622
623 query = rhnSQL.prepare(_non_ascii_changelog_data)
624 query.execute()
625
626 if nrows == 0:
627 msg = "No non-ASCII changelog entries to process."
628 print(msg)
629 if debug:
630 log.writeMessage(msg)
631 return
632
633 if verbose:
634 print("Processing %s non-ASCII changelog entries" % nrows)
635
636 pb = ProgressBar(prompt='standby: ', endTag=' - Complete!',
637 finalSize=nrows, finalBarLength=40, stream=sys.stdout)
638 pb.printAll(1)
639
640 update_name = rhnSQL.prepare(_update_changelog_data_name)
641 update_text = rhnSQL.prepare(_update_changelog_data_text)
642
643 while (True):
644 row = query.fetchone_dict()
645 if not row:
646 break
647
648 pb.addTo(1)
649 pb.printIncrement()
650
651 name_u = row['name'].decode('utf8', 'ignore')
652 name_fixed = name_u
653 if len(row['name']) != len(name_u):
654 name_fixed = convert(name_u)
655 if name_fixed != name_u:
656 if debug and verbose:
657 log.writeMessage("Fixing record %s: name: '%s'" % (row['id'], row['name']))
658 update_name.execute(id=row['id'], name=name_fixed)
659
660 text_u = row['text'].decode('utf8', 'ignore')
661 text_fixed = text_u
662 if len(row['text']) != len(text_u):
663 text_fixed = convert(text_u)
664 if text_fixed != text_u:
665 if debug and verbose:
666 log.writeMessage("Fixing record %s: text: '%s'" % (row['id'], row['text']))
667 update_text.execute(id=row['id'], text=text_fixed)
668
669 rhnSQL.commit()
670
671 pb.printComplete()
672
673
674 if __name__ == '__main__':
675 main()
676