tools/ftp/spider_yum.py

   1 #!/usr/bin/env python
   2 import argparse
   3 import sys
   4 import os
   5 import re
   6 import json
   7 import urllib2
   8 from decimal import Decimal
   9 from tempfile import NamedTemporaryFile
  10
  11 platform_names = {
  12         'redhat': 'RedHat Enterprise Linux {0}',
  13         'centos': 'CentOS {0}',
  14         'sl': 'Scientific Linux {0}',
  15         'fedora': 'Fedora {0}',
  16         'oraclelinux': 'Oracle Enterprise Linux {0}',
  17         'ami201503-': 'Amazon Linux AMI201503 {0}',
  18 }
  19 platform_sort = {
  20         'redhat': 1,
  21         'centos': 2,
  22         'sl': 3,
  23         'fedora': 4,
  24         'oraclelinux': 5,
  25         'ami201503-': 6,
  26 }
  27 archs = ['x86_64', 'i386', 'i686', 'ppc64le']
  28
  29 def generate_platform(dirname, familyprefix, ver, installer, systemd):
  30         for f in platform_names.keys():
  31                 yield ('%s-%s' % (f, ver), {
  32                         't': platform_names[f].format(ver),
  33                         'p': os.path.join(dirname, '{0}-{1}'.format(familyprefix, ver)),
  34                         'f': f,
  35                         'i': installer,
  36                         'd': systemd,
  37                         's': platform_sort[f]*1000-ver,
  38                         'found': False,
  39                         })
  40
  41 def get_redhat_systemd(ver):
  42         return (ver >= 7)
  43
  44 platforms = {}
  45 for v in range(5, 7+1):
  46         platforms.update(dict(generate_platform('redhat', 'rhel', v, 'yum', get_redhat_systemd(v))))
  47 for v in range(24, 25+1):
  48         platforms.update(dict(generate_platform('fedora', 'fedora', v, 'dnf', True)))
  49
  50 re_reporpm = re.compile('^pgdg-([a-z0-9-]+)([0-9]{2})-[^-]+-(\d+)\.noarch\.rpm$')
  51 re_versiondirs = re.compile(r'^\d+(\.\d+)?$')
  52
  53 if __name__ == "__main__":
  54         parser = argparse.ArgumentParser(description="Spider repo RPMs")
  55         parser.add_argument('yumroot', type=str, help='YUM root path')
  56         parser.add_argument('target', type=str, help='Target URL or filename')
  57
  58         args = parser.parse_args()
  59
  60         versions = sorted([v for v in os.listdir(args.yumroot) if re_versiondirs.match(v)], key=Decimal, reverse=True)
  61         reporpms = {}
  62         for v in versions:
  63                 reporpms[v] = {}
  64                 vroot = os.path.join(args.yumroot, v)
  65                 for dirpath, dirnames, filenames in os.walk(vroot):
  66                         rmatches = filter(None, (re_reporpm.match(f) for f in sorted(filenames, reverse=True)))
  67
  68                         if rmatches:
  69                                 familypath = os.path.join(*dirpath.split('/')[-2:])
  70                                 (familypath, arch) = familypath.rsplit('-', 1)
  71
  72                                 for r in rmatches:
  73                                         shortdist, shortver, ver = r.groups(1)
  74
  75                                         found = False
  76                                         for p, pinfo in platforms.items():
  77                                                 if pinfo['p'] == familypath and pinfo['f'] == shortdist:
  78                                                         if not reporpms[v].has_key(p):
  79                                                                 reporpms[v][p] = {}
  80                                                         reporpms[v][p][arch] = max(ver, reporpms[v].get(p, 0))
  81                                                         platforms[p]['found'] = True
  82                                                         break
  83                                         else:
  84                                                 # DEBUG
  85 #                                               print "%s (%s) not found in platform list" % (familypath, shortdist)
  86                                                 pass
  87
  88         # Filter all platforms that are not used
  89         platforms = {k:v for k,v in platforms.iteritems() if v['found']}
  90         for k,v in platforms.iteritems():
  91                 del v['found']
  92
  93         j = json.dumps({'platforms': platforms, 'reporpms': reporpms})
  94
  95         if args.target.startswith('http://') or args.target.startswith('https://'):
  96                 o = urllib2.build_opener(urllib2.HTTPHandler)
  97                 r = urllib2.Request(sys.argv[2], data=j)
  98                 r.add_header('Content-type', 'application/json')
  99                 r.add_header('Host', 'www.postgresql.org')
 100                 r.get_method = lambda: 'PUT'
 101                 u = o.open(r)
 102                 x = u.read()
 103                 if x != "NOT CHANGED" and x != "OK":
 104                         print "Failed to upload: %s" % x
 105                         sys.exit(1)
 106         else:
 107                 with NamedTemporaryFile(dir=os.path.dirname(os.path.abspath(args.target))) as f:
 108                         f.write(j)
 109                         f.flush()
 110                         if os.path.isfile(args.target):
 111                                 os.unlink(args.target)
 112                         os.link(f.name, args.target)