utils/packager.py

   1 #!/usr/bin/python -u
   2
   3 """
   4 Utility to upload or remove the packages from the packages repository.
   5 """
   6
   7 import logging, os, sys, optparse, socket, tempfile, shutil
   8 import common
   9 from autotest_lib.client.common_lib import utils as client_utils
  10 from autotest_lib.client.common_lib import global_config, error
  11 from autotest_lib.client.common_lib import base_packages, packages
  12 from autotest_lib.server import utils as server_utils
  13
  14 c = global_config.global_config
  15 logging.basicConfig(level=logging.DEBUG)
  16
  17 def get_exclude_string(client_dir):
  18     '''
  19     Get the exclude string for the tar command to exclude specific
  20     subdirectories inside client_dir.
  21     For profilers we need to exclude everything except the __ini__.py
  22     file so that the profilers can be imported.
  23     '''
  24     exclude_string = ('--exclude=deps/* --exclude=tests/* '
  25                       '--exclude=site_tests/*')
  26
  27     # Get the profilers directory
  28     prof_dir = os.path.join(client_dir, 'profilers')
  29
  30     # Include the __init__.py file for the profilers and exclude all its
  31     # subdirectories
  32     for f in os.listdir(prof_dir):
  33         if os.path.isdir(os.path.join(prof_dir, f)):
  34             exclude_string += ' --exclude=profilers/%s' % f
  35
  36     # The '.' here is needed to zip the files in the current
  37     # directory. We use '-C' for tar to change to the required
  38     # directory i.e. src_dir and then zip up the files in that
  39     # directory(which is '.') excluding the ones in the exclude_dirs
  40     exclude_string += " ."
  41
  42     return exclude_string
  43
  44
  45 def parse_args():
  46     parser = optparse.OptionParser()
  47     parser.add_option("-d", "--dependency", help="package the dependency"
  48                       " from client/deps directory and upload to the repo",
  49                       dest="dep")
  50     parser.add_option("-p", "--profiler", help="package the profiler "
  51                       "from client/profilers directory and upload to the repo",
  52                       dest="prof")
  53     parser.add_option("-t", "--test", help="package the test from client/tests"
  54                       " or client/site_tests and upload to the repo.",
  55                       dest="test")
  56     parser.add_option("-c", "--client", help="package the client "
  57                       "directory alone without the tests, deps and profilers",
  58                       dest="client", action="store_true", default=False)
  59     parser.add_option("-f", "--file", help="simply uploads the specified"
  60                       "file on to the repo", dest="file")
  61     parser.add_option("-r", "--repository", help="the URL of the packages"
  62                       "repository location to upload the packages to.",
  63                       dest="repo", default=None)
  64     parser.add_option("--all", help="Upload all the files locally "
  65                       "to all the repos specified in global_config.ini. "
  66                       "(includes the client, tests, deps and profilers)",
  67                       dest="all", action="store_true", default=False)
  68
  69     options, args = parser.parse_args()
  70     return options, args
  71
  72
  73 # Method to upload or remove package depending on the flag passed to it.
  74 def process_packages(pkgmgr, pkg_type, pkg_names, src_dir,
  75                     remove=False):
  76     exclude_string = ' .'
  77     names = [p.strip() for p in pkg_names.split(',')]
  78     for name in names:
  79         print "Processing %s ... " % name
  80         if pkg_type=='client':
  81             pkg_dir = src_dir
  82             exclude_string  = get_exclude_string(pkg_dir)
  83         elif pkg_type=='test':
  84             # if the package is a test then look whether it is in client/tests
  85             # or client/site_tests
  86             pkg_dir = os.path.join(get_test_dir(name, src_dir), name)
  87         else:
  88             # for the profilers and deps
  89             pkg_dir = os.path.join(src_dir, name)
  90
  91         pkg_name = pkgmgr.get_tarball_name(name, pkg_type)
  92         if not remove:
  93             # Tar the source and upload
  94             temp_dir = tempfile.mkdtemp()
  95             try:
  96                 try:
  97                     base_packages.check_diskspace(temp_dir)
  98                 except error.RepoDiskFullError, e:
  99                     msg = ("Temporary directory for packages %s does not have "
 100                            "enough space available: %s" % (temp_dir, e))
 101                     raise error.RepoDiskFullError(msg)
 102                 tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir,
 103                                                   temp_dir, exclude_string)
 104                 pkgmgr.upload_pkg(tarball_path, update_checksum=True)
 105             finally:
 106                 # remove the temporary directory
 107                 shutil.rmtree(temp_dir)
 108         else:
 109             pkgmgr.remove_pkg(pkg_name, remove_checksum=True)
 110         print "Done."
 111
 112
 113 def tar_packages(pkgmgr, pkg_type, pkg_names, src_dir, temp_dir):
 114     """Tar all packages up and return a list of each tar created"""
 115     tarballs = []
 116     exclude_string = ' .'
 117     names = [p.strip() for p in pkg_names.split(',')]
 118     for name in names:
 119         print "Processing %s ... " % name
 120         if pkg_type=='client':
 121             pkg_dir = src_dir
 122             exclude_string  = get_exclude_string(pkg_dir)
 123         elif pkg_type=='test':
 124             # if the package is a test then look whether it is in client/tests
 125             # or client/site_tests
 126             pkg_dir = os.path.join(get_test_dir(name, src_dir), name)
 127         else:
 128             # for the profilers and deps
 129             pkg_dir = os.path.join(src_dir, name)
 130
 131         pkg_name = pkgmgr.get_tarball_name(name, pkg_type)
 132         tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir,
 133                                               temp_dir, exclude_string)
 134
 135         tarballs.append(tarball_path)
 136
 137     return tarballs
 138
 139
 140 def process_all_packages(pkgmgr, client_dir, remove=False):
 141     """Process a full upload of packages as a directory upload."""
 142     test_dir = os.path.join(client_dir, "tests")
 143     site_test_dir = os.path.join(client_dir, "site_tests")
 144     dep_dir = os.path.join(client_dir, "deps")
 145     prof_dir = os.path.join(client_dir, "profilers")
 146     # Directory where all are kept
 147     temp_dir = tempfile.mkdtemp()
 148     try:
 149         base_packages.check_diskspace(temp_dir)
 150     except error.RepoDiskFullError, e:
 151         print ("Temp destination for packages is full %s, aborting upload: %s"
 152                % (temp_dir, e))
 153         os.rmdir(temp_dir)
 154         sys.exit(1)
 155
 156     # process tests
 157     tests_list = get_subdir_list('tests', client_dir)
 158     tests = ','.join(tests_list)
 159
 160     # process site_tests
 161     site_tests_list = get_subdir_list('site_tests', client_dir)
 162     site_tests = ','.join(site_tests_list)
 163
 164     # process deps
 165     deps_list = get_subdir_list('deps', client_dir)
 166     deps = ','.join(deps_list)
 167
 168     # process profilers
 169     profilers_list = get_subdir_list('profilers', client_dir)
 170     profilers = ','.join(profilers_list)
 171
 172     # Update md5sum
 173     if not remove:
 174         tar_packages(pkgmgr, 'profiler', profilers, prof_dir, temp_dir)
 175         tar_packages(pkgmgr, 'dep', deps, dep_dir, temp_dir)
 176         tar_packages(pkgmgr, 'test', site_tests, client_dir, temp_dir)
 177         tar_packages(pkgmgr, 'test', tests, client_dir, temp_dir)
 178         tar_packages(pkgmgr, 'client', 'autotest', client_dir, temp_dir)
 179         cwd = os.getcwd()
 180         os.chdir(temp_dir)
 181         client_utils.system('md5sum * > packages.checksum')
 182         os.chdir(cwd)
 183         pkgmgr.upload_pkg(temp_dir)
 184         client_utils.run('rm -rf ' + temp_dir)
 185     else:
 186         process_packages(pkgmgr, 'test', tests, client_dir,remove=remove)
 187         process_packages(pkgmgr, 'test', site_tests, client_dir, remove=remove)
 188         process_packages(pkgmgr, 'client', 'autotest', client_dir,
 189                          remove=remove)
 190         process_packages(pkgmgr, 'dep', deps, dep_dir, remove=remove)
 191         process_packages(pkgmgr, 'profiler', profilers, prof_dir,
 192                          remove=remove)
 193
 194
 195 # Get the list of sub directories present in a directory
 196 def get_subdir_list(name, client_dir):
 197     dir_name = os.path.join(client_dir, name)
 198     return [f for f in
 199             os.listdir(dir_name)
 200             if os.path.isdir(os.path.join(dir_name, f)) ]
 201
 202
 203 # Look whether the test is present in client/tests and client/site_tests dirs
 204 def get_test_dir(name, client_dir):
 205     names_test = os.listdir(os.path.join(client_dir, 'tests'))
 206     names_site_test = os.listdir(os.path.join(client_dir, 'site_tests'))
 207     if name in names_test:
 208         src_dir = os.path.join(client_dir, 'tests')
 209     elif name in names_site_test:
 210         src_dir = os.path.join(client_dir, 'site_tests')
 211     else:
 212         print "Test %s not found" % name
 213         sys.exit(0)
 214     return src_dir
 215
 216
 217 def main():
 218     # get options and args
 219     options, args = parse_args()
 220
 221     server_dir = server_utils.get_server_dir()
 222     autotest_dir = os.path.abspath(os.path.join(server_dir, '..'))
 223
 224     # extract the pkg locations from global config
 225     repo_urls = c.get_config_value('PACKAGES', 'fetch_location',
 226                                    type=list, default=[])
 227     upload_paths = c.get_config_value('PACKAGES', 'upload_location',
 228                                       type=list, default=[])
 229     # Having no upload paths basically means you're not using packaging.
 230     if len(upload_paths) == 0:
 231         return
 232
 233     client_dir = os.path.join(autotest_dir, "client")
 234
 235     # Bail out if the client directory does not exist
 236     if not os.path.exists(client_dir):
 237         sys.exit(0)
 238
 239     dep_dir = os.path.join(client_dir, "deps")
 240     prof_dir = os.path.join(client_dir, "profilers")
 241
 242     if len(args)==0 or args[0] not in ['upload','remove']:
 243         print("Either 'upload' or 'remove' needs to be specified "
 244               "for the package")
 245         sys.exit(0)
 246
 247     if args[0]=='upload':
 248         remove_flag=False
 249     elif args[0]=='remove':
 250         remove_flag=True
 251     else:
 252         # we should not be getting here
 253         assert(False)
 254
 255     if options.repo:
 256         upload_path_list = [options.repo]
 257     else:
 258         upload_path_list = upload_paths
 259
 260     pkgmgr = packages.PackageManager(autotest_dir, repo_urls=repo_urls,
 261                                      upload_paths=upload_path_list,
 262                                      run_function_dargs={'timeout':600})
 263
 264     if options.all:
 265         process_all_packages(pkgmgr, client_dir, remove=remove_flag)
 266
 267     if options.client:
 268         process_packages(pkgmgr, 'client', 'autotest', client_dir,
 269                          remove=remove_flag)
 270
 271     if options.dep:
 272         process_packages(pkgmgr, 'dep', options.dep, dep_dir,
 273                          remove=remove_flag)
 274
 275     if options.test:
 276         process_packages(pkgmgr, 'test', options.test, client_dir,
 277                          remove=remove_flag)
 278
 279     if options.prof:
 280         process_packages(pkgmgr, 'profiler', options.prof, prof_dir,
 281                          remove=remove_flag)
 282
 283     if options.file:
 284         if remove_flag:
 285             pkgmgr.remove_pkg(options.file, remove_checksum=True)
 286         else:
 287             pkgmgr.upload_pkg(options.file, update_checksum=True)
 288
 289
 290 if __name__ == "__main__":
 291     main()