4 Utility to upload or remove the packages from the packages repository.
7 import logging
, os
, sys
, optparse
, socket
, tempfile
, shutil
9 from autotest_lib
.client
.common_lib
import utils
as client_utils
10 from autotest_lib
.client
.common_lib
import global_config
, error
11 from autotest_lib
.client
.common_lib
import base_packages
, packages
12 from autotest_lib
.server
import utils
as server_utils
14 c
= global_config
.global_config
15 logging
.basicConfig(level
=logging
.DEBUG
)
17 def get_exclude_string(client_dir
):
19 Get the exclude string for the tar command to exclude specific
20 subdirectories inside client_dir.
21 For profilers we need to exclude everything except the __ini__.py
22 file so that the profilers can be imported.
24 exclude_string
= ('--exclude=deps/* --exclude=tests/* '
25 '--exclude=site_tests/*')
27 # Get the profilers directory
28 prof_dir
= os
.path
.join(client_dir
, 'profilers')
30 # Include the __init__.py file for the profilers and exclude all its
32 for f
in os
.listdir(prof_dir
):
33 if os
.path
.isdir(os
.path
.join(prof_dir
, f
)):
34 exclude_string
+= ' --exclude=profilers/%s' % f
36 # The '.' here is needed to zip the files in the current
37 # directory. We use '-C' for tar to change to the required
38 # directory i.e. src_dir and then zip up the files in that
39 # directory(which is '.') excluding the ones in the exclude_dirs
40 exclude_string
+= " ."
46 parser
= optparse
.OptionParser()
47 parser
.add_option("-d", "--dependency", help="package the dependency"
48 " from client/deps directory and upload to the repo",
50 parser
.add_option("-p", "--profiler", help="package the profiler "
51 "from client/profilers directory and upload to the repo",
53 parser
.add_option("-t", "--test", help="package the test from client/tests"
54 " or client/site_tests and upload to the repo.",
56 parser
.add_option("-c", "--client", help="package the client "
57 "directory alone without the tests, deps and profilers",
58 dest
="client", action
="store_true", default
=False)
59 parser
.add_option("-f", "--file", help="simply uploads the specified"
60 "file on to the repo", dest
="file")
61 parser
.add_option("-r", "--repository", help="the URL of the packages"
62 "repository location to upload the packages to.",
63 dest
="repo", default
=None)
64 parser
.add_option("--all", help="Upload all the files locally "
65 "to all the repos specified in global_config.ini. "
66 "(includes the client, tests, deps and profilers)",
67 dest
="all", action
="store_true", default
=False)
69 options
, args
= parser
.parse_args()
73 # Method to upload or remove package depending on the flag passed to it.
74 def process_packages(pkgmgr
, pkg_type
, pkg_names
, src_dir
, repo_url
,
77 names
= [p
.strip() for p
in pkg_names
.split(',')]
79 print "Processing %s ... " % name
80 if pkg_type
=='client':
82 exclude_string
= get_exclude_string(pkg_dir
)
83 elif pkg_type
=='test':
84 # if the package is a test then look whether it is in client/tests
85 # or client/site_tests
86 pkg_dir
= os
.path
.join(get_test_dir(name
, src_dir
), name
)
88 # for the profilers and deps
89 pkg_dir
= os
.path
.join(src_dir
, name
)
91 pkg_name
= pkgmgr
.get_tarball_name(name
, pkg_type
)
93 # Tar the source and upload
94 temp_dir
= tempfile
.mkdtemp()
97 base_packages
.check_diskspace(temp_dir
)
98 except error
.RepoDiskFullError
, e
:
99 msg
= ("Temporary directory for packages %s does not have "
100 "enough space available: %s" % (temp_dir
, e
))
101 raise error
.RepoDiskFullError(msg
)
102 tarball_path
= pkgmgr
.tar_package(pkg_name
, pkg_dir
,
103 temp_dir
, exclude_string
)
104 pkgmgr
.upload_pkg(tarball_path
, repo_url
, update_checksum
=True)
106 # remove the temporary directory
107 shutil
.rmtree(temp_dir
)
109 pkgmgr
.remove_pkg(pkg_name
, repo_url
, remove_checksum
=True)
113 def tar_packages(pkgmgr
, pkg_type
, pkg_names
, src_dir
, temp_dir
):
114 """Tar all packages up and return a list of each tar created"""
116 exclude_string
= ' .'
117 names
= [p
.strip() for p
in pkg_names
.split(',')]
119 print "Processing %s ... " % name
120 if pkg_type
=='client':
122 exclude_string
= get_exclude_string(pkg_dir
)
123 elif pkg_type
=='test':
124 # if the package is a test then look whether it is in client/tests
125 # or client/site_tests
126 pkg_dir
= os
.path
.join(get_test_dir(name
, src_dir
), name
)
128 # for the profilers and deps
129 pkg_dir
= os
.path
.join(src_dir
, name
)
131 pkg_name
= pkgmgr
.get_tarball_name(name
, pkg_type
)
132 tarball_path
= pkgmgr
.tar_package(pkg_name
, pkg_dir
,
133 temp_dir
, exclude_string
)
135 tarballs
.append(tarball_path
)
140 def process_all_packages(pkgmgr
, client_dir
, upload_paths
, remove
=False):
141 """Process a full upload of packages as a directory upload."""
142 test_dir
= os
.path
.join(client_dir
, "tests")
143 site_test_dir
= os
.path
.join(client_dir
, "site_tests")
144 dep_dir
= os
.path
.join(client_dir
, "deps")
145 prof_dir
= os
.path
.join(client_dir
, "profilers")
146 # Directory where all are kept
147 temp_dir
= tempfile
.mkdtemp()
149 base_packages
.check_diskspace(temp_dir
)
150 except error
.RepoDiskFullError
, e
:
151 print ("Temp destination for packages is full %s, aborting upload: %s"
157 tests_list
= get_subdir_list('tests', client_dir
)
158 tests
= ','.join(tests_list
)
161 site_tests_list
= get_subdir_list('site_tests', client_dir
)
162 site_tests
= ','.join(site_tests_list
)
165 deps_list
= get_subdir_list('deps', client_dir
)
166 deps
= ','.join(deps_list
)
169 profilers_list
= get_subdir_list('profilers', client_dir
)
170 profilers
= ','.join(profilers_list
)
174 tar_packages(pkgmgr
, 'profiler', profilers
, prof_dir
, temp_dir
)
175 tar_packages(pkgmgr
, 'dep', deps
, dep_dir
, temp_dir
)
176 tar_packages(pkgmgr
, 'test', site_tests
, client_dir
, temp_dir
)
177 tar_packages(pkgmgr
, 'test', tests
, client_dir
, temp_dir
)
178 tar_packages(pkgmgr
, 'client', 'autotest', client_dir
, temp_dir
)
181 client_utils
.system('md5sum * > packages.checksum')
183 for path
in upload_paths
:
184 print "Uploading to: " + path
185 pkgmgr
.upload_pkg(temp_dir
, path
)
186 client_utils
.run('rm -rf ' + temp_dir
)
188 for repo_url
in upload_paths
:
189 process_packages(pkgmgr
, 'test', tests
, client_dir
, repo_url
,
191 process_packages(pkgmgr
, 'test', site_tests
, client_dir
, repo_url
,
193 process_packages(pkgmgr
, 'client', 'autotest', client_dir
, repo_url
,
195 process_packages(pkgmgr
, 'dep', deps
, dep_dir
, repo_url
,
197 process_packages(pkgmgr
, 'profiler', profilers
, prof_dir
, repo_url
,
201 # Get the list of sub directories present in a directory
202 def get_subdir_list(name
, client_dir
):
203 dir_name
= os
.path
.join(client_dir
, name
)
206 if os
.path
.isdir(os
.path
.join(dir_name
, f
)) ]
209 # Look whether the test is present in client/tests and client/site_tests dirs
210 def get_test_dir(name
, client_dir
):
211 names_test
= os
.listdir(os
.path
.join(client_dir
, 'tests'))
212 names_site_test
= os
.listdir(os
.path
.join(client_dir
, 'site_tests'))
213 if name
in names_test
:
214 src_dir
= os
.path
.join(client_dir
, 'tests')
215 elif name
in names_site_test
:
216 src_dir
= os
.path
.join(client_dir
, 'site_tests')
218 print "Test %s not found" % name
224 # get options and args
225 options
, args
= parse_args()
227 server_dir
= server_utils
.get_server_dir()
228 autotest_dir
= os
.path
.abspath(os
.path
.join(server_dir
, '..'))
230 # extract the pkg locations from global config
231 repo_urls
= c
.get_config_value('PACKAGES', 'fetch_location',
232 type=list, default
=[])
233 upload_paths
= c
.get_config_value('PACKAGES', 'upload_location',
234 type=list, default
=[])
235 # Having no upload paths basically means you're not using packaging.
236 if len(upload_paths
) == 0:
239 pkgmgr
= packages
.PackageManager(autotest_dir
, repo_urls
=repo_urls
,
240 upload_paths
=upload_paths
,
241 run_function_dargs
={'timeout':600})
243 client_dir
= os
.path
.join(autotest_dir
, "client")
245 # Bail out if the client directory does not exist
246 if not os
.path
.exists(client_dir
):
249 dep_dir
= os
.path
.join(client_dir
, "deps")
250 prof_dir
= os
.path
.join(client_dir
, "profilers")
252 if len(args
)==0 or args
[0] not in ['upload','remove']:
253 print("Either 'upload' or 'remove' needs to be specified "
257 if args
[0]=='upload':
259 elif args
[0]=='remove':
262 # we should not be getting here
267 upload_path_list
= [options
.repo
]
269 upload_path_list
= upload_paths
270 process_all_packages(pkgmgr
, client_dir
, upload_path_list
,
274 process_packages(pkgmgr
, 'client', 'autotest', client_dir
,
275 options
.repo
, remove
=remove_flag
)
278 process_packages(pkgmgr
, 'dep', options
.dep
, dep_dir
,
279 options
.repo
, remove
=remove_flag
)
282 process_packages(pkgmgr
, 'test', options
.test
, client_dir
,
283 options
.repo
, remove
=remove_flag
)
286 process_packages(pkgmgr
, 'profiler', options
.prof
, prof_dir
,
287 options
.repo
, remove
=remove_flag
)
291 pkgmgr
.remove_pkg(options
.file, options
.repo
, remove_checksum
=True)
293 pkgmgr
.upload_pkg(options
.file, options
.repo
, update_checksum
=True)
296 if __name__
== "__main__":