2 KVM test utility functions.
4 @copyright: 2008-2009 Red Hat Inc.
7 import time
, string
, random
, socket
, os
, signal
, re
, logging
, commands
, cPickle
8 import fcntl
, shelve
, ConfigParser
, threading
, sys
, UserDict
, inspect
10 from autotest_lib
.client
.bin
import utils
, os_dep
11 from autotest_lib
.client
.common_lib
import error
, logging_config
12 import rss_client
, aexpect
17 KOJI_INSTALLED
= False
19 # From include/linux/sockios.h
20 SIOCSIFHWADDR
= 0x8924
21 SIOCGIFHWADDR
= 0x8927
25 # From linux/include/linux/if_tun.h
26 TUNSETIFF
= 0x400454ca
27 TUNGETIFF
= 0x800454d2
28 TUNGETFEATURES
= 0x800454cf
34 def _lock_file(filename
):
35 f
= open(filename
, "w")
36 fcntl
.lockf(f
, fcntl
.LOCK_EX
)
41 fcntl
.lockf(f
, fcntl
.LOCK_UN
)
47 Tests whether a given object is a VM object.
49 @param obj: Python object.
51 return obj
.__class
__.__name
__ == "VM"
54 class NetError(Exception):
58 class TAPModuleError(NetError
):
59 def __init__(self
, devname
, action
="open", details
=None):
60 NetError
.__init
__(self
, devname
)
61 self
.devname
= devname
62 self
.details
= details
65 e_msg
= "Can't %s %s" % (self
.action
, self
.devname
)
66 if self
.details
is not None:
67 e_msg
+= " : %s" % self
.details
71 class TAPNotExistError(NetError
):
72 def __init__(self
, ifname
):
73 NetError
.__init
__(self
, ifname
)
77 return "Interface %s does not exist" % self
.ifname
80 class TAPCreationError(NetError
):
81 def __init__(self
, ifname
, details
=None):
82 NetError
.__init
__(self
, ifname
, details
)
84 self
.details
= details
87 e_msg
= "Cannot create TAP device %s" % self
.ifname
88 if self
.details
is not None:
89 e_msg
+= ": %s" % self
.details
93 class TAPBringUpError(NetError
):
94 def __init__(self
, ifname
):
95 NetError
.__init
__(self
, ifname
)
99 return "Cannot bring up TAP %s" % self
.ifname
102 class BRAddIfError(NetError
):
103 def __init__(self
, ifname
, brname
, details
):
104 NetError
.__init
__(self
, ifname
, brname
, details
)
107 self
.details
= details
110 return ("Can not add if %s to bridge %s: %s" %
111 (self
.ifname
, self
.brname
, self
.details
))
114 class HwAddrSetError(NetError
):
115 def __init__(self
, ifname
, mac
):
116 NetError
.__init
__(self
, ifname
, mac
)
121 return "Can not set mac %s to interface %s" % (self
.mac
, self
.ifname
)
124 class HwAddrGetError(NetError
):
125 def __init__(self
, ifname
):
126 NetError
.__init
__(self
, ifname
)
130 return "Can not get mac of interface %s" % self
.ifname
133 class Env(UserDict
.IterableUserDict
):
135 A dict-like object containing global objects used by tests.
137 def __init__(self
, filename
=None, version
=0):
139 Create an empty Env object or load an existing one from a file.
141 If the version recorded in the file is lower than version, or if some
142 error occurs during unpickling, or if filename is not supplied,
143 create an empty Env object.
145 @param filename: Path to an env file.
146 @param version: Required env version (int).
148 UserDict
.IterableUserDict
.__init
__(self
)
149 empty
= {"version": version
}
151 self
._filename
= filename
153 f
= open(filename
, "r")
154 env
= cPickle
.load(f
)
156 if env
.get("version", 0) >= version
:
159 logging
.warn("Incompatible env file found. Not using it.")
161 # Almost any exception can be raised during unpickling, so let's
170 def save(self
, filename
=None):
172 Pickle the contents of the Env object into a file.
174 @param filename: Filename to pickle the dict into. If not supplied,
175 use the filename from which the dict was loaded.
177 filename
= filename
or self
._filename
178 f
= open(filename
, "w")
179 cPickle
.dump(self
.data
, f
)
183 def get_all_vms(self
):
185 Return a list of all VM objects in this Env object.
187 return [o
for o
in self
.values() if is_vm(o
)]
190 def get_vm(self
, name
):
192 Return a VM object by its name.
194 @param name: VM name.
196 return self
.get("vm__%s" % name
)
199 def register_vm(self
, name
, vm
):
201 Register a VM in this Env object.
203 @param name: VM name.
204 @param vm: VM object.
206 self
["vm__%s" % name
] = vm
209 def unregister_vm(self
, name
):
213 @param name: VM name.
215 del self
["vm__%s" % name
]
218 def register_installer(self
, installer
):
220 Register a installer that was just run
222 The installer will be available for other tests, so that
223 information about the installed KVM modules and qemu-kvm can be used by
226 self
['last_installer'] = installer
229 def previous_installer(self
):
231 Return the last installer that was registered
233 return self
.get('last_installer')
236 class Params(UserDict
.IterableUserDict
):
238 A dict-like object passed to every test.
240 def objects(self
, key
):
242 Return the names of objects defined using a given key.
244 @param key: The name of the key whose value lists the objects
247 return self
.get(key
, "").split()
250 def object_params(self
, obj_name
):
252 Return a dict-like object containing the parameters of an individual
255 This method behaves as follows: the suffix '_' + obj_name is removed
256 from all key names that have it. Other key names are left unchanged.
257 The values of keys with the suffix overwrite the values of their
260 @param obj_name: The name of the object (objects are listed by the
263 suffix
= "_" + obj_name
264 new_dict
= self
.copy()
266 if key
.endswith(suffix
):
267 new_key
= key
.split(suffix
)[0]
268 new_dict
[new_key
] = self
[key
]
272 # Functions related to MAC/IP addresses
274 def _open_mac_pool(lock_mode
):
275 lock_file
= open("/tmp/mac_lock", "w+")
276 fcntl
.lockf(lock_file
, lock_mode
)
277 pool
= shelve
.open("/tmp/address_pool")
278 return pool
, lock_file
281 def _close_mac_pool(pool
, lock_file
):
283 fcntl
.lockf(lock_file
, fcntl
.LOCK_UN
)
287 def _generate_mac_address_prefix(mac_pool
):
289 Generate a random MAC address prefix and add it to the MAC pool dictionary.
290 If there's a MAC prefix there already, do not update the MAC pool and just
291 return what's in there. By convention we will set KVM autotest MAC
292 addresses to start with 0x9a.
294 @param mac_pool: The MAC address pool object.
295 @return: The MAC address prefix.
297 if "prefix" in mac_pool
:
298 prefix
= mac_pool
["prefix"]
299 logging
.debug("Used previously generated MAC address prefix for this "
302 r
= random
.SystemRandom()
303 prefix
= "9a:%02x:%02x:%02x:" % (r
.randint(0x00, 0xff),
304 r
.randint(0x00, 0xff),
305 r
.randint(0x00, 0xff))
306 mac_pool
["prefix"] = prefix
307 logging
.debug("Generated MAC address prefix for this host: %s", prefix
)
311 def generate_mac_address(vm_instance
, nic_index
):
313 Randomly generate a MAC address and add it to the MAC address pool.
315 Try to generate a MAC address based on a randomly generated MAC address
316 prefix and add it to a persistent dictionary.
317 key = VM instance + NIC index, value = MAC address
318 e.g. {'20100310-165222-Wt7l:0': '9a:5d:94:6a:9b:f9'}
320 @param vm_instance: The instance attribute of a VM.
321 @param nic_index: The index of the NIC.
322 @return: MAC address string.
324 mac_pool
, lock_file
= _open_mac_pool(fcntl
.LOCK_EX
)
325 key
= "%s:%s" % (vm_instance
, nic_index
)
329 prefix
= _generate_mac_address_prefix(mac_pool
)
330 r
= random
.SystemRandom()
331 while key
not in mac_pool
:
332 mac
= prefix
+ "%02x:%02x" % (r
.randint(0x00, 0xff),
333 r
.randint(0x00, 0xff))
334 if mac
in mac_pool
.values():
337 logging
.debug("Generated MAC address for NIC %s: %s", key
, mac
)
338 _close_mac_pool(mac_pool
, lock_file
)
342 def free_mac_address(vm_instance
, nic_index
):
344 Remove a MAC address from the address pool.
346 @param vm_instance: The instance attribute of a VM.
347 @param nic_index: The index of the NIC.
349 mac_pool
, lock_file
= _open_mac_pool(fcntl
.LOCK_EX
)
350 key
= "%s:%s" % (vm_instance
, nic_index
)
352 logging
.debug("Freeing MAC address for NIC %s: %s", key
, mac_pool
[key
])
354 _close_mac_pool(mac_pool
, lock_file
)
357 def set_mac_address(vm_instance
, nic_index
, mac
):
359 Set a MAC address in the pool.
361 @param vm_instance: The instance attribute of a VM.
362 @param nic_index: The index of the NIC.
364 mac_pool
, lock_file
= _open_mac_pool(fcntl
.LOCK_EX
)
365 mac_pool
["%s:%s" % (vm_instance
, nic_index
)] = mac
366 _close_mac_pool(mac_pool
, lock_file
)
369 def get_mac_address(vm_instance
, nic_index
):
371 Return a MAC address from the pool.
373 @param vm_instance: The instance attribute of a VM.
374 @param nic_index: The index of the NIC.
375 @return: MAC address string.
377 mac_pool
, lock_file
= _open_mac_pool(fcntl
.LOCK_SH
)
378 mac
= mac_pool
.get("%s:%s" % (vm_instance
, nic_index
))
379 _close_mac_pool(mac_pool
, lock_file
)
383 def verify_ip_address_ownership(ip
, macs
, timeout
=10.0):
385 Use arping and the ARP cache to make sure a given IP address belongs to one
386 of the given MAC addresses.
388 @param ip: An IP address.
389 @param macs: A list or tuple of MAC addresses.
390 @return: True iff ip is assigned to a MAC address in macs.
392 # Compile a regex that matches the given IP address and any of the given
394 mac_regex
= "|".join("(%s)" % mac
for mac
in macs
)
395 regex
= re
.compile(r
"\b%s\b.*\b(%s)\b" % (ip
, mac_regex
), re
.IGNORECASE
)
397 # Check the ARP cache
398 o
= commands
.getoutput("%s -n" % find_command("arp"))
402 # Get the name of the bridge device for arping
403 o
= commands
.getoutput("%s route get %s" % (find_command("ip"), ip
))
404 dev
= re
.findall("dev\s+\S+", o
, re
.IGNORECASE
)
407 dev
= dev
[0].split()[-1]
409 # Send an ARP request
410 o
= commands
.getoutput("%s -f -c 3 -I %s %s" %
411 (find_command("arping"), dev
, ip
))
412 return bool(regex
.search(o
))
415 # Utility functions for dealing with external processes
417 def find_command(cmd
):
418 for dir in ["/usr/local/sbin", "/usr/local/bin",
419 "/usr/sbin", "/usr/bin", "/sbin", "/bin"]:
420 file = os
.path
.join(dir, cmd
)
421 if os
.path
.exists(file):
423 raise ValueError('Missing command: %s' % cmd
)
428 Return True if a given PID exists.
430 @param pid: Process ID number.
439 def safe_kill(pid
, signal
):
441 Attempt to send a signal to a given process that may or may not exist.
443 @param signal: Signal number.
452 def kill_process_tree(pid
, sig
=signal
.SIGKILL
):
453 """Signal a process and all of its children.
455 If the process does not exist -- return.
457 @param pid: The pid of the process to signal.
458 @param sig: The signal to send to the processes.
460 if not safe_kill(pid
, signal
.SIGSTOP
):
462 children
= commands
.getoutput("ps --ppid=%d -o pid=" % pid
).split()
463 for child
in children
:
464 kill_process_tree(int(child
), sig
)
466 safe_kill(pid
, signal
.SIGCONT
)
469 def get_latest_kvm_release_tag(release_listing
):
471 Fetches the latest release tag for KVM.
473 @param release_listing: URL that contains a list of the Source Forge
477 release_page
= utils
.urlopen(release_listing
)
478 data
= release_page
.read()
480 rx
= re
.compile("kvm-(\d+).tar.gz", re
.IGNORECASE
)
481 matches
= rx
.findall(data
)
482 # In all regexp matches to something that looks like a release tag,
483 # get the largest integer. That will be our latest release tag.
484 latest_tag
= max(int(x
) for x
in matches
)
485 return str(latest_tag
)
487 message
= "Could not fetch latest KVM release tag: %s" % str(e
)
488 logging
.error(message
)
489 raise error
.TestError(message
)
492 def get_git_branch(repository
, branch
, srcdir
, commit
=None, lbranch
=None):
494 Retrieves a given git code repository.
496 @param repository: Git repository URL
498 logging
.info("Fetching git [REP '%s' BRANCH '%s' COMMIT '%s'] -> %s",
499 repository
, branch
, commit
, srcdir
)
500 if not os
.path
.exists(srcdir
):
504 if os
.path
.exists(".git"):
505 utils
.system("git reset --hard")
507 utils
.system("git init")
512 utils
.system("git fetch -q -f -u -t %s %s:%s" %
513 (repository
, branch
, lbranch
))
514 utils
.system("git checkout %s" % lbranch
)
516 utils
.system("git checkout %s" % commit
)
518 h
= utils
.system_output('git log --pretty=format:"%H" -1')
520 desc
= "tag %s" % utils
.system_output("git describe")
521 except error
.CmdError
:
522 desc
= "no tag found"
524 logging
.info("Commit hash for %s is %s (%s)", repository
, h
.strip(), desc
)
528 def check_kvm_source_dir(source_dir
):
530 Inspects the kvm source directory and verifies its disposition. In some
531 occasions build may be dependant on the source directory disposition.
532 The reason why the return codes are numbers is that we might have more
533 changes on the source directory layout, so it's not scalable to just use
534 strings like 'old_repo', 'new_repo' and such.
536 @param source_dir: Source code path that will be inspected.
539 has_qemu_dir
= os
.path
.isdir('qemu')
540 has_kvm_dir
= os
.path
.isdir('kvm')
542 logging
.debug("qemu directory detected, source dir layout 1")
544 if has_kvm_dir
and not has_qemu_dir
:
545 logging
.debug("kvm directory detected, source dir layout 2")
548 raise error
.TestError("Unknown source dir layout, cannot proceed.")
551 # Functions and classes used for logging into guests and transferring files
553 class LoginError(Exception):
554 def __init__(self
, msg
, output
):
555 Exception.__init
__(self
, msg
, output
)
560 return "%s (output: %r)" % (self
.msg
, self
.output
)
563 class LoginAuthenticationError(LoginError
):
567 class LoginTimeoutError(LoginError
):
568 def __init__(self
, output
):
569 LoginError
.__init
__(self
, "Login timeout expired", output
)
572 class LoginProcessTerminatedError(LoginError
):
573 def __init__(self
, status
, output
):
574 LoginError
.__init
__(self
, None, output
)
578 return ("Client process terminated (status: %s, output: %r)" %
579 (self
.status
, self
.output
))
582 class LoginBadClientError(LoginError
):
583 def __init__(self
, client
):
584 LoginError
.__init
__(self
, None, None)
588 return "Unknown remote shell client: %r" % self
.client
591 class SCPError(Exception):
592 def __init__(self
, msg
, output
):
593 Exception.__init
__(self
, msg
, output
)
598 return "%s (output: %r)" % (self
.msg
, self
.output
)
601 class SCPAuthenticationError(SCPError
):
605 class SCPAuthenticationTimeoutError(SCPAuthenticationError
):
606 def __init__(self
, output
):
607 SCPAuthenticationError
.__init
__(self
, "Authentication timeout expired",
611 class SCPTransferTimeoutError(SCPError
):
612 def __init__(self
, output
):
613 SCPError
.__init
__(self
, "Transfer timeout expired", output
)
616 class SCPTransferFailedError(SCPError
):
617 def __init__(self
, status
, output
):
618 SCPError
.__init
__(self
, None, output
)
622 return ("SCP transfer failed (status: %s, output: %r)" %
623 (self
.status
, self
.output
))
626 def _remote_login(session
, username
, password
, prompt
, timeout
=10):
628 Log into a remote host (guest) using SSH or Telnet. Wait for questions
629 and provide answers. If timeout expires while waiting for output from the
630 child (e.g. a password prompt or a shell prompt) -- fail.
632 @brief: Log into a remote host (guest) using SSH or Telnet.
634 @param session: An Expect or ShellSession instance to operate on
635 @param username: The username to send in reply to a login prompt
636 @param password: The password to send in reply to a password prompt
637 @param prompt: The shell prompt that indicates a successful login
638 @param timeout: The maximal time duration (in seconds) to wait for each
639 step of the login procedure (i.e. the "Are you sure" prompt, the
640 password prompt, the shell prompt, etc)
641 @raise LoginTimeoutError: If timeout expires
642 @raise LoginAuthenticationError: If authentication fails
643 @raise LoginProcessTerminatedError: If the client terminates during login
644 @raise LoginError: If some other error occurs
646 password_prompt_count
= 0
647 login_prompt_count
= 0
651 match
, text
= session
.read_until_last_line_matches(
652 [r
"[Aa]re you sure", r
"[Pp]assword:\s*$", r
"[Ll]ogin:\s*$",
653 r
"[Cc]onnection.*closed", r
"[Cc]onnection.*refused",
654 r
"[Pp]lease wait", r
"[Ww]arning", prompt
],
655 timeout
=timeout
, internal_timeout
=0.5)
656 if match
== 0: # "Are you sure you want to continue connecting"
657 logging
.debug("Got 'Are you sure...'; sending 'yes'")
658 session
.sendline("yes")
660 elif match
== 1: # "password:"
661 if password_prompt_count
== 0:
662 logging
.debug("Got password prompt; sending '%s'", password
)
663 session
.sendline(password
)
664 password_prompt_count
+= 1
667 raise LoginAuthenticationError("Got password prompt twice",
669 elif match
== 2: # "login:"
670 if login_prompt_count
== 0 and password_prompt_count
== 0:
671 logging
.debug("Got username prompt; sending '%s'", username
)
672 session
.sendline(username
)
673 login_prompt_count
+= 1
676 if login_prompt_count
> 0:
677 msg
= "Got username prompt twice"
679 msg
= "Got username prompt after password prompt"
680 raise LoginAuthenticationError(msg
, text
)
681 elif match
== 3: # "Connection closed"
682 raise LoginError("Client said 'connection closed'", text
)
683 elif match
== 4: # "Connection refused"
684 raise LoginError("Client said 'connection refused'", text
)
685 elif match
== 5: # "Please wait"
686 logging
.debug("Got 'Please wait'")
689 elif match
== 6: # "Warning added RSA"
690 logging
.debug("Got 'Warning added RSA to known host list")
692 elif match
== 7: # prompt
693 logging
.debug("Got shell prompt -- logged in")
695 except aexpect
.ExpectTimeoutError
, e
:
696 raise LoginTimeoutError(e
.output
)
697 except aexpect
.ExpectProcessTerminatedError
, e
:
698 raise LoginProcessTerminatedError(e
.status
, e
.output
)
701 def remote_login(client
, host
, port
, username
, password
, prompt
, linesep
="\n",
702 log_filename
=None, timeout
=10):
704 Log into a remote host (guest) using SSH/Telnet/Netcat.
706 @param client: The client to use ('ssh', 'telnet' or 'nc')
707 @param host: Hostname or IP address
708 @param port: Port to connect to
709 @param username: Username (if required)
710 @param password: Password (if required)
711 @param prompt: Shell prompt (regular expression)
712 @param linesep: The line separator to use when sending lines
713 (e.g. '\\n' or '\\r\\n')
714 @param log_filename: If specified, log all output to this file
715 @param timeout: The maximal time duration (in seconds) to wait for
716 each step of the login procedure (i.e. the "Are you sure" prompt
717 or the password prompt)
718 @raise LoginBadClientError: If an unknown client is requested
719 @raise: Whatever _remote_login() raises
720 @return: A ShellSession object.
723 cmd
= ("ssh -o UserKnownHostsFile=/dev/null "
724 "-o PreferredAuthentications=password -p %s %s@%s" %
725 (port
, username
, host
))
726 elif client
== "telnet":
727 cmd
= "telnet -l %s %s %s" % (username
, host
, port
)
729 cmd
= "nc %s %s" % (host
, port
)
731 raise LoginBadClientError(client
)
733 logging
.debug("Trying to login with command '%s'", cmd
)
734 session
= aexpect
.ShellSession(cmd
, linesep
=linesep
, prompt
=prompt
)
736 _remote_login(session
, username
, password
, prompt
, timeout
)
741 session
.set_output_func(log_line
)
742 session
.set_output_params((log_filename
,))
746 def wait_for_login(client
, host
, port
, username
, password
, prompt
, linesep
="\n",
747 log_filename
=None, timeout
=240, internal_timeout
=10):
749 Make multiple attempts to log into a remote host (guest) until one succeeds
752 @param timeout: Total time duration to wait for a successful login
753 @param internal_timeout: The maximal time duration (in seconds) to wait for
754 each step of the login procedure (e.g. the "Are you sure" prompt
755 or the password prompt)
757 @raise: Whatever remote_login() raises
758 @return: A ShellSession object.
760 logging
.debug("Attempting to log into %s:%s using %s (timeout %ds)",
761 host
, port
, client
, timeout
)
762 end_time
= time
.time() + timeout
763 while time
.time() < end_time
:
765 return remote_login(client
, host
, port
, username
, password
, prompt
,
766 linesep
, log_filename
, internal_timeout
)
767 except LoginError
, e
:
770 # Timeout expired; try one more time but don't catch exceptions
771 return remote_login(client
, host
, port
, username
, password
, prompt
,
772 linesep
, log_filename
, internal_timeout
)
775 def _remote_scp(session
, password_list
, transfer_timeout
=600, login_timeout
=10):
777 Transfer file(s) to a remote host (guest) using SCP. Wait for questions
778 and provide answers. If login_timeout expires while waiting for output
779 from the child (e.g. a password prompt), fail. If transfer_timeout expires
780 while waiting for the transfer to complete, fail.
782 @brief: Transfer files using SCP, given a command line.
784 @param session: An Expect or ShellSession instance to operate on
785 @param password_list: Password list to send in reply to the password prompt
786 @param transfer_timeout: The time duration (in seconds) to wait for the
787 transfer to complete.
788 @param login_timeout: The maximal time duration (in seconds) to wait for
789 each step of the login procedure (i.e. the "Are you sure" prompt or
791 @raise SCPAuthenticationError: If authentication fails
792 @raise SCPTransferTimeoutError: If the transfer fails to complete in time
793 @raise SCPTransferFailedError: If the process terminates with a nonzero
795 @raise SCPError: If some other error occurs
797 password_prompt_count
= 0
798 timeout
= login_timeout
799 authentication_done
= False
801 scp_type
= len(password_list
)
805 match
, text
= session
.read_until_last_line_matches(
806 [r
"[Aa]re you sure", r
"[Pp]assword:\s*$", r
"lost connection"],
807 timeout
=timeout
, internal_timeout
=0.5)
808 if match
== 0: # "Are you sure you want to continue connecting"
809 logging
.debug("Got 'Are you sure...'; sending 'yes'")
810 session
.sendline("yes")
812 elif match
== 1: # "password:"
813 if password_prompt_count
== 0:
814 logging
.debug("Got password prompt; sending '%s'" %
815 password_list
[password_prompt_count
])
816 session
.sendline(password_list
[password_prompt_count
])
817 password_prompt_count
+= 1
818 timeout
= transfer_timeout
820 authentication_done
= True
822 elif password_prompt_count
== 1 and scp_type
== 2:
823 logging
.debug("Got password prompt; sending '%s'" %
824 password_list
[password_prompt_count
])
825 session
.sendline(password_list
[password_prompt_count
])
826 password_prompt_count
+= 1
827 timeout
= transfer_timeout
828 authentication_done
= True
831 raise SCPAuthenticationError("Got password prompt twice",
833 elif match
== 2: # "lost connection"
834 raise SCPError("SCP client said 'lost connection'", text
)
835 except aexpect
.ExpectTimeoutError
, e
:
836 if authentication_done
:
837 raise SCPTransferTimeoutError(e
.output
)
839 raise SCPAuthenticationTimeoutError(e
.output
)
840 except aexpect
.ExpectProcessTerminatedError
, e
:
842 logging
.debug("SCP process terminated with status 0")
845 raise SCPTransferFailedError(e
.status
, e
.output
)
848 def remote_scp(command
, password_list
, log_filename
=None, transfer_timeout
=600,
851 Transfer file(s) to a remote host (guest) using SCP.
853 @brief: Transfer files using SCP, given a command line.
855 @param command: The command to execute
856 (e.g. "scp -r foobar root@localhost:/tmp/").
857 @param password_list: Password list to send in reply to a password prompt.
858 @param log_filename: If specified, log all output to this file
859 @param transfer_timeout: The time duration (in seconds) to wait for the
860 transfer to complete.
861 @param login_timeout: The maximal time duration (in seconds) to wait for
862 each step of the login procedure (i.e. the "Are you sure" prompt
863 or the password prompt)
864 @raise: Whatever _remote_scp() raises
866 logging
.debug("Trying to SCP with command '%s', timeout %ss",
867 command
, transfer_timeout
)
869 output_func
= log_line
870 output_params
= (log_filename
,)
874 session
= aexpect
.Expect(command
,
875 output_func
=output_func
,
876 output_params
=output_params
)
878 _remote_scp(session
, password_list
, transfer_timeout
, login_timeout
)
883 def scp_to_remote(host
, port
, username
, password
, local_path
, remote_path
,
884 log_filename
=None, timeout
=600):
886 Copy files to a remote host (guest) through scp.
888 @param host: Hostname or IP address
889 @param username: Username (if required)
890 @param password: Password (if required)
891 @param local_path: Path on the local machine where we are copying from
892 @param remote_path: Path on the remote machine where we are copying to
893 @param log_filename: If specified, log all output to this file
894 @param timeout: The time duration (in seconds) to wait for the transfer
896 @raise: Whatever remote_scp() raises
898 command
= ("scp -v -o UserKnownHostsFile=/dev/null "
899 "-o PreferredAuthentications=password -r -P %s %s %s@%s:%s" %
900 (port
, local_path
, username
, host
, remote_path
))
902 password_list
.append(password
)
903 return remote_scp(command
, password_list
, log_filename
, timeout
)
907 def scp_from_remote(host
, port
, username
, password
, remote_path
, local_path
,
908 log_filename
=None, timeout
=600):
910 Copy files from a remote host (guest).
912 @param host: Hostname or IP address
913 @param username: Username (if required)
914 @param password: Password (if required)
915 @param local_path: Path on the local machine where we are copying from
916 @param remote_path: Path on the remote machine where we are copying to
917 @param log_filename: If specified, log all output to this file
918 @param timeout: The time duration (in seconds) to wait for the transfer
920 @raise: Whatever remote_scp() raises
922 command
= ("scp -v -o UserKnownHostsFile=/dev/null "
923 "-o PreferredAuthentications=password -r -P %s %s@%s:%s %s" %
924 (port
, username
, host
, remote_path
, local_path
))
926 password_list
.append(password
)
927 remote_scp(command
, password_list
, log_filename
, timeout
)
930 def scp_between_remotes(src
, dst
, port
, s_passwd
, d_passwd
, s_name
, d_name
,
931 s_path
, d_path
, log_filename
=None, timeout
=600):
933 Copy files from a remote host (guest) to another remote host (guest).
935 @param src/dst: Hostname or IP address of src and dst
936 @param s_name/d_name: Username (if required)
937 @param s_passwd/d_passwd: Password (if required)
938 @param s_path/d_path: Path on the remote machine where we are copying
940 @param log_filename: If specified, log all output to this file
941 @param timeout: The time duration (in seconds) to wait for the transfer
944 @return: True on success and False on failure.
946 command
= ("scp -v -o UserKnownHostsFile=/dev/null -o "
947 "PreferredAuthentications=password -r -P %s %s@%s:%s %s@%s:%s" %
948 (port
, s_name
, src
, s_path
, d_name
, dst
, d_path
))
950 password_list
.append(s_passwd
)
951 password_list
.append(d_passwd
)
952 return remote_scp(command
, password_list
, log_filename
, timeout
)
955 def copy_files_to(address
, client
, username
, password
, port
, local_path
,
956 remote_path
, log_filename
=None, verbose
=False, timeout
=600):
958 Copy files to a remote host (guest) using the selected client.
960 @param client: Type of transfer client
961 @param username: Username (if required)
962 @param password: Password (if requried)
963 @param local_path: Path on the local machine where we are copying from
964 @param remote_path: Path on the remote machine where we are copying to
965 @param address: Address of remote host(guest)
966 @param log_filename: If specified, log all output to this file (SCP only)
967 @param verbose: If True, log some stats using logging.debug (RSS only)
968 @param timeout: The time duration (in seconds) to wait for the transfer to
970 @raise: Whatever remote_scp() raises
973 scp_to_remote(address
, port
, username
, password
, local_path
,
974 remote_path
, log_filename
, timeout
)
975 elif client
== "rss":
978 log_func
= logging
.debug
979 c
= rss_client
.FileUploadClient(address
, port
, log_func
)
980 c
.upload(local_path
, remote_path
, timeout
)
984 def copy_files_from(address
, client
, username
, password
, port
, remote_path
,
985 local_path
, log_filename
=None, verbose
=False, timeout
=600):
987 Copy files from a remote host (guest) using the selected client.
989 @param client: Type of transfer client
990 @param username: Username (if required)
991 @param password: Password (if requried)
992 @param remote_path: Path on the remote machine where we are copying from
993 @param local_path: Path on the local machine where we are copying to
994 @param address: Address of remote host(guest)
995 @param log_filename: If specified, log all output to this file (SCP only)
996 @param verbose: If True, log some stats using logging.debug (RSS only)
997 @param timeout: The time duration (in seconds) to wait for the transfer to
999 @raise: Whatever remote_scp() raises
1002 scp_from_remote(address
, port
, username
, password
, remote_path
,
1003 local_path
, log_filename
, timeout
)
1004 elif client
== "rss":
1007 log_func
= logging
.debug
1008 c
= rss_client
.FileDownloadClient(address
, port
, log_func
)
1009 c
.download(remote_path
, local_path
, timeout
)
1013 # The following are utility functions related to ports.
1015 def is_port_free(port
, address
):
1017 Return True if the given port is available for use.
1019 @param port: Port number
1023 #s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
1024 if address
== "localhost":
1025 s
.bind(("localhost", port
))
1028 s
.connect((address
, port
))
1030 except socket
.error
:
1031 if address
== "localhost":
1039 def find_free_port(start_port
, end_port
, address
="localhost"):
1041 Return a host free port in the range [start_port, end_port].
1043 @param start_port: First port that will be checked.
1044 @param end_port: Port immediately after the last one that will be checked.
1046 for i
in range(start_port
, end_port
):
1047 if is_port_free(i
, address
):
1052 def find_free_ports(start_port
, end_port
, count
, address
="localhost"):
1054 Return count of host free ports in the range [start_port, end_port].
1056 @count: Initial number of ports known to be free in the range.
1057 @param start_port: First port that will be checked.
1058 @param end_port: Port immediately after the last one that will be checked.
1062 while i
< end_port
and count
> 0:
1063 if is_port_free(i
, address
):
1070 # An easy way to log lines to files when the logging system can't be used
1072 _open_log_files
= {}
1073 _log_file_dir
= "/tmp"
1076 def log_line(filename
, line
):
1078 Write a line to a file. '\n' is appended to the line.
1080 @param filename: Path of file to write to, either absolute or relative to
1081 the dir set by set_log_file_dir().
1082 @param line: Line to write.
1084 global _open_log_files
, _log_file_dir
1085 if filename
not in _open_log_files
:
1086 path
= get_path(_log_file_dir
, filename
)
1088 os
.makedirs(os
.path
.dirname(path
))
1091 _open_log_files
[filename
] = open(path
, "w")
1092 timestr
= time
.strftime("%Y-%m-%d %H:%M:%S")
1093 _open_log_files
[filename
].write("%s: %s\n" % (timestr
, line
))
1094 _open_log_files
[filename
].flush()
1097 def set_log_file_dir(dir):
1099 Set the base directory for log files created by log_line().
1101 @param dir: Directory for log files.
1103 global _log_file_dir
1107 # The following are miscellaneous utility functions.
1109 def get_path(base_path
, user_path
):
1111 Translate a user specified path to a real path.
1112 If user_path is relative, append it to base_path.
1113 If user_path is absolute, return it as is.
1115 @param base_path: The base path of relative user specified paths.
1116 @param user_path: The user specified path.
1118 if os
.path
.isabs(user_path
):
1121 return os
.path
.join(base_path
, user_path
)
1124 def generate_random_string(length
):
1126 Return a random string using alphanumeric characters.
1128 @length: length of the string that will be generated.
1130 r
= random
.SystemRandom()
1132 chars
= string
.letters
+ string
.digits
1134 str += r
.choice(chars
)
1138 def generate_random_id():
1140 Return a random string suitable for use as a qemu id.
1142 return "id" + generate_random_string(6)
1145 def generate_tmp_file_name(file, ext
=None, dir='/tmp/'):
1147 Returns a temporary file name. The file is not created.
1150 file_name
= (file + '-' + time
.strftime("%Y%m%d-%H%M%S-") +
1151 generate_random_string(4))
1153 file_name
+= '.' + ext
1154 file_name
= os
.path
.join(dir, file_name
)
1155 if not os
.path
.exists(file_name
):
1161 def format_str_for_message(str):
1163 Format str so that it can be appended to a message.
1164 If str consists of one line, prefix it with a space.
1165 If str consists of multiple lines, prefix it with a newline.
1167 @param str: string that will be formatted.
1169 lines
= str.splitlines()
1170 num_lines
= len(lines
)
1171 str = "\n".join(lines
)
1174 elif num_lines
== 1:
1180 def wait_for(func
, timeout
, first
=0.0, step
=1.0, text
=None):
1182 If func() evaluates to True before timeout expires, return the
1183 value of func(). Otherwise return None.
1185 @brief: Wait until func() evaluates to True.
1187 @param timeout: Timeout in seconds
1188 @param first: Time to sleep before first attempt
1189 @param steps: Time to sleep between attempts in seconds
1190 @param text: Text to print while waiting, for debug purposes
1192 start_time
= time
.time()
1193 end_time
= time
.time() + timeout
1197 while time
.time() < end_time
:
1199 logging
.debug("%s (%f secs)", text
, (time
.time() - start_time
))
1207 logging
.debug("Timeout elapsed")
1211 def get_hash_from_file(hash_path
, dvd_basename
):
1213 Get the a hash from a given DVD image from a hash file
1214 (Hash files are usually named MD5SUM or SHA1SUM and are located inside the
1215 download directories of the DVDs)
1217 @param hash_path: Local path to a hash file.
1218 @param cd_image: Basename of a CD image
1220 hash_file
= open(hash_path
, 'r')
1221 for line
in hash_file
.readlines():
1222 if dvd_basename
in line
:
1223 return line
.split()[0]
1226 def run_tests(parser
, job
):
1228 Runs the sequence of KVM tests based on the list of dictionaries
1229 generated by the configuration system, handling dependencies.
1231 @param parser: Config parser object.
1232 @param job: Autotest job object.
1234 @return: True, if all tests ran passed, False if any of them failed.
1236 for i
, d
in enumerate(parser
.get_dicts()):
1237 logging
.info("Test %4d: %s" % (i
+ 1, d
["shortname"]))
1242 for dict in parser
.get_dicts():
1243 if dict.get("skip") == "yes":
1245 dependencies_satisfied
= True
1246 for dep
in dict.get("dep"):
1247 for test_name
in status_dict
.keys():
1248 if not dep
in test_name
:
1250 # So the only really non-fatal state is WARN,
1251 # All the others make it not safe to proceed with dependency
1253 if status_dict
[test_name
] not in ['GOOD', 'WARN']:
1254 dependencies_satisfied
= False
1256 test_iterations
= int(dict.get("iterations", 1))
1257 test_tag
= dict.get("shortname")
1259 if dependencies_satisfied
:
1260 # Setting up profilers during test execution.
1261 profilers
= dict.get("profilers", "").split()
1262 for profiler
in profilers
:
1263 job
.profilers
.add(profiler
)
1264 # We need only one execution, profiled, hence we're passing
1265 # the profile_only parameter to job.run_test().
1266 profile_only
= bool(profilers
) or None
1267 current_status
= job
.run_test_detail(dict.get("vm_type"),
1270 iterations
=test_iterations
,
1271 profile_only
=profile_only
)
1272 for profiler
in profilers
:
1273 job
.profilers
.delete(profiler
)
1275 # We will force the test to fail as TestNA during preprocessing
1276 dict['dependency_failed'] = 'yes'
1277 current_status
= job
.run_test_detail(dict.get("vm_type"),
1280 iterations
=test_iterations
)
1282 if not current_status
:
1284 status_dict
[dict.get("name")] = current_status
1289 def display_attributes(instance
):
1291 Inspects a given class instance attributes and displays them, convenient
1294 logging
.debug("Attributes set:")
1295 for member
in inspect
.getmembers(instance
):
1296 name
, value
= member
1297 attribute
= getattr(instance
, name
)
1298 if not (name
.startswith("__") or callable(attribute
) or not value
):
1299 logging
.debug(" %s: %s", name
, value
)
1302 def get_full_pci_id(pci_id
):
1304 Get full PCI ID of pci_id.
1306 @param pci_id: PCI ID of a device.
1308 cmd
= "lspci -D | awk '/%s/ {print $1}'" % pci_id
1309 status
, full_id
= commands
.getstatusoutput(cmd
)
1315 def get_vendor_from_pci_id(pci_id
):
1317 Check out the device vendor ID according to pci_id.
1319 @param pci_id: PCI ID of a device.
1321 cmd
= "lspci -n | awk '/%s/ {print $3}'" % pci_id
1322 return re
.sub(":", " ", commands
.getoutput(cmd
))
1325 class Thread(threading
.Thread
):
1327 Run a function in a background thread.
1329 def __init__(self
, target
, args
=(), kwargs
={}):
1331 Initialize the instance.
1333 @param target: Function to run in the thread.
1334 @param args: Arguments to pass to target.
1335 @param kwargs: Keyword arguments to pass to target.
1337 threading
.Thread
.__init
__(self
)
1338 self
._target
= target
1340 self
._kwargs
= kwargs
1345 Run target (passed to the constructor). No point in calling this
1346 function directly. Call start() to make this function run in a new
1353 self
._retval
= self
._target
(*self
._args
, **self
._kwargs
)
1355 self
._e
= sys
.exc_info()
1358 # Avoid circular references (start() may be called only once so
1359 # it's OK to delete these)
1360 del self
._target
, self
._args
, self
._kwargs
1363 def join(self
, timeout
=None, suppress_exception
=False):
1365 Join the thread. If target raised an exception, re-raise it.
1366 Otherwise, return the value returned by target.
1368 @param timeout: Timeout value to pass to threading.Thread.join().
1369 @param suppress_exception: If True, don't re-raise the exception.
1371 threading
.Thread
.join(self
, timeout
)
1374 if not suppress_exception
:
1375 # Because the exception was raised in another thread, we
1376 # need to explicitly insert the current context into it
1377 s
= error
.exception_context(self
._e
[1])
1378 s
= error
.join_contexts(error
.get_context(), s
)
1379 error
.set_exception_context(self
._e
[1], s
)
1380 raise self
._e
[0], self
._e
[1], self
._e
[2]
1384 # Avoid circular references (join() may be called multiple times
1385 # so we can't delete these)
1390 def parallel(targets
):
1392 Run multiple functions in parallel.
1394 @param targets: A sequence of tuples or functions. If it's a sequence of
1395 tuples, each tuple will be interpreted as (target, args, kwargs) or
1396 (target, args) or (target,) depending on its length. If it's a
1397 sequence of functions, the functions will be called without
1399 @return: A list of the values returned by the functions called.
1402 for target
in targets
:
1403 if isinstance(target
, tuple) or isinstance(target
, list):
1409 return [t
.join() for t
in threads
]
1412 class VirtLoggingConfig(logging_config
.LoggingConfig
):
1414 Used with the sole purpose of providing convenient logging setup
1415 for the KVM test auxiliary programs.
1417 def configure_logging(self
, results_dir
=None, verbose
=False):
1418 super(VirtLoggingConfig
, self
).configure_logging(use_console
=True,
1422 class PciAssignable(object):
1424 Request PCI assignable devices on host. It will check whether to request
1425 PF (physical Functions) or VF (Virtual Functions).
1427 def __init__(self
, type="vf", driver
=None, driver_option
=None,
1428 names
=None, devices_requested
=None):
1430 Initialize parameter 'type' which could be:
1431 vf: Virtual Functions
1432 pf: Physical Function (actual hardware)
1433 mixed: Both includes VFs and PFs
1435 If pass through Physical NIC cards, we need to specify which devices
1436 to be assigned, e.g. 'eth1 eth2'.
1438 If pass through Virtual Functions, we need to specify how many vfs
1439 are going to be assigned, e.g. passthrough_count = 8 and max_vfs in
1442 @param type: PCI device type.
1443 @param driver: Kernel module for the PCI assignable device.
1444 @param driver_option: Module option to specify the maximum number of
1445 VFs (eg 'max_vfs=7')
1446 @param names: Physical NIC cards correspondent network interfaces,
1448 @param devices_requested: Number of devices being requested.
1451 self
.driver
= driver
1452 self
.driver_option
= driver_option
1454 self
.name_list
= names
.split()
1455 if devices_requested
:
1456 self
.devices_requested
= int(devices_requested
)
1458 self
.devices_requested
= None
1461 def _get_pf_pci_id(self
, name
, search_str
):
1463 Get the PF PCI ID according to name.
1465 @param name: Name of the PCI device.
1466 @param search_str: Search string to be used on lspci.
1468 cmd
= "ethtool -i %s | awk '/bus-info/ {print $2}'" % name
1469 s
, pci_id
= commands
.getstatusoutput(cmd
)
1470 if not (s
or "Cannot get driver information" in pci_id
):
1472 cmd
= "lspci | awk '/%s/ {print $1}'" % search_str
1473 pci_ids
= [id for id in commands
.getoutput(cmd
).splitlines()]
1474 nic_id
= int(re
.search('[0-9]+', name
).group(0))
1475 if (len(pci_ids
) - 1) < nic_id
:
1477 return pci_ids
[nic_id
]
1480 def _release_dev(self
, pci_id
):
1482 Release a single PCI device.
1484 @param pci_id: PCI ID of a given PCI device.
1486 base_dir
= "/sys/bus/pci"
1487 full_id
= get_full_pci_id(pci_id
)
1488 vendor_id
= get_vendor_from_pci_id(pci_id
)
1489 drv_path
= os
.path
.join(base_dir
, "devices/%s/driver" % full_id
)
1490 if 'pci-stub' in os
.readlink(drv_path
):
1491 cmd
= "echo '%s' > %s/new_id" % (vendor_id
, drv_path
)
1495 stub_path
= os
.path
.join(base_dir
, "drivers/pci-stub")
1496 cmd
= "echo '%s' > %s/unbind" % (full_id
, stub_path
)
1500 driver
= self
.dev_drivers
[pci_id
]
1501 cmd
= "echo '%s' > %s/bind" % (full_id
, driver
)
1508 def get_vf_devs(self
):
1510 Catch all VFs PCI IDs.
1512 @return: List with all PCI IDs for the Virtual Functions avaliable
1514 if not self
.sr_iov_setup():
1517 cmd
= "lspci | awk '/Virtual Function/ {print $1}'"
1518 return commands
.getoutput(cmd
).split()
1521 def get_pf_devs(self
):
1523 Catch all PFs PCI IDs.
1525 @return: List with all PCI IDs for the physical hardware requested
1528 for name
in self
.name_list
:
1529 pf_id
= self
._get
_pf
_pci
_id
(name
, "Ethernet")
1532 pf_ids
.append(pf_id
)
1536 def get_devs(self
, count
):
1538 Check out all devices' PCI IDs according to their name.
1540 @param count: count number of PCI devices needed for pass through
1541 @return: a list of all devices' PCI IDs
1543 if self
.type == "vf":
1544 vf_ids
= self
.get_vf_devs()
1545 elif self
.type == "pf":
1546 vf_ids
= self
.get_pf_devs()
1547 elif self
.type == "mixed":
1548 vf_ids
= self
.get_vf_devs()
1549 vf_ids
.extend(self
.get_pf_devs())
1550 return vf_ids
[0:count
]
1553 def get_vfs_count(self
):
1555 Get VFs count number according to lspci.
1557 # FIXME: Need to think out a method of identify which
1558 # 'virtual function' belongs to which physical card considering
1559 # that if the host has more than one 82576 card. PCI_ID?
1560 cmd
= "lspci | grep 'Virtual Function' | wc -l"
1561 return int(commands
.getoutput(cmd
))
1564 def check_vfs_count(self
):
1566 Check VFs count number according to the parameter driver_options.
1568 # Network card 82576 has two network interfaces and each can be
1569 # virtualized up to 7 virtual functions, therefore we multiply
1570 # two for the value of driver_option 'max_vfs'.
1571 expected_count
= int((re
.findall("(\d)", self
.driver_option
)[0])) * 2
1572 return (self
.get_vfs_count
== expected_count
)
1575 def is_binded_to_stub(self
, full_id
):
1577 Verify whether the device with full_id is already binded to pci-stub.
1579 @param full_id: Full ID for the given PCI device
1581 base_dir
= "/sys/bus/pci"
1582 stub_path
= os
.path
.join(base_dir
, "drivers/pci-stub")
1583 if os
.path
.exists(os
.path
.join(stub_path
, full_id
)):
1588 def sr_iov_setup(self
):
1590 Ensure the PCI device is working in sr_iov mode.
1592 Check if the PCI hardware device drive is loaded with the appropriate,
1593 parameters (number of VFs), and if it's not, perform setup.
1595 @return: True, if the setup was completed successfuly, False otherwise.
1598 s
, o
= commands
.getstatusoutput('lsmod | grep %s' % self
.driver
)
1601 elif not self
.check_vfs_count():
1602 os
.system("modprobe -r %s" % self
.driver
)
1607 # Re-probe driver with proper number of VFs
1609 cmd
= "modprobe %s %s" % (self
.driver
, self
.driver_option
)
1610 logging
.info("Loading the driver '%s' with option '%s'",
1611 self
.driver
, self
.driver_option
)
1612 s
, o
= commands
.getstatusoutput(cmd
)
1618 def request_devs(self
):
1620 Implement setup process: unbind the PCI device and then bind it
1621 to the pci-stub driver.
1623 @return: a list of successfully requested devices' PCI IDs.
1625 base_dir
= "/sys/bus/pci"
1626 stub_path
= os
.path
.join(base_dir
, "drivers/pci-stub")
1628 self
.pci_ids
= self
.get_devs(self
.devices_requested
)
1629 logging
.debug("The following pci_ids were found: %s", self
.pci_ids
)
1630 requested_pci_ids
= []
1631 self
.dev_drivers
= {}
1633 # Setup all devices specified for assignment to guest
1634 for pci_id
in self
.pci_ids
:
1635 full_id
= get_full_pci_id(pci_id
)
1638 drv_path
= os
.path
.join(base_dir
, "devices/%s/driver" % full_id
)
1639 dev_prev_driver
= os
.path
.realpath(os
.path
.join(drv_path
,
1640 os
.readlink(drv_path
)))
1641 self
.dev_drivers
[pci_id
] = dev_prev_driver
1643 # Judge whether the device driver has been binded to stub
1644 if not self
.is_binded_to_stub(full_id
):
1645 logging
.debug("Binding device %s to stub", full_id
)
1646 vendor_id
= get_vendor_from_pci_id(pci_id
)
1647 stub_new_id
= os
.path
.join(stub_path
, 'new_id')
1648 unbind_dev
= os
.path
.join(drv_path
, 'unbind')
1649 stub_bind
= os
.path
.join(stub_path
, 'bind')
1651 info_write_to_files
= [(vendor_id
, stub_new_id
),
1652 (full_id
, unbind_dev
),
1653 (full_id
, stub_bind
)]
1655 for content
, file in info_write_to_files
:
1657 utils
.open_write_close(file, content
)
1659 logging
.debug("Failed to write %s to file %s", content
,
1663 if not self
.is_binded_to_stub(full_id
):
1664 logging
.error("Binding device %s to stub failed", pci_id
)
1667 logging
.debug("Device %s already binded to stub", pci_id
)
1668 requested_pci_ids
.append(pci_id
)
1669 self
.pci_ids
= requested_pci_ids
1673 def release_devs(self
):
1675 Release all PCI devices currently assigned to VMs back to the
1676 virtualization host.
1679 for pci_id
in self
.dev_drivers
:
1680 if not self
._release
_dev
(pci_id
):
1681 logging
.error("Failed to release device %s to host", pci_id
)
1683 logging
.info("Released device %s successfully", pci_id
)
1688 class KojiClient(object):
1690 Stablishes a connection with the build system, either koji or brew.
1692 This class provides convenience methods to retrieve information on packages
1693 and the packages themselves hosted on the build system. Packages should be
1694 specified in the KojiPgkSpec syntax.
1697 CMD_LOOKUP_ORDER
= ['/usr/bin/brew', '/usr/bin/koji' ]
1699 CONFIG_MAP
= {'/usr/bin/brew': '/etc/brewkoji.conf',
1700 '/usr/bin/koji': '/etc/koji.conf'}
1703 def __init__(self
, cmd
=None):
1705 Verifies whether the system has koji or brew installed, then loads
1706 the configuration file that will be used to download the files.
1709 @param cmd: Optional command name, either 'brew' or 'koji'. If not
1710 set, get_default_command() is used and to look for
1714 if not KOJI_INSTALLED
:
1715 raise ValueError('No koji/brew installed on the machine')
1717 # Instance variables used by many methods
1720 self
.config_options
= {}
1723 # Set koji command or get default
1725 self
.command
= self
.get_default_command()
1729 # Check koji command
1730 if not self
.is_command_valid():
1731 raise ValueError('Koji command "%s" is not valid' % self
.command
)
1733 # Assuming command is valid, set configuration file and read it
1734 self
.config
= self
.CONFIG_MAP
[self
.command
]
1737 # Setup koji session
1738 server_url
= self
.config_options
['server']
1739 session_options
= self
.get_session_options()
1740 self
.session
= koji
.ClientSession(server_url
,
1744 def read_config(self
, check_is_valid
=True):
1746 Reads options from the Koji configuration file
1748 By default it checks if the koji configuration is valid
1750 @type check_valid: boolean
1751 @param check_valid: whether to include a check on the configuration
1756 if not self
.is_config_valid():
1757 raise ValueError('Koji config "%s" is not valid' % self
.config
)
1759 config
= ConfigParser
.ConfigParser()
1760 config
.read(self
.config
)
1762 basename
= os
.path
.basename(self
.command
)
1763 for name
, value
in config
.items(basename
):
1764 self
.config_options
[name
] = value
1767 def get_session_options(self
):
1769 Filter only options necessary for setting up a cobbler client session
1771 @returns: only the options used for session setup
1773 session_options
= {}
1774 for name
, value
in self
.config_options
.items():
1775 if name
in ('user', 'password', 'debug_xmlrpc', 'debug'):
1776 session_options
[name
] = value
1777 return session_options
1780 def is_command_valid(self
):
1782 Checks if the currently set koji command is valid
1784 @returns: True or False
1786 koji_command_ok
= True
1788 if not os
.path
.isfile(self
.command
):
1789 logging
.error('Koji command "%s" is not a regular file',
1791 koji_command_ok
= False
1793 if not os
.access(self
.command
, os
.X_OK
):
1794 logging
.warn('Koji command "%s" is not executable: this is '
1795 'not fatal but indicates an unexpected situation',
1798 if not self
.command
in self
.CONFIG_MAP
.keys():
1799 logging
.error('Koji command "%s" does not have a configuration '
1800 'file associated to it', self
.command
)
1801 koji_command_ok
= False
1803 return koji_command_ok
1806 def is_config_valid(self
):
1808 Checks if the currently set koji configuration is valid
1810 @returns: True or False
1812 koji_config_ok
= True
1814 if not os
.path
.isfile(self
.config
):
1815 logging
.error('Koji config "%s" is not a regular file', self
.config
)
1816 koji_config_ok
= False
1818 if not os
.access(self
.config
, os
.R_OK
):
1819 logging
.error('Koji config "%s" is not readable', self
.config
)
1820 koji_config_ok
= False
1822 config
= ConfigParser
.ConfigParser()
1823 config
.read(self
.config
)
1824 basename
= os
.path
.basename(self
.command
)
1825 if not config
.has_section(basename
):
1826 logging
.error('Koji configuration file "%s" does not have a '
1827 'section "%s", named after the base name of the '
1828 'currently set koji command "%s"', self
.config
,
1829 basename
, self
.command
)
1830 koji_config_ok
= False
1832 return koji_config_ok
1835 def get_default_command(self
):
1837 Looks up for koji or brew "binaries" on the system
1839 Systems with plain koji usually don't have a brew cmd, while systems
1840 with koji, have *both* koji and brew utilities. So we look for brew
1841 first, and if found, we consider that the system is configured for
1842 brew. If not, we consider this is a system with plain koji.
1844 @returns: either koji or brew command line executable path, or None
1847 for command
in self
.CMD_LOOKUP_ORDER
:
1848 if os
.path
.isfile(command
):
1849 koji_command
= command
1852 koji_command_basename
= os
.path
.basename(koji_command
)
1854 koji_command
= os_dep
.command(koji_command_basename
)
1861 def get_pkg_info(self
, pkg
):
1863 Returns information from Koji on the package
1865 @type pkg: KojiPkgSpec
1866 @param pkg: information about the package, as a KojiPkgSpec instance
1868 @returns: information from Koji about the specified package
1871 if pkg
.build
is not None:
1872 info
= self
.session
.getBuild(int(pkg
.build
))
1873 elif pkg
.tag
is not None and pkg
.package
is not None:
1874 builds
= self
.session
.listTagged(pkg
.tag
,
1877 package
=pkg
.package
)
1883 def is_pkg_valid(self
, pkg
):
1885 Checks if this package is altogether valid on Koji
1887 This verifies if the build or tag specified in the package
1888 specification actually exist on the Koji server
1890 @returns: True or False
1894 if not self
.is_pkg_spec_build_valid(pkg
):
1897 if not self
.is_pkg_spec_tag_valid(pkg
):
1904 def is_pkg_spec_build_valid(self
, pkg
):
1906 Checks if build is valid on Koji
1908 @param pkg: a Pkg instance
1910 if pkg
.build
is not None:
1911 info
= self
.session
.getBuild(int(pkg
.build
))
1917 def is_pkg_spec_tag_valid(self
, pkg
):
1919 Checks if tag is valid on Koji
1921 @type pkg: KojiPkgSpec
1922 @param pkg: a package specification
1924 if pkg
.tag
is not None:
1925 tag
= self
.session
.getTag(pkg
.tag
)
1931 def get_pkg_rpm_info(self
, pkg
, arch
=None):
1933 Returns a list of infomation on the RPM packages found on koji
1935 @type pkg: KojiPkgSpec
1936 @param pkg: a package specification
1938 @param arch: packages built for this architecture, but also including
1939 architecture independent (noarch) packages
1942 arch
= utils
.get_arch()
1944 info
= self
.get_pkg_info(pkg
)
1946 rpms
= self
.session
.listRPMs(buildID
=info
['id'],
1947 arches
=[arch
, 'noarch'])
1949 rpms
= [d
for d
in rpms
if d
['name'] in pkg
.subpackages
]
1953 def get_pkg_rpm_names(self
, pkg
, arch
=None):
1955 Gets the names for the RPM packages specified in pkg
1957 @type pkg: KojiPkgSpec
1958 @param pkg: a package specification
1960 @param arch: packages built for this architecture, but also including
1961 architecture independent (noarch) packages
1964 arch
= utils
.get_arch()
1965 rpms
= self
.get_pkg_rpm_info(pkg
, arch
)
1966 return [rpm
['name'] for rpm
in rpms
]
1969 def get_pkg_rpm_file_names(self
, pkg
, arch
=None):
1971 Gets the file names for the RPM packages specified in pkg
1973 @type pkg: KojiPkgSpec
1974 @param pkg: a package specification
1976 @param arch: packages built for this architecture, but also including
1977 architecture independent (noarch) packages
1980 arch
= utils
.get_arch()
1982 rpms
= self
.get_pkg_rpm_info(pkg
, arch
)
1984 arch_rpm_name
= koji
.pathinfo
.rpm(rpm
)
1985 rpm_name
= os
.path
.basename(arch_rpm_name
)
1986 rpm_names
.append(rpm_name
)
1990 def get_pkg_urls(self
, pkg
, arch
=None):
1992 Gets the urls for the packages specified in pkg
1994 @type pkg: KojiPkgSpec
1995 @param pkg: a package specification
1997 @param arch: packages built for this architecture, but also including
1998 architecture independent (noarch) packages
2000 info
= self
.get_pkg_info(pkg
)
2001 rpms
= self
.get_pkg_rpm_info(pkg
, arch
)
2004 rpm_name
= koji
.pathinfo
.rpm(rpm
)
2005 url
= ("%s/%s/%s/%s/%s" % (self
.config_options
['pkgurl'],
2006 info
['package_name'],
2007 info
['version'], info
['release'],
2009 rpm_urls
.append(url
)
2013 def get_pkgs(self
, pkg
, dst_dir
, arch
=None):
2015 Download the packages
2017 @type pkg: KojiPkgSpec
2018 @param pkg: a package specification
2019 @type dst_dir: string
2020 @param dst_dir: the destination directory, where the downloaded
2021 packages will be saved on
2023 @param arch: packages built for this architecture, but also including
2024 architecture independent (noarch) packages
2026 rpm_urls
= self
.get_pkg_urls(pkg
, arch
)
2027 for url
in rpm_urls
:
2029 os
.path
.join(dst_dir
, os
.path
.basename(url
)))
2032 DEFAULT_KOJI_TAG
= None
2033 def set_default_koji_tag(tag
):
2035 Sets the default tag that will be used
2037 global DEFAULT_KOJI_TAG
2038 DEFAULT_KOJI_TAG
= tag
2041 def get_default_koji_tag():
2042 return DEFAULT_KOJI_TAG
2047 A package specification syntax parser for Koji
2049 This holds information on either tag or build, and packages to be fetched
2050 from koji and possibly installed (features external do this class).
2052 New objects can be created either by providing information in the textual
2053 format or by using the actual parameters for tag, build, package and sub-
2054 packages. The textual format is useful for command line interfaces and
2055 configuration files, while using parameters is better for using this in
2056 a programatic fashion.
2058 The following sets of examples are interchangeable. Specifying all packages
2059 part of build number 1000:
2061 >>> from kvm_utils import KojiPkgSpec
2062 >>> pkg = KojiPkgSpec('1000')
2064 >>> pkg = KojiPkgSpec(build=1000)
2066 Specifying only a subset of packages of build number 1000:
2068 >>> pkg = KojiPkgSpec('1000:kernel,kernel-devel')
2070 >>> pkg = KojiPkgSpec(build=1000,
2071 subpackages=['kernel', 'kernel-devel'])
2073 Specifying the latest build for the 'kernel' package tagged with 'dist-f14':
2075 >>> pkg = KojiPkgSpec('dist-f14:kernel')
2077 >>> pkg = KojiPkgSpec(tag='dist-f14', package='kernel')
2079 Specifying the 'kernel' package using the default tag:
2081 >>> kvm_utils.set_default_koji_tag('dist-f14')
2082 >>> pkg = KojiPkgSpec('kernel')
2084 >>> pkg = KojiPkgSpec(package='kernel')
2086 Specifying the 'kernel' package using the default tag:
2088 >>> kvm_utils.set_default_koji_tag('dist-f14')
2089 >>> pkg = KojiPkgSpec('kernel')
2091 >>> pkg = KojiPkgSpec(package='kernel')
2093 If you do not specify a default tag, and give a package name without an
2094 explicit tag, your package specification is considered invalid:
2096 >>> print kvm_utils.get_default_koji_tag()
2098 >>> print kvm_utils.KojiPkgSpec('kernel').is_valid()
2101 >>> print kvm_utils.KojiPkgSpec(package='kernel').is_valid()
2107 def __init__(self
, text
='', tag
=None, build
=None,
2108 package
=None, subpackages
=[]):
2110 Instantiates a new KojiPkgSpec object
2113 @param text: a textual representation of a package on Koji that
2116 @param tag: a koji tag, example: Fedora-14-RELEASE
2117 (see U{http://fedoraproject.org/wiki/Koji#Tags_and_Targets})
2119 @param build: a koji build, example: 1001
2120 (see U{http://fedoraproject.org/wiki/Koji#Koji_Architecture})
2121 @type package: string
2122 @param package: a koji package, example: python
2123 (see U{http://fedoraproject.org/wiki/Koji#Koji_Architecture})
2124 @type subpackages: list of strings
2125 @param subpackages: a list of package names, usually a subset of
2126 the RPM packages generated by a given build
2129 # Set to None to indicate 'not set' (and be able to use 'is')
2133 self
.subpackages
= []
2135 self
.default_tag
= None
2137 # Textual representation takes precedence (most common use case)
2143 self
.package
= package
2144 self
.subpackages
= subpackages
2146 # Set the default tag, if set, as a fallback
2147 if not self
.build
and not self
.tag
:
2148 default_tag
= get_default_koji_tag()
2149 if default_tag
is not None:
2150 self
.tag
= default_tag
2153 def parse(self
, text
):
2155 Parses a textual representation of a package specification
2158 @param text: textual representation of a package in koji
2160 parts
= text
.count(self
.SEP
) + 1
2167 part1
, part2
= text
.split(self
.SEP
)
2170 self
.subpackages
= part2
.split(',')
2173 self
.package
= part2
2175 # Instead of erroring on more arguments, we simply ignore them
2176 # This makes the parser suitable for future syntax additions, such
2177 # as specifying the package architecture
2178 part1
, part2
, part3
= text
.split(self
.SEP
)[0:3]
2180 self
.package
= part2
2181 self
.subpackages
= part3
.split(',')
2184 def _is_invalid_neither_tag_or_build(self
):
2186 Checks if this package is invalid due to not having either a valid
2187 tag or build set, that is, both are empty.
2189 @returns: True if this is invalid and False if it's valid
2191 return (self
.tag
is None and self
.build
is None)
2194 def _is_invalid_package_but_no_tag(self
):
2196 Checks if this package is invalid due to having a package name set
2197 but tag or build set, that is, both are empty.
2199 @returns: True if this is invalid and False if it's valid
2201 return (self
.package
and not self
.tag
)
2204 def _is_invalid_subpackages_but_no_main_package(self
):
2206 Checks if this package is invalid due to having a tag set (this is Ok)
2207 but specifying subpackage names without specifying the main package
2210 Specifying subpackages without a main package name is only valid when
2211 a build is used instead of a tag.
2213 @returns: True if this is invalid and False if it's valid
2215 return (self
.tag
and self
.subpackages
and not self
.package
)
2220 Checks if this package specification is valid.
2222 Being valid means that it has enough and not conflicting information.
2223 It does not validate that the packages specified actually existe on
2226 @returns: True or False
2228 if self
._is
_invalid
_neither
_tag
_or
_build
():
2230 elif self
._is
_invalid
_package
_but
_no
_tag
():
2232 elif self
._is
_invalid
_subpackages
_but
_no
_main
_package
():
2238 def describe_invalid(self
):
2240 Describes why this is not valid, in a human friendly way
2242 if self
._is
_invalid
_neither
_tag
_or
_build
():
2243 return 'neither a tag or build are set, and of them should be set'
2244 elif self
._is
_invalid
_package
_but
_no
_tag
():
2245 return 'package name specified but no tag is set'
2246 elif self
._is
_invalid
_subpackages
_but
_no
_main
_package
():
2247 return 'subpackages specified but no main package is set'
2249 return 'unkwown reason, seems to be valid'
2254 Describe this package specification, in a human friendly way
2256 @returns: package specification description
2260 if not self
.subpackages
:
2261 description
+= 'all subpackages from %s ' % self
.package
2263 description
+= ('only subpackage(s) %s from package %s ' %
2264 (', '.join(self
.subpackages
), self
.package
))
2267 description
+= 'from build %s' % self
.build
2269 description
+= 'tagged with %s' % self
.tag
2271 raise ValueError, 'neither build or tag is set'
2275 return ('Invalid package specification: %s' %
2276 self
.describe_invalid())
2280 return ("<KojiPkgSpec tag=%s build=%s pkg=%s subpkgs=%s>" %
2281 (self
.tag
, self
.build
, self
.package
,
2282 ", ".join(self
.subpackages
)))
2285 def umount(src
, mount_point
, type):
2287 Umount the src mounted in mount_point.
2290 @mount_point: mount point
2291 @type: file system type
2294 mount_string
= "%s %s %s" % (src
, mount_point
, type)
2295 if mount_string
in file("/etc/mtab").read():
2296 umount_cmd
= "umount %s" % mount_point
2298 utils
.system(umount_cmd
)
2300 except error
.CmdError
:
2303 logging
.debug("%s is not mounted under %s", src
, mount_point
)
2307 def mount(src
, mount_point
, type, perm
="rw"):
2309 Mount the src into mount_point of the host.
2312 @mount_point: mount point
2313 @type: file system type
2314 @perm: mount premission
2316 umount(src
, mount_point
, type)
2317 mount_string
= "%s %s %s %s" % (src
, mount_point
, type, perm
)
2319 if mount_string
in file("/etc/mtab").read():
2320 logging
.debug("%s is already mounted in %s with %s",
2321 src
, mount_point
, perm
)
2324 mount_cmd
= "mount -t %s %s %s -o %s" % (type, src
, mount_point
, perm
)
2326 utils
.system(mount_cmd
)
2327 except error
.CmdError
:
2330 logging
.debug("Verify the mount through /etc/mtab")
2331 if mount_string
in file("/etc/mtab").read():
2332 logging
.debug("%s is successfully mounted", src
)
2335 logging
.error("Can't find mounted NFS share - /etc/mtab contents \n%s",
2336 file("/etc/mtab").read())
2340 def install_host_kernel(job
, params
):
2342 Install a host kernel, given the appropriate params.
2344 @param job: Job object.
2345 @param params: Dict with host kernel install params.
2347 install_type
= params
.get('host_kernel_install_type')
2349 rpm_url
= params
.get('host_kernel_rpm_url')
2351 koji_cmd
= params
.get('host_kernel_koji_cmd')
2352 koji_build
= params
.get('host_kernel_koji_build')
2353 koji_tag
= params
.get('host_kernel_koji_tag')
2355 git_repo
= params
.get('host_kernel_git_repo')
2356 git_branch
= params
.get('host_kernel_git_branch')
2357 git_commit
= params
.get('host_kernel_git_commit')
2358 patch_list
= params
.get('host_kernel_patch_list')
2360 patch_list
= patch_list
.split()
2361 kernel_config
= params
.get('host_kernel_config')
2363 if install_type
== 'rpm':
2364 logging
.info('Installing host kernel through rpm')
2365 dst
= os
.path
.join("/tmp", os
.path
.basename(rpm_url
))
2366 k
= utils
.get_file(rpm_url
, dst
)
2367 host_kernel
= job
.kernel(k
)
2368 host_kernel
.install(install_vmlinux
=False)
2371 elif install_type
in ['koji', 'brew']:
2372 k_deps
= KojiPkgSpec(tag
=koji_tag
, package
='kernel',
2373 subpackages
=['kernel-devel', 'kernel-firmware'])
2374 k
= KojiPkgSpec(tag
=koji_tag
, package
='kernel',
2375 subpackages
=['kernel'])
2377 c
= KojiClient(koji_cmd
)
2378 logging
.info('Fetching kernel dependencies (-devel, -firmware)')
2379 c
.get_pkgs(k_deps
, job
.tmpdir
)
2380 logging
.info('Installing kernel dependencies (-devel, -firmware) '
2381 'through %s', install_type
)
2382 k_deps_rpm_file_names
= [os
.path
.join(job
.tmpdir
, rpm_file_name
) for
2383 rpm_file_name
in c
.get_pkg_rpm_file_names(k_deps
)]
2384 utils
.run('rpm -U --force %s' % " ".join(k_deps_rpm_file_names
))
2386 c
.get_pkgs(k
, job
.tmpdir
)
2387 k_rpm
= os
.path
.join(job
.tmpdir
,
2388 c
.get_pkg_rpm_file_names(k
)[0])
2389 host_kernel
= job
.kernel(k_rpm
)
2390 host_kernel
.install(install_vmlinux
=False)
2393 elif install_type
== 'git':
2394 logging
.info('Chose to install host kernel through git, proceeding')
2395 repodir
= os
.path
.join("/tmp", 'kernel_src')
2396 r
= get_git_branch(git_repo
, git_branch
, repodir
, git_commit
)
2397 host_kernel
= job
.kernel(r
)
2399 host_kernel
.patch(patch_list
)
2400 host_kernel
.config(kernel_config
)
2402 host_kernel
.install()
2406 logging
.info('Chose %s, using the current kernel for the host',
2410 def if_nametoindex(ifname
):
2412 Map an interface name into its corresponding index.
2413 Returns 0 on error, as 0 is not a valid index
2415 @param ifname: interface name
2418 ctrl_sock
= socket
.socket(socket
.AF_INET
, socket
.SOCK_DGRAM
, 0)
2419 ifr
= struct
.pack("16si", ifname
, 0)
2420 r
= fcntl
.ioctl(ctrl_sock
, SIOCGIFINDEX
, ifr
)
2421 index
= struct
.unpack("16si", r
)[1]
2426 def vnet_hdr_probe(tapfd
):
2428 Check if the IFF_VNET_HDR is support by tun.
2430 @param tapfd: the file descriptor of /dev/net/tun
2432 u
= struct
.pack("I", 0)
2434 r
= fcntl
.ioctl(tapfd
, TUNGETFEATURES
, u
)
2435 except OverflowError:
2437 flags
= struct
.unpack("I", r
)[0]
2438 if flags
& IFF_VNET_HDR
:
2444 def open_tap(devname
, ifname
, vnet_hdr
=True):
2446 Open a tap device and returns its file descriptor which is used by
2447 fd=<fd> parameter of qemu-kvm.
2449 @param ifname: TAP interface name
2450 @param vnet_hdr: Whether enable the vnet header
2453 tapfd
= os
.open(devname
, os
.O_RDWR
)
2455 raise TAPModuleError(devname
, "open", e
)
2456 flags
= IFF_TAP | IFF_NO_PI
2457 if vnet_hdr
and vnet_hdr_probe(tapfd
):
2458 flags |
= IFF_VNET_HDR
2460 ifr
= struct
.pack("16sh", ifname
, flags
)
2462 r
= fcntl
.ioctl(tapfd
, TUNSETIFF
, ifr
)
2463 except IOError, details
:
2464 raise TAPCreationError(ifname
, details
)
2465 ifname
= struct
.unpack("16sh", r
)[0].strip("\x00")
2469 def add_to_bridge(ifname
, brname
):
2471 Add a TAP device to bridge
2473 @param ifname: Name of TAP device
2474 @param brname: Name of the bridge
2476 ctrl_sock
= socket
.socket(socket
.AF_INET
, socket
.SOCK_DGRAM
, 0)
2477 index
= if_nametoindex(ifname
)
2479 raise TAPNotExistError(ifname
)
2480 ifr
= struct
.pack("16si", brname
, index
)
2482 r
= fcntl
.ioctl(ctrl_sock
, SIOCBRADDIF
, ifr
)
2483 except IOError, details
:
2484 raise BRAddIfError(ifname
, brname
, details
)
2488 def bring_up_ifname(ifname
):
2490 Bring up an interface
2492 @param ifname: Name of the interface
2494 ctrl_sock
= socket
.socket(socket
.AF_INET
, socket
.SOCK_DGRAM
, 0)
2495 ifr
= struct
.pack("16si", ifname
, IFF_UP
)
2497 fcntl
.ioctl(ctrl_sock
, SIOCSIFFLAGS
, ifr
)
2499 raise TAPBringUpError(ifname
)
2503 def if_set_macaddress(ifname
, mac
):
2505 Set the mac address for an interface
2507 @param ifname: Name of the interface
2510 ctrl_sock
= socket
.socket(socket
.AF_INET
, socket
.SOCK_DGRAM
, 0)
2512 ifr
= struct
.pack("256s", ifname
)
2514 mac_dev
= fcntl
.ioctl(ctrl_sock
, SIOCGIFHWADDR
, ifr
)[18:24]
2515 mac_dev
= ":".join(["%02x" % ord(m
) for m
in mac_dev
])
2517 raise HwAddrGetError(ifname
)
2519 if mac_dev
.lower() == mac
.lower():
2522 ifr
= struct
.pack("16sH14s", ifname
, 1,
2523 "".join([chr(int(m
, 16)) for m
in mac
.split(":")]))
2525 fcntl
.ioctl(ctrl_sock
, SIOCSIFHWADDR
, ifr
)
2528 raise HwAddrSetError(ifname
, mac
)