2 // This file is part of Moodle - http://moodle.org/
4 // Moodle is free software: you can redistribute it and/or modify
5 // it under the terms of the GNU General Public License as published by
6 // the Free Software Foundation, either version 3 of the License, or
7 // (at your option) any later version.
9 // Moodle is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 // GNU General Public License for more details.
14 // You should have received a copy of the GNU General Public License
15 // along with Moodle. If not, see <http://www.gnu.org/licenses/>.
18 * Behat hooks steps definitions.
20 * This methods are used by Behat CLI command.
24 * @copyright 2012 David MonllaĆ³
25 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
28 // NOTE: no MOODLE_INTERNAL test here, this file may be required by behat before including /config.php.
30 require_once(__DIR__
. '/../../behat/behat_base.php');
32 use Behat\Testwork\Hook\Scope\BeforeSuiteScope
,
33 Behat\Testwork\Hook\Scope\AfterSuiteScope
,
34 Behat\Behat\Hook\Scope\BeforeFeatureScope
,
35 Behat\Behat\Hook\Scope\AfterFeatureScope
,
36 Behat\Behat\Hook\Scope\BeforeScenarioScope
,
37 Behat\Behat\Hook\Scope\AfterScenarioScope
,
38 Behat\Behat\Hook\Scope\BeforeStepScope
,
39 Behat\Behat\Hook\Scope\AfterStepScope
,
40 Behat\Mink\Exception\DriverException
as DriverException
,
41 WebDriver\Exception\NoSuchWindow
as NoSuchWindow
,
42 WebDriver\Exception\UnexpectedAlertOpen
as UnexpectedAlertOpen
,
43 WebDriver\Exception\UnknownError
as UnknownError
,
44 WebDriver\Exception\CurlExec
as CurlExec
,
45 WebDriver\Exception\NoAlertOpenError
as NoAlertOpenError
;
48 * Hooks to the behat process.
50 * Behat accepts hooks after and before each
51 * suite, feature, scenario and step.
53 * They can not call other steps as part of their process
54 * like regular steps definitions does.
56 * Throws generic Exception because they are captured by Behat.
60 * @copyright 2012 David MonllaĆ³
61 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
63 class behat_hooks
extends behat_base
{
66 * @var Last browser session start time.
68 protected static $lastbrowsersessionstart = 0;
71 * @var For actions that should only run once.
73 protected static $initprocessesfinished = false;
76 * Some exceptions can only be caught in a before or after step hook,
77 * they can not be thrown there as they will provoke a framework level
78 * failure, but we can store them here to fail the step in i_look_for_exceptions()
79 * which result will be parsed by the framework as the last step result.
81 * @var Null or the exception last step throw in the before or after hook.
83 protected static $currentstepexception = null;
86 * If we are saving any kind of dump on failure we should use the same parent dir during a run.
88 * @var The parent dir name
90 protected static $faildumpdirname = false;
93 * Keeps track of time taken by feature to execute.
95 * @var array list of feature timings
97 protected static $timings = array();
100 * Keeps track of current running suite name.
102 * @var string current running suite name
104 protected static $runningsuite = '';
107 * Hook to capture BeforeSuite event so as to give access to moodle codebase.
108 * This will try and catch any exception and exists if anything fails.
110 * @param BeforeSuiteScope $scope scope passed by event fired before suite.
113 public static function before_suite_hook(BeforeSuiteScope
$scope) {
114 // If behat has been initialised then no need to do this again.
115 if (self
::$initprocessesfinished) {
120 self
::before_suite($scope);
121 } catch (behat_stop_exception
$e) {
122 echo $e->getMessage() . PHP_EOL
;
128 * Gives access to moodle codebase, ensures all is ready and sets up the test lock.
130 * Includes config.php to use moodle codebase with $CFG->behat_*
131 * instead of $CFG->prefix and $CFG->dataroot, called once per suite.
133 * @param BeforeSuiteScope $scope scope passed by event fired before suite.
135 * @throws behat_stop_exception
137 public static function before_suite(BeforeSuiteScope
$scope) {
140 // Defined only when the behat CLI command is running, the moodle init setup process will
141 // read this value and switch to $CFG->behat_dataroot and $CFG->behat_prefix instead of
143 if (!defined('BEHAT_TEST')) {
144 define('BEHAT_TEST', 1);
147 if (!defined('CLI_SCRIPT')) {
148 define('CLI_SCRIPT', 1);
151 // With BEHAT_TEST we will be using $CFG->behat_* instead of $CFG->dataroot, $CFG->prefix and $CFG->wwwroot.
152 require_once(__DIR__
. '/../../../config.php');
154 // Now that we are MOODLE_INTERNAL.
155 require_once(__DIR__
. '/../../behat/classes/behat_command.php');
156 require_once(__DIR__
. '/../../behat/classes/behat_selectors.php');
157 require_once(__DIR__
. '/../../behat/classes/behat_context_helper.php');
158 require_once(__DIR__
. '/../../behat/classes/util.php');
159 require_once(__DIR__
. '/../../testing/classes/test_lock.php');
160 require_once(__DIR__
. '/../../testing/classes/nasty_strings.php');
162 // Avoids vendor/bin/behat to be executed directly without test environment enabled
163 // to prevent undesired db & dataroot modifications, this is also checked
164 // before each scenario (accidental user deletes) in the BeforeScenario hook.
166 if (!behat_util
::is_test_mode_enabled()) {
167 throw new behat_stop_exception('Behat only can run if test mode is enabled. More info in ' .
168 behat_command
::DOCS_URL
. '#Running_tests');
171 // Reset all data, before checking for check_server_status.
172 // If not done, then it can return apache error, while running tests.
173 behat_util
::clean_tables_updated_by_scenario_list();
174 behat_util
::reset_all_data();
176 // Check if server is running and using same version for cli and apache.
177 behat_util
::check_server_status();
179 // Prevents using outdated data, upgrade script would start and tests would fail.
180 if (!behat_util
::is_test_data_updated()) {
181 $commandpath = 'php admin/tool/behat/cli/init.php';
182 throw new behat_stop_exception("Your behat test site is outdated, please run\n\n " .
183 $commandpath . "\n\nfrom your moodle dirroot to drop and install the behat test site again.");
185 // Avoid parallel tests execution, it continues when the previous lock is released.
186 test_lock
::acquire('behat');
188 // Store the browser reset time if reset after N seconds is specified in config.php.
189 if (!empty($CFG->behat_restart_browser_after
)) {
190 // Store the initial browser session opening.
191 self
::$lastbrowsersessionstart = time();
194 if (!empty($CFG->behat_faildump_path
) && !is_writable($CFG->behat_faildump_path
)) {
195 throw new behat_stop_exception('You set $CFG->behat_faildump_path to a non-writable directory');
198 // Handle interrupts on PHP7.
199 if (extension_loaded('pcntl')) {
200 $disabled = explode(',', ini_get('disable_functions'));
201 if (!in_array('pcntl_signal', $disabled)) {
208 * Gives access to moodle codebase, to keep track of feature start time.
210 * @param BeforeFeatureScope $scope scope passed by event fired before feature.
213 public static function before_feature(BeforeFeatureScope
$scope) {
214 if (!defined('BEHAT_FEATURE_TIMING_FILE')) {
217 $file = $scope->getFeature()->getFile();
218 self
::$timings[$file] = microtime(true);
222 * Gives access to moodle codebase, to keep track of feature end time.
224 * @param AfterFeatureScope $scope scope passed by event fired after feature.
227 public static function after_feature(AfterFeatureScope
$scope) {
228 if (!defined('BEHAT_FEATURE_TIMING_FILE')) {
231 $file = $scope->getFeature()->getFile();
232 self
::$timings[$file] = microtime(true) - self
::$timings[$file];
233 // Probably didn't actually run this, don't output it.
234 if (self
::$timings[$file] < 1) {
235 unset(self
::$timings[$file]);
240 * Gives access to moodle codebase, to keep track of suite timings.
242 * @param AfterSuiteScope $scope scope passed by event fired after suite.
245 public static function after_suite(AfterSuiteScope
$scope) {
246 if (!defined('BEHAT_FEATURE_TIMING_FILE')) {
249 $realroot = realpath(__DIR__
.'/../../../').'/';
250 foreach (self
::$timings as $k => $v) {
251 $new = str_replace($realroot, '', $k);
252 self
::$timings[$new] = round($v, 1);
253 unset(self
::$timings[$k]);
255 if ($existing = @json_decode
(file_get_contents(BEHAT_FEATURE_TIMING_FILE
), true)) {
256 self
::$timings = array_merge($existing, self
::$timings);
258 arsort(self
::$timings);
259 @file_put_contents
(BEHAT_FEATURE_TIMING_FILE
, json_encode(self
::$timings, JSON_PRETTY_PRINT
));
263 * Hook to capture before scenario event to get scope.
265 * @param BeforeScenarioScope $scope scope passed by event fired before scenario.
268 public function before_scenario_hook(BeforeScenarioScope
$scope) {
270 $this->before_scenario($scope);
271 } catch (behat_stop_exception
$e) {
272 echo $e->getMessage() . PHP_EOL
;
278 * Resets the test environment.
280 * @param BeforeScenarioScope $scope scope passed by event fired before scenario.
281 * @throws behat_stop_exception If here we are not using the test database it should be because of a coding error
283 public function before_scenario(BeforeScenarioScope
$scope) {
286 // As many checks as we can.
287 if (!defined('BEHAT_TEST') ||
288 !defined('BEHAT_SITE_RUNNING') ||
289 php_sapi_name() != 'cli' ||
290 !behat_util
::is_test_mode_enabled() ||
291 !behat_util
::is_test_site()) {
292 throw new behat_stop_exception('Behat only can modify the test database and the test dataroot!');
295 $moreinfo = 'More info in ' . behat_command
::DOCS_URL
. '#Running_tests';
296 $driverexceptionmsg = 'Selenium server is not running, you need to start it to run tests that involve Javascript. ' . $moreinfo;
298 $session = $this->getSession();
299 } catch (CurlExec
$e) {
300 // Exception thrown by WebDriver, so only @javascript tests will be caugth; in
301 // behat_util::check_server_status() we already checked that the server is running.
302 throw new behat_stop_exception($driverexceptionmsg);
303 } catch (DriverException
$e) {
304 throw new behat_stop_exception($driverexceptionmsg);
305 } catch (UnknownError
$e) {
306 // Generic 'I have no idea' Selenium error. Custom exception to provide more feedback about possible solutions.
307 throw new behat_stop_exception($e->getMessage());
310 $suitename = $scope->getSuite()->getName();
312 // Register behat selectors for theme, if suite is changed. We do it for every suite change.
313 if ($suitename !== self
::$runningsuite) {
314 behat_context_helper
::set_environment($scope->getEnvironment());
316 // We need the Mink session to do it and we do it only before the first scenario.
317 $namedpartialclass = 'behat_partial_named_selector';
318 $namedexactclass = 'behat_exact_named_selector';
320 // If override selector exist, then set it as default behat selectors class.
321 $overrideclass = behat_config_util
::get_behat_theme_selector_override_classname($suitename, 'named_partial', true);
322 if (class_exists($overrideclass)) {
323 $namedpartialclass = $overrideclass;
326 // If override selector exist, then set it as default behat selectors class.
327 $overrideclass = behat_config_util
::get_behat_theme_selector_override_classname($suitename, 'named_exact', true);
328 if (class_exists($overrideclass)) {
329 $namedexactclass = $overrideclass;
332 $this->getSession()->getSelectorsHandler()->registerSelector('named_partial', new $namedpartialclass());
333 $this->getSession()->getSelectorsHandler()->registerSelector('named_exact', new $namedexactclass());
336 // Reset mink session between the scenarios.
340 \core\session\manager
::init_empty_session();
342 // Ignore E_NOTICE and E_WARNING during reset, as this might be caused because of some existing process
343 // running ajax. This will be investigated in another issue.
344 $errorlevel = error_reporting();
345 error_reporting($errorlevel & ~E_NOTICE
& ~E_WARNING
);
346 behat_util
::reset_all_data();
347 error_reporting($errorlevel);
349 // Assign valid data to admin user (some generator-related code needs a valid user).
350 $user = $DB->get_record('user', array('username' => 'admin'));
351 \core\session\manager
::set_user($user);
353 // Reset the browser if specified in config.php.
354 if (!empty($CFG->behat_restart_browser_after
) && $this->running_javascript()) {
356 if (self
::$lastbrowsersessionstart +
$CFG->behat_restart_browser_after
< $now) {
358 self
::$lastbrowsersessionstart = $now;
362 // Set the theme if not default.
363 if ($suitename !== "default") {
364 set_config('theme', $suitename);
365 self
::$runningsuite = $suitename;
368 // Start always in the the homepage.
370 // Let's be conservative as we never know when new upstream issues will affect us.
371 $session->visit($this->locate_path('/'));
372 } catch (UnknownError
$e) {
373 throw new behat_stop_exception($e->getMessage());
376 // Checking that the root path is a Moodle test site.
377 if (self
::is_first_scenario()) {
378 $notestsiteexception = new behat_stop_exception('The base URL (' . $CFG->wwwroot
. ') is not a behat test site, ' .
379 'ensure you started the built-in web server in the correct directory or your web server is correctly started and set up');
380 $this->find("xpath", "//head/child::title[normalize-space(.)='" . behat_util
::BEHATSITENAME
. "']", $notestsiteexception);
382 self
::$initprocessesfinished = true;
385 // Run all test with medium (1024x768) screen size, to avoid responsive problems.
386 $this->resize_window('medium');
390 * Executed after scenario to go to a page where no JS is executed.
391 * This will ensure there are no unwanted ajax calls from browser and
392 * site can be reset safely.
394 * @param AfterScenarioScope $scope scope passed by event fired after scenario.
397 public function after_scenario(AfterScenarioScope
$scope) {
399 $this->wait_for_pending_js();
400 $this->getSession()->reset();
401 } catch (DriverException
$e) {
402 // Try restart session, if DriverException caught.
404 $this->getSession()->restart();
405 } catch (DriverException
$e) {
406 // Do nothing, as this will be caught while starting session in before_scenario.
412 * Wait for JS to complete before beginning interacting with the DOM.
414 * Executed only when running against a real browser. We wrap it
415 * all in a try & catch to forward the exception to i_look_for_exceptions
416 * so the exception will be at scenario level, which causes a failure, by
417 * default would be at framework level, which will stop the execution of
420 * @param BeforeStepScope $scope scope passed by event fired before step.
423 public function before_step_javascript(BeforeStepScope
$scope) {
424 self
::$currentstepexception = null;
427 if ($this->running_javascript()) {
429 $this->wait_for_pending_js();
430 } catch (Exception
$e) {
431 self
::$currentstepexception = $e;
437 * Wait for JS to complete after finishing the step.
439 * With this we ensure that there are not AJAX calls
442 * Executed only when running against a real browser. We wrap it
443 * all in a try & catch to forward the exception to i_look_for_exceptions
444 * so the exception will be at scenario level, which causes a failure, by
445 * default would be at framework level, which will stop the execution of
448 * @param AfterStepScope $scope scope passed by event fired after step..
451 public function after_step_javascript(AfterStepScope
$scope) {
454 // Save the page content if the step failed.
455 if (!empty($CFG->behat_faildump_path
) &&
456 $scope->getTestResult()->getResultCode() === Behat\Testwork\Tester\Result\TestResult
::FAILED
) {
457 $this->take_contentdump($scope);
460 // Abort any open transactions to prevent subsequent tests hanging.
461 // This does the same as abort_all_db_transactions(), but doesn't call error_log() as we don't
462 // want to see a message in the behat output.
463 if (($scope->getTestResult() instanceof \Behat\Behat\Tester\Result\ExecutedStepResult
) &&
464 $scope->getTestResult()->hasException()) {
465 if ($DB && $DB->is_transaction_started()) {
466 $DB->force_transaction_rollback();
471 if (!$this->running_javascript()) {
475 // Save a screenshot if the step failed.
476 if (!empty($CFG->behat_faildump_path
) &&
477 $scope->getTestResult()->getResultCode() === Behat\Testwork\Tester\Result\TestResult
::FAILED
) {
478 $this->take_screenshot($scope);
482 $this->wait_for_pending_js();
483 self
::$currentstepexception = null;
484 } catch (UnexpectedAlertOpen
$e) {
485 self
::$currentstepexception = $e;
487 // Accepting the alert so the framework can continue properly running
488 // the following scenarios. Some browsers already closes the alert, so
489 // wrapping in a try & catch.
491 $this->getSession()->getDriver()->getWebDriverSession()->accept_alert();
492 } catch (Exception
$e) {
493 // Catching the generic one as we never know how drivers reacts here.
495 } catch (Exception
$e) {
496 self
::$currentstepexception = $e;
501 * Executed after scenario having switch window to restart session.
502 * This is needed to close all extra browser windows and starting
503 * one browser window.
505 * @param AfterScenarioScope $scope scope passed by event fired after scenario.
506 * @AfterScenario @_switch_window
508 public function after_scenario_switchwindow(AfterScenarioScope
$scope) {
509 for ($count = 0; $count < self
::EXTENDED_TIMEOUT
; $count++
) {
511 $this->getSession()->restart();
513 } catch (DriverException
$e) {
514 // Wait for timeout and try again.
515 sleep(self
::TIMEOUT
);
518 // If session is not restarted above then it will try to start session before next scenario
519 // and if that fails then exception will be thrown.
523 * Getter for self::$faildumpdirname
527 protected function get_run_faildump_dir() {
528 return self
::$faildumpdirname;
532 * Take screenshot when a step fails.
535 * @param AfterStepScope $scope scope passed by event after step.
537 protected function take_screenshot(AfterStepScope
$scope) {
538 // Goutte can't save screenshots.
539 if (!$this->running_javascript()) {
543 // Some drivers (e.g. chromedriver) may throw an exception while trying to take a screenshot. If this isn't handled,
544 // the behat run dies. We don't want to lose the information about the failure that triggered the screenshot,
545 // so let's log the exception message to a file (to explain why there's no screenshot) and allow the run to continue,
546 // handling the failure as normal.
548 list ($dir, $filename) = $this->get_faildump_filename($scope, 'png');
549 $this->saveScreenshot($filename, $dir);
550 } catch (Exception
$e) {
551 // Catching all exceptions as we don't know what the driver might throw.
552 list ($dir, $filename) = $this->get_faildump_filename($scope, 'txt');
553 $message = "Could not save screenshot due to an error\n" . $e->getMessage();
554 file_put_contents($dir . DIRECTORY_SEPARATOR
. $filename, $message);
559 * Take a dump of the page content when a step fails.
562 * @param AfterStepScope $scope scope passed by event after step.
564 protected function take_contentdump(AfterStepScope
$scope) {
565 list ($dir, $filename) = $this->get_faildump_filename($scope, 'html');
568 // Driver may throw an exception during getContent(), so do it first to avoid getting an empty file.
569 $content = $this->getSession()->getPage()->getContent();
570 } catch (Exception
$e) {
571 // Catching all exceptions as we don't know what the driver might throw.
572 $content = "Could not save contentdump due to an error\n" . $e->getMessage();
574 file_put_contents($dir . DIRECTORY_SEPARATOR
. $filename, $content);
578 * Determine the full pathname to store a failure-related dump.
580 * This is used for content such as the DOM, and screenshots.
582 * @param AfterStepScope $scope scope passed by event after step.
583 * @param String $filetype The file suffix to use. Limited to 4 chars.
585 protected function get_faildump_filename(AfterStepScope
$scope, $filetype) {
588 // All the contentdumps should be in the same parent dir.
589 if (!$faildumpdir = self
::get_run_faildump_dir()) {
590 $faildumpdir = self
::$faildumpdirname = date('Ymd_His');
592 $dir = $CFG->behat_faildump_path
. DIRECTORY_SEPARATOR
. $faildumpdir;
594 if (!is_dir($dir) && !mkdir($dir, $CFG->directorypermissions
, true)) {
595 // It shouldn't, we already checked that the directory is writable.
596 throw new Exception('No directories can be created inside $CFG->behat_faildump_path, check the directory permissions.');
599 // We will always need to know the full path.
600 $dir = $CFG->behat_faildump_path
. DIRECTORY_SEPARATOR
. $faildumpdir;
603 // The scenario title + the failed step text.
604 // We want a i-am-the-scenario-title_i-am-the-failed-step.$filetype format.
605 $filename = $scope->getFeature()->getTitle() . '_' . $scope->getStep()->getText();
607 // As file name is limited to 255 characters. Leaving 5 chars for line number and 4 chars for the file.
608 // extension as we allow .png for images and .html for DOM contents.
611 // Suffix suite name to faildump file, if it's not default suite.
612 $suitename = $scope->getSuite()->getName();
613 if ($suitename != 'default') {
614 $suitename = '_' . $suitename;
615 $filenamelen = $filenamelen - strlen($suitename);
617 // No need to append suite name for default.
621 $filename = preg_replace('/([^a-zA-Z0-9\_]+)/', '-', $filename);
622 $filename = substr($filename, 0, $filenamelen) . $suitename . '_' . $scope->getStep()->getLine() . '.' . $filetype;
624 return array($dir, $filename);
628 * Internal step definition to find exceptions, debugging() messages and PHP debug messages.
630 * Part of behat_hooks class as is part of the testing framework, is auto-executed
631 * after each step so no features will splicitly use it.
633 * @Given /^I look for exceptions$/
634 * @throw Exception Unknown type, depending on what we caught in the hook or basic \Exception.
635 * @see Moodle\BehatExtension\EventDispatcher\Tester\ChainedStepTester
637 public function i_look_for_exceptions() {
638 // If the step already failed in a hook throw the exception.
639 if (!is_null(self
::$currentstepexception)) {
640 throw self
::$currentstepexception;
643 $this->look_for_exceptions();
647 * Returns whether the first scenario of the suite is running
651 protected static function is_first_scenario() {
652 return !(self
::$initprocessesfinished);
657 * Behat stop exception
659 * This exception is thrown from before suite or scenario if any setup problem found.
662 * @copyright 2016 Rajesh Taneja <rajesh@moodle.com>
663 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
665 class behat_stop_exception
extends \Exception
{