MDL-56211 behat: Add suite name to faildump path
[moodle.git] / lib / tests / behat / behat_hooks.php
blob36d92799a2c75a3297f6682ee1f9b75ef9852179
1 <?php
2 // This file is part of Moodle - http://moodle.org/
3 //
4 // Moodle is free software: you can redistribute it and/or modify
5 // it under the terms of the GNU General Public License as published by
6 // the Free Software Foundation, either version 3 of the License, or
7 // (at your option) any later version.
8 //
9 // Moodle is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 // GNU General Public License for more details.
14 // You should have received a copy of the GNU General Public License
15 // along with Moodle. If not, see <http://www.gnu.org/licenses/>.
17 /**
18 * Behat hooks steps definitions.
20 * This methods are used by Behat CLI command.
22 * @package core
23 * @category test
24 * @copyright 2012 David MonllaĆ³
25 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
28 // NOTE: no MOODLE_INTERNAL test here, this file may be required by behat before including /config.php.
30 require_once(__DIR__ . '/../../behat/behat_base.php');
32 use Behat\Testwork\Hook\Scope\BeforeSuiteScope,
33 Behat\Testwork\Hook\Scope\AfterSuiteScope,
34 Behat\Behat\Hook\Scope\BeforeFeatureScope,
35 Behat\Behat\Hook\Scope\AfterFeatureScope,
36 Behat\Behat\Hook\Scope\BeforeScenarioScope,
37 Behat\Behat\Hook\Scope\AfterScenarioScope,
38 Behat\Behat\Hook\Scope\BeforeStepScope,
39 Behat\Behat\Hook\Scope\AfterStepScope,
40 Behat\Mink\Exception\DriverException as DriverException,
41 WebDriver\Exception\NoSuchWindow as NoSuchWindow,
42 WebDriver\Exception\UnexpectedAlertOpen as UnexpectedAlertOpen,
43 WebDriver\Exception\UnknownError as UnknownError,
44 WebDriver\Exception\CurlExec as CurlExec,
45 WebDriver\Exception\NoAlertOpenError as NoAlertOpenError;
47 /**
48 * Hooks to the behat process.
50 * Behat accepts hooks after and before each
51 * suite, feature, scenario and step.
53 * They can not call other steps as part of their process
54 * like regular steps definitions does.
56 * Throws generic Exception because they are captured by Behat.
58 * @package core
59 * @category test
60 * @copyright 2012 David MonllaĆ³
61 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
63 class behat_hooks extends behat_base {
65 /**
66 * @var Last browser session start time.
68 protected static $lastbrowsersessionstart = 0;
70 /**
71 * @var For actions that should only run once.
73 protected static $initprocessesfinished = false;
75 /**
76 * Some exceptions can only be caught in a before or after step hook,
77 * they can not be thrown there as they will provoke a framework level
78 * failure, but we can store them here to fail the step in i_look_for_exceptions()
79 * which result will be parsed by the framework as the last step result.
81 * @var Null or the exception last step throw in the before or after hook.
83 protected static $currentstepexception = null;
85 /**
86 * If we are saving any kind of dump on failure we should use the same parent dir during a run.
88 * @var The parent dir name
90 protected static $faildumpdirname = false;
92 /**
93 * Keeps track of time taken by feature to execute.
95 * @var array list of feature timings
97 protected static $timings = array();
99 /**
100 * Keeps track of current running suite name.
102 * @var string current running suite name
104 protected static $runningsuite = '';
107 * Hook to capture BeforeSuite event so as to give access to moodle codebase.
108 * This will try and catch any exception and exists if anything fails.
110 * @param BeforeSuiteScope $scope scope passed by event fired before suite.
111 * @BeforeSuite
113 public static function before_suite_hook(BeforeSuiteScope $scope) {
114 // If behat has been initialised then no need to do this again.
115 if (self::$initprocessesfinished) {
116 return;
119 try {
120 self::before_suite($scope);
121 } catch (behat_stop_exception $e) {
122 echo $e->getMessage() . PHP_EOL;
123 exit(1);
128 * Gives access to moodle codebase, ensures all is ready and sets up the test lock.
130 * Includes config.php to use moodle codebase with $CFG->behat_*
131 * instead of $CFG->prefix and $CFG->dataroot, called once per suite.
133 * @param BeforeSuiteScope $scope scope passed by event fired before suite.
134 * @static
135 * @throws behat_stop_exception
137 public static function before_suite(BeforeSuiteScope $scope) {
138 global $CFG;
140 // Defined only when the behat CLI command is running, the moodle init setup process will
141 // read this value and switch to $CFG->behat_dataroot and $CFG->behat_prefix instead of
142 // the normal site.
143 if (!defined('BEHAT_TEST')) {
144 define('BEHAT_TEST', 1);
147 if (!defined('CLI_SCRIPT')) {
148 define('CLI_SCRIPT', 1);
151 // With BEHAT_TEST we will be using $CFG->behat_* instead of $CFG->dataroot, $CFG->prefix and $CFG->wwwroot.
152 require_once(__DIR__ . '/../../../config.php');
154 // Now that we are MOODLE_INTERNAL.
155 require_once(__DIR__ . '/../../behat/classes/behat_command.php');
156 require_once(__DIR__ . '/../../behat/classes/behat_selectors.php');
157 require_once(__DIR__ . '/../../behat/classes/behat_context_helper.php');
158 require_once(__DIR__ . '/../../behat/classes/util.php');
159 require_once(__DIR__ . '/../../testing/classes/test_lock.php');
160 require_once(__DIR__ . '/../../testing/classes/nasty_strings.php');
162 // Avoids vendor/bin/behat to be executed directly without test environment enabled
163 // to prevent undesired db & dataroot modifications, this is also checked
164 // before each scenario (accidental user deletes) in the BeforeScenario hook.
166 if (!behat_util::is_test_mode_enabled()) {
167 throw new behat_stop_exception('Behat only can run if test mode is enabled. More info in ' .
168 behat_command::DOCS_URL . '#Running_tests');
171 // Reset all data, before checking for check_server_status.
172 // If not done, then it can return apache error, while running tests.
173 behat_util::clean_tables_updated_by_scenario_list();
174 behat_util::reset_all_data();
176 // Check if server is running and using same version for cli and apache.
177 behat_util::check_server_status();
179 // Prevents using outdated data, upgrade script would start and tests would fail.
180 if (!behat_util::is_test_data_updated()) {
181 $commandpath = 'php admin/tool/behat/cli/init.php';
182 throw new behat_stop_exception("Your behat test site is outdated, please run\n\n " .
183 $commandpath . "\n\nfrom your moodle dirroot to drop and install the behat test site again.");
185 // Avoid parallel tests execution, it continues when the previous lock is released.
186 test_lock::acquire('behat');
188 // Store the browser reset time if reset after N seconds is specified in config.php.
189 if (!empty($CFG->behat_restart_browser_after)) {
190 // Store the initial browser session opening.
191 self::$lastbrowsersessionstart = time();
194 if (!empty($CFG->behat_faildump_path) && !is_writable($CFG->behat_faildump_path)) {
195 throw new behat_stop_exception('You set $CFG->behat_faildump_path to a non-writable directory');
198 // Handle interrupts on PHP7.
199 if (extension_loaded('pcntl')) {
200 $disabled = explode(',', ini_get('disable_functions'));
201 if (!in_array('pcntl_signal', $disabled)) {
202 declare(ticks = 1);
208 * Gives access to moodle codebase, to keep track of feature start time.
210 * @param BeforeFeatureScope $scope scope passed by event fired before feature.
211 * @BeforeFeature
213 public static function before_feature(BeforeFeatureScope $scope) {
214 if (!defined('BEHAT_FEATURE_TIMING_FILE')) {
215 return;
217 $file = $scope->getFeature()->getFile();
218 self::$timings[$file] = microtime(true);
222 * Gives access to moodle codebase, to keep track of feature end time.
224 * @param AfterFeatureScope $scope scope passed by event fired after feature.
225 * @AfterFeature
227 public static function after_feature(AfterFeatureScope $scope) {
228 if (!defined('BEHAT_FEATURE_TIMING_FILE')) {
229 return;
231 $file = $scope->getFeature()->getFile();
232 self::$timings[$file] = microtime(true) - self::$timings[$file];
233 // Probably didn't actually run this, don't output it.
234 if (self::$timings[$file] < 1) {
235 unset(self::$timings[$file]);
240 * Gives access to moodle codebase, to keep track of suite timings.
242 * @param AfterSuiteScope $scope scope passed by event fired after suite.
243 * @AfterSuite
245 public static function after_suite(AfterSuiteScope $scope) {
246 if (!defined('BEHAT_FEATURE_TIMING_FILE')) {
247 return;
249 $realroot = realpath(__DIR__.'/../../../').'/';
250 foreach (self::$timings as $k => $v) {
251 $new = str_replace($realroot, '', $k);
252 self::$timings[$new] = round($v, 1);
253 unset(self::$timings[$k]);
255 if ($existing = @json_decode(file_get_contents(BEHAT_FEATURE_TIMING_FILE), true)) {
256 self::$timings = array_merge($existing, self::$timings);
258 arsort(self::$timings);
259 @file_put_contents(BEHAT_FEATURE_TIMING_FILE, json_encode(self::$timings, JSON_PRETTY_PRINT));
263 * Hook to capture before scenario event to get scope.
265 * @param BeforeScenarioScope $scope scope passed by event fired before scenario.
266 * @BeforeScenario
268 public function before_scenario_hook(BeforeScenarioScope $scope) {
269 try {
270 $this->before_scenario($scope);
271 } catch (behat_stop_exception $e) {
272 echo $e->getMessage() . PHP_EOL;
273 exit(1);
278 * Resets the test environment.
280 * @param BeforeScenarioScope $scope scope passed by event fired before scenario.
281 * @throws behat_stop_exception If here we are not using the test database it should be because of a coding error
283 public function before_scenario(BeforeScenarioScope $scope) {
284 global $DB, $CFG;
286 // As many checks as we can.
287 if (!defined('BEHAT_TEST') ||
288 !defined('BEHAT_SITE_RUNNING') ||
289 php_sapi_name() != 'cli' ||
290 !behat_util::is_test_mode_enabled() ||
291 !behat_util::is_test_site()) {
292 throw new behat_stop_exception('Behat only can modify the test database and the test dataroot!');
295 $moreinfo = 'More info in ' . behat_command::DOCS_URL . '#Running_tests';
296 $driverexceptionmsg = 'Selenium server is not running, you need to start it to run tests that involve Javascript. ' . $moreinfo;
297 try {
298 $session = $this->getSession();
299 } catch (CurlExec $e) {
300 // Exception thrown by WebDriver, so only @javascript tests will be caugth; in
301 // behat_util::check_server_status() we already checked that the server is running.
302 throw new behat_stop_exception($driverexceptionmsg);
303 } catch (DriverException $e) {
304 throw new behat_stop_exception($driverexceptionmsg);
305 } catch (UnknownError $e) {
306 // Generic 'I have no idea' Selenium error. Custom exception to provide more feedback about possible solutions.
307 throw new behat_stop_exception($e->getMessage());
310 $suitename = $scope->getSuite()->getName();
312 // Register behat selectors for theme, if suite is changed. We do it for every suite change.
313 if ($suitename !== self::$runningsuite) {
314 behat_context_helper::set_environment($scope->getEnvironment());
316 // We need the Mink session to do it and we do it only before the first scenario.
317 $namedpartialclass = 'behat_partial_named_selector';
318 $namedexactclass = 'behat_exact_named_selector';
320 // If override selector exist, then set it as default behat selectors class.
321 $overrideclass = behat_config_util::get_behat_theme_selector_override_classname($suitename, 'named_partial', true);
322 if (class_exists($overrideclass)) {
323 $namedpartialclass = $overrideclass;
326 // If override selector exist, then set it as default behat selectors class.
327 $overrideclass = behat_config_util::get_behat_theme_selector_override_classname($suitename, 'named_exact', true);
328 if (class_exists($overrideclass)) {
329 $namedexactclass = $overrideclass;
332 $this->getSession()->getSelectorsHandler()->registerSelector('named_partial', new $namedpartialclass());
333 $this->getSession()->getSelectorsHandler()->registerSelector('named_exact', new $namedexactclass());
336 // Reset mink session between the scenarios.
337 $session->reset();
339 // Reset $SESSION.
340 \core\session\manager::init_empty_session();
342 // Ignore E_NOTICE and E_WARNING during reset, as this might be caused because of some existing process
343 // running ajax. This will be investigated in another issue.
344 $errorlevel = error_reporting();
345 error_reporting($errorlevel & ~E_NOTICE & ~E_WARNING);
346 behat_util::reset_all_data();
347 error_reporting($errorlevel);
349 // Assign valid data to admin user (some generator-related code needs a valid user).
350 $user = $DB->get_record('user', array('username' => 'admin'));
351 \core\session\manager::set_user($user);
353 // Reset the browser if specified in config.php.
354 if (!empty($CFG->behat_restart_browser_after) && $this->running_javascript()) {
355 $now = time();
356 if (self::$lastbrowsersessionstart + $CFG->behat_restart_browser_after < $now) {
357 $session->restart();
358 self::$lastbrowsersessionstart = $now;
362 // Set the theme if not default.
363 if ($suitename !== "default") {
364 set_config('theme', $suitename);
365 self::$runningsuite = $suitename;
368 // Start always in the the homepage.
369 try {
370 // Let's be conservative as we never know when new upstream issues will affect us.
371 $session->visit($this->locate_path('/'));
372 } catch (UnknownError $e) {
373 throw new behat_stop_exception($e->getMessage());
376 // Checking that the root path is a Moodle test site.
377 if (self::is_first_scenario()) {
378 $notestsiteexception = new behat_stop_exception('The base URL (' . $CFG->wwwroot . ') is not a behat test site, ' .
379 'ensure you started the built-in web server in the correct directory or your web server is correctly started and set up');
380 $this->find("xpath", "//head/child::title[normalize-space(.)='" . behat_util::BEHATSITENAME . "']", $notestsiteexception);
382 self::$initprocessesfinished = true;
385 // Run all test with medium (1024x768) screen size, to avoid responsive problems.
386 $this->resize_window('medium');
390 * Executed after scenario to go to a page where no JS is executed.
391 * This will ensure there are no unwanted ajax calls from browser and
392 * site can be reset safely.
394 * @param AfterScenarioScope $scope scope passed by event fired after scenario.
395 * @AfterScenario
397 public function after_scenario(AfterScenarioScope $scope) {
398 try {
399 $this->wait_for_pending_js();
400 $this->getSession()->reset();
401 } catch (DriverException $e) {
402 // Try restart session, if DriverException caught.
403 try {
404 $this->getSession()->restart();
405 } catch (DriverException $e) {
406 // Do nothing, as this will be caught while starting session in before_scenario.
412 * Wait for JS to complete before beginning interacting with the DOM.
414 * Executed only when running against a real browser. We wrap it
415 * all in a try & catch to forward the exception to i_look_for_exceptions
416 * so the exception will be at scenario level, which causes a failure, by
417 * default would be at framework level, which will stop the execution of
418 * the run.
420 * @param BeforeStepScope $scope scope passed by event fired before step.
421 * @BeforeStep
423 public function before_step_javascript(BeforeStepScope $scope) {
424 self::$currentstepexception = null;
426 // Only run if JS.
427 if ($this->running_javascript()) {
428 try {
429 $this->wait_for_pending_js();
430 } catch (Exception $e) {
431 self::$currentstepexception = $e;
437 * Wait for JS to complete after finishing the step.
439 * With this we ensure that there are not AJAX calls
440 * still in progress.
442 * Executed only when running against a real browser. We wrap it
443 * all in a try & catch to forward the exception to i_look_for_exceptions
444 * so the exception will be at scenario level, which causes a failure, by
445 * default would be at framework level, which will stop the execution of
446 * the run.
448 * @param AfterStepScope $scope scope passed by event fired after step..
449 * @AfterStep
451 public function after_step_javascript(AfterStepScope $scope) {
452 global $CFG, $DB;
454 // Save the page content if the step failed.
455 if (!empty($CFG->behat_faildump_path) &&
456 $scope->getTestResult()->getResultCode() === Behat\Testwork\Tester\Result\TestResult::FAILED) {
457 $this->take_contentdump($scope);
460 // Abort any open transactions to prevent subsequent tests hanging.
461 // This does the same as abort_all_db_transactions(), but doesn't call error_log() as we don't
462 // want to see a message in the behat output.
463 if (($scope->getTestResult() instanceof \Behat\Behat\Tester\Result\ExecutedStepResult) &&
464 $scope->getTestResult()->hasException()) {
465 if ($DB && $DB->is_transaction_started()) {
466 $DB->force_transaction_rollback();
470 // Only run if JS.
471 if (!$this->running_javascript()) {
472 return;
475 // Save a screenshot if the step failed.
476 if (!empty($CFG->behat_faildump_path) &&
477 $scope->getTestResult()->getResultCode() === Behat\Testwork\Tester\Result\TestResult::FAILED) {
478 $this->take_screenshot($scope);
481 try {
482 $this->wait_for_pending_js();
483 self::$currentstepexception = null;
484 } catch (UnexpectedAlertOpen $e) {
485 self::$currentstepexception = $e;
487 // Accepting the alert so the framework can continue properly running
488 // the following scenarios. Some browsers already closes the alert, so
489 // wrapping in a try & catch.
490 try {
491 $this->getSession()->getDriver()->getWebDriverSession()->accept_alert();
492 } catch (Exception $e) {
493 // Catching the generic one as we never know how drivers reacts here.
495 } catch (Exception $e) {
496 self::$currentstepexception = $e;
501 * Executed after scenario having switch window to restart session.
502 * This is needed to close all extra browser windows and starting
503 * one browser window.
505 * @param AfterScenarioScope $scope scope passed by event fired after scenario.
506 * @AfterScenario @_switch_window
508 public function after_scenario_switchwindow(AfterScenarioScope $scope) {
509 for ($count = 0; $count < self::EXTENDED_TIMEOUT; $count++) {
510 try {
511 $this->getSession()->restart();
512 break;
513 } catch (DriverException $e) {
514 // Wait for timeout and try again.
515 sleep(self::TIMEOUT);
518 // If session is not restarted above then it will try to start session before next scenario
519 // and if that fails then exception will be thrown.
523 * Getter for self::$faildumpdirname
525 * @return string
527 protected function get_run_faildump_dir() {
528 return self::$faildumpdirname;
532 * Take screenshot when a step fails.
534 * @throws Exception
535 * @param AfterStepScope $scope scope passed by event after step.
537 protected function take_screenshot(AfterStepScope $scope) {
538 // Goutte can't save screenshots.
539 if (!$this->running_javascript()) {
540 return false;
543 // Some drivers (e.g. chromedriver) may throw an exception while trying to take a screenshot. If this isn't handled,
544 // the behat run dies. We don't want to lose the information about the failure that triggered the screenshot,
545 // so let's log the exception message to a file (to explain why there's no screenshot) and allow the run to continue,
546 // handling the failure as normal.
547 try {
548 list ($dir, $filename) = $this->get_faildump_filename($scope, 'png');
549 $this->saveScreenshot($filename, $dir);
550 } catch (Exception $e) {
551 // Catching all exceptions as we don't know what the driver might throw.
552 list ($dir, $filename) = $this->get_faildump_filename($scope, 'txt');
553 $message = "Could not save screenshot due to an error\n" . $e->getMessage();
554 file_put_contents($dir . DIRECTORY_SEPARATOR . $filename, $message);
559 * Take a dump of the page content when a step fails.
561 * @throws Exception
562 * @param AfterStepScope $scope scope passed by event after step.
564 protected function take_contentdump(AfterStepScope $scope) {
565 list ($dir, $filename) = $this->get_faildump_filename($scope, 'html');
567 try {
568 // Driver may throw an exception during getContent(), so do it first to avoid getting an empty file.
569 $content = $this->getSession()->getPage()->getContent();
570 } catch (Exception $e) {
571 // Catching all exceptions as we don't know what the driver might throw.
572 $content = "Could not save contentdump due to an error\n" . $e->getMessage();
574 file_put_contents($dir . DIRECTORY_SEPARATOR . $filename, $content);
578 * Determine the full pathname to store a failure-related dump.
580 * This is used for content such as the DOM, and screenshots.
582 * @param AfterStepScope $scope scope passed by event after step.
583 * @param String $filetype The file suffix to use. Limited to 4 chars.
585 protected function get_faildump_filename(AfterStepScope $scope, $filetype) {
586 global $CFG;
588 // All the contentdumps should be in the same parent dir.
589 if (!$faildumpdir = self::get_run_faildump_dir()) {
590 $faildumpdir = self::$faildumpdirname = date('Ymd_His');
592 $dir = $CFG->behat_faildump_path . DIRECTORY_SEPARATOR . $faildumpdir;
594 if (!is_dir($dir) && !mkdir($dir, $CFG->directorypermissions, true)) {
595 // It shouldn't, we already checked that the directory is writable.
596 throw new Exception('No directories can be created inside $CFG->behat_faildump_path, check the directory permissions.');
598 } else {
599 // We will always need to know the full path.
600 $dir = $CFG->behat_faildump_path . DIRECTORY_SEPARATOR . $faildumpdir;
603 // The scenario title + the failed step text.
604 // We want a i-am-the-scenario-title_i-am-the-failed-step.$filetype format.
605 $filename = $scope->getFeature()->getTitle() . '_' . $scope->getStep()->getText();
607 // As file name is limited to 255 characters. Leaving 5 chars for line number and 4 chars for the file.
608 // extension as we allow .png for images and .html for DOM contents.
609 $filenamelen = 245;
611 // Suffix suite name to faildump file, if it's not default suite.
612 $suitename = $scope->getSuite()->getName();
613 if ($suitename != 'default') {
614 $suitename = '_' . $suitename;
615 $filenamelen = $filenamelen - strlen($suitename);
616 } else {
617 // No need to append suite name for default.
618 $suitename = '';
621 $filename = preg_replace('/([^a-zA-Z0-9\_]+)/', '-', $filename);
622 $filename = substr($filename, 0, $filenamelen) . $suitename . '_' . $scope->getStep()->getLine() . '.' . $filetype;
624 return array($dir, $filename);
628 * Internal step definition to find exceptions, debugging() messages and PHP debug messages.
630 * Part of behat_hooks class as is part of the testing framework, is auto-executed
631 * after each step so no features will splicitly use it.
633 * @Given /^I look for exceptions$/
634 * @throw Exception Unknown type, depending on what we caught in the hook or basic \Exception.
635 * @see Moodle\BehatExtension\EventDispatcher\Tester\ChainedStepTester
637 public function i_look_for_exceptions() {
638 // If the step already failed in a hook throw the exception.
639 if (!is_null(self::$currentstepexception)) {
640 throw self::$currentstepexception;
643 $this->look_for_exceptions();
647 * Returns whether the first scenario of the suite is running
649 * @return bool
651 protected static function is_first_scenario() {
652 return !(self::$initprocessesfinished);
657 * Behat stop exception
659 * This exception is thrown from before suite or scenario if any setup problem found.
661 * @package core_test
662 * @copyright 2016 Rajesh Taneja <rajesh@moodle.com>
663 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
665 class behat_stop_exception extends \Exception {