MDL-54628 behat: Show parallel run command with proper rerun option

[moodle.git] / admin / tool / behat / cli / run.php

<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.

/**
 * Wrapper to run previously set-up behat tests in parallel.
 *
 * @package    tool_behat
 * @copyright  2014 NetSpot Pty Ltd
 * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
 */

if (isset($_SERVER['REMOTE_ADDR'])) {
    die(); // No access from web!
}

define('CLI_SCRIPT', true);
define('ABORT_AFTER_CONFIG', true);
define('CACHE_DISABLE_ALL', true);
define('NO_OUTPUT_BUFFERING', true);

require_once(__DIR__ .'/../../../../config.php');
require_once(__DIR__.'/../../../../lib/clilib.php');
require_once(__DIR__.'/../../../../lib/behat/lib.php');
require_once(__DIR__.'/../../../../lib/behat/classes/behat_command.php');
require_once(__DIR__.'/../../../../lib/behat/classes/behat_config_manager.php');

error_reporting(E_ALL | E_STRICT);
ini_set('display_errors', '1');
ini_set('log_errors', '1');

list($options, $unrecognised) = cli_get_params(
    array(
        'stop-on-failure' => 0,
        'verbose'  => false,
        'replace'  => false,
        'help'     => false,
        'tags'     => '',
        'profile'  => '',
        'feature'  => '',
        'suite'    => '',
        'fromrun'  => 1,
        'torun'    => 0,
        'single-run' => false,
        'rerun' => 0,
    ),
    array(
        'h' => 'help',
        't' => 'tags',
        'p' => 'profile',
        's' => 'single-run',
    )
);

// Checking run.php CLI script usage.
$help = "
Behat utilities to run behat tests in parallel

Usage:
  php run.php [--BEHAT_OPTION=\"value\"] [--feature=\"value\"] [--replace] [--fromrun=value --torun=value] [--help]

Options:
--BEHAT_OPTION     Any combination of behat option specified in http://behat.readthedocs.org/en/v2.5/guides/6.cli.html
--feature          Only execute specified feature file (Absolute path of feature file).
--suite            Specified theme scenarios will be executed.
--replace          Replace args string with run process number, useful for output.
--fromrun          Execute run starting from (Used for parallel runs on different vms)
--torun            Execute run till (Used for parallel runs on different vms)
--rerun            Re-run scenarios that failed during last execution.

-h, --help         Print out this help

Example from Moodle root directory:
\$ php admin/tool/behat/cli/run.php --tags=\"@javascript\"

More info in http://docs.moodle.org/dev/Acceptance_testing#Running_tests
";

if (!empty($options['help'])) {
    echo $help;
    exit(0);
}

$parallelrun = behat_config_manager::get_parallel_test_runs($options['fromrun']);

// Default torun is maximum parallel runs.
if (empty($options['torun'])) {
    $options['torun'] = $parallelrun;
}

// Capture signals and ensure we clean symlinks.
if (extension_loaded('pcntl')) {
    $disabled = explode(',', ini_get('disable_functions'));
    if (!in_array('pcntl_signal', $disabled)) {
        pcntl_signal(SIGTERM, "signal_handler");
        pcntl_signal(SIGINT, "signal_handler");
    }
}

$time = microtime(true);
array_walk($unrecognised, function (&$v) {
    if ($x = preg_filter("#^(-+\w+)=(.+)#", "\$1=\"\$2\"", $v)) {
        $v = $x;
    } else if (!preg_match("#^-#", $v)) {
        $v = escapeshellarg($v);
    }
});
$extraopts = $unrecognised;

if ($options['profile']) {
    $profile = $options['profile'];

    // If profile passed is not set, then exit.
    if (!isset($CFG->behat_config[$profile]) && !isset($CFG->behat_profiles[$profile]) &&
        !(isset($options['replace']) && (strpos($options['profile'], $options['replace']) >= 0 ))) {
        echo "Invalid profile passed: " . $profile . PHP_EOL;
        exit(1);
    }

    $extraopts['profile'] = '--profile="' . $profile . '"';
    // By default, profile tags will be used.
    if (!empty($CFG->behat_config[$profile]['filters']['tags'])) {
        $tags = $CFG->behat_config[$profile]['filters']['tags'];
    }
}

// Command line tags have precedence (std behat behavior).
if ($options['tags']) {
    $tags = $options['tags'];
    $extraopts['tags'] = '--tags="' . $tags . '"';
}

// Add suite option if specified.
if ($options['suite']) {
    $extraopts['suite'] = '--suite="' . $options['suite'] . '"';
}

// Feature should be added to last, for behat command.
if ($options['feature']) {
    $extraopts['feature'] = $options['feature'];
    // Only run 1 process as process.
    // Feature file is picked from absolute path provided, so no need to check for behat.yml.
    $options['torun'] = $options['fromrun'];
}

// Set of options to pass to behat.
$extraoptstr = implode(' ', $extraopts);

// If empty parallelrun then just check with user if it's a run single behat test.
if (empty($parallelrun)) {
    $cwd = getcwd();
    chdir(__DIR__);
    $runtestscommand = behat_command::get_behat_command(false, false, true);
    $runtestscommand .= ' --config ' . behat_config_manager::get_behat_cli_config_filepath();
    $runtestscommand .= ' ' . $extraoptstr;
    echo "Running single behat site:" . PHP_EOL;
    passthru("php $runtestscommand", $code);
    chdir($cwd);
    exit($code);
}

// If rerun is passed then ensure we just run the failed processes.
$lastfailedstatus = 0;
if ($options['rerun']) {
    $lastfailedstatus = get_last_failed_status($options['fromrun'], $options['torun']);
    unset($options['rerun']);
    $extraoptstr .= ' --rerun';
}

$cmds = array();
echo "Running " . ($options['torun'] - $options['fromrun'] + 1) . " parallel behat sites:" . PHP_EOL;

for ($i = $options['fromrun']; $i <= $options['torun']; $i++) {
    // Bypass if not failed in last run.
    if ($lastfailedstatus && !($i & $lastfailedstatus)) {
        continue;
    }

    $CFG->behatrunprocess = $i;

    // Options parameters to be added to each run.
    $myopts = !empty($options['replace']) ? str_replace($options['replace'], $i, $extraoptstr) : $extraoptstr;

    $behatcommand = behat_command::get_behat_command(false, false, true);
    $behatconfigpath = behat_config_manager::get_behat_cli_config_filepath($i);

    // Command to execute behat run.
    $cmds[BEHAT_PARALLEL_SITE_NAME . $i] = $behatcommand . ' --config ' . $behatconfigpath . " " . $myopts;
    echo "[" . BEHAT_PARALLEL_SITE_NAME . $i . "] " . $cmds[BEHAT_PARALLEL_SITE_NAME . $i] . PHP_EOL;

    // Remove any old last failed status files.
    $filepath = behat_command::get_last_failed_test_status_file_path($options['fromrun']);
    if (file_exists($filepath)) {
        @unlink($filepath);
    }
}

if (empty($cmds)) {
    echo "No commands to execute " . PHP_EOL;
    exit(1);
}

// Create site symlink if necessary.
if (!behat_config_manager::create_parallel_site_links($options['fromrun'], $options['torun'])) {
    echo "Check permissions. If on windows, make sure you are running this command as admin" . PHP_EOL;
    exit(1);
}

// Execute all commands, relative to moodle root directory.
$processes = cli_execute_parallel($cmds, __DIR__ . "/../../../../");
$stoponfail = empty($options['stop-on-failure']) ? false : true;

// Print header.
print_process_start_info($processes);

// Print combined run o/p from processes.
$exitcodes = print_combined_run_output($processes, $stoponfail);
$time = round(microtime(true) - $time, 1);
echo "Finished in " . gmdate("G\h i\m s\s", $time) . PHP_EOL . PHP_EOL;

ksort($exitcodes);

// Print exit info from each run.
// Status bits contains pass/fail status of parallel runs.
$status = 0;
foreach ($exitcodes as $name => $exitcode) {
    if ($exitcode) {
        $runno = str_replace(BEHAT_PARALLEL_SITE_NAME, '', $name);
        $status |= (1 << ($runno - 1));
    }
}

// Run finished. Show exit code and output from individual process.
$verbose = empty($options['verbose']) ? false : true;

// Show exit code from each process, if any process failed.
if ($verbose || $status) {
    // Echo exit codes.
    echo "Exit codes for each behat run: " . PHP_EOL;
    foreach ($exitcodes as $run => $exitcode) {
        echo $run . ": " . $exitcode . PHP_EOL;
    }

    // Show failed re-run commands.
    if ($status) {
        echo "To re-run failed processes, you can use following commands:" . PHP_EOL;
        foreach ($cmds as $name => $cmd) {
            if (!empty($exitcodes[$name])) {
                // Show rerun command only for the failed runs.
                $runno = str_replace(BEHAT_PARALLEL_SITE_NAME, '', $name);
                if ((1 << ($runno - 1)) & $status) {
                    $extraopts['fromrun'] = '--fromrun=' . $runno;
                    $extraopts['torun'] = '--torun=' . $runno;
                    $extraopts['rerun'] = '--rerun';
                    $extraoptstr = implode(' ', $extraopts);

                    $myopts = !empty($options['replace']) ? str_replace($options['replace'], $runno, $extraoptstr) : $extraoptstr;

                    $behatcommand = behat_command::get_behat_command(true, true, true);
                    echo "[" . $name . "] " . $behatcommand . ' ' . $myopts . PHP_EOL;
                }

                // Save information about this failure.
                $filepath = behat_command::get_last_failed_test_status_file_path($runno);
                if (!file_put_contents($filepath, $status)) {
                    behat_error(BEHAT_EXITCODE_PERMISSIONS, 'File ' . $filepath . ' can not be created');
                }
            }
        }
    }
    echo PHP_EOL;
}

print_each_process_info($processes, $verbose, $status);

// Remove site symlink if necessary.
behat_config_manager::drop_parallel_site_links();

exit($status);

/**
 * Signal handler for terminal exit.
 *
 * @param int $signal signal number.
 */
function signal_handler($signal) {
    switch ($signal) {
        case SIGTERM:
        case SIGKILL:
        case SIGINT:
            // Remove site symlink if necessary.
            behat_config_manager::drop_parallel_site_links();
            exit(1);
    }
}

/**
 * Prints header from the first process.
 *
 * @param array $processes list of processes to loop though.
 */
function print_process_start_info($processes) {
    $printed = false;
    // Keep looping though processes, till we get first process o/p.
    while (!$printed) {
        usleep(10000);
        foreach ($processes as $name => $process) {
            // Exit if any process has stopped.
            if (!$process->isRunning()) {
                $printed = true;
                break;
            }

            $op = explode(PHP_EOL, $process->getOutput());
            if (count($op) >= 3) {
                foreach ($op as $line) {
                    if (trim($line) && (strpos($line, '.') !== 0)) {
                        echo $line . PHP_EOL;
                    }
                }
                $printed = true;
            }
        }
    }
}

/**
 * Loop though all processes and print combined o/p
 *
 * @param array $processes list of processes to loop though.
 * @param bool $stoponfail Stop all processes and exit if failed.
 * @return array list of exit codes from all processes.
 */
function print_combined_run_output($processes, $stoponfail = false) {
    $exitcodes = array();
    $maxdotsonline = 70;
    $remainingprintlen = $maxdotsonline;
    $progresscount = 0;
    while (count($exitcodes) != count($processes)) {
        usleep(10000);
        foreach ($processes as $name => $process) {
            if ($process->isRunning()) {
                $op = $process->getIncrementalOutput();
                if (trim($op)) {
                    $update = preg_filter('#^\s*([FS\.\-]+)(?:\s+\d+)?\s*$#', '$1', $op);
                    // Exit process if anything fails.
                    if ($stoponfail && (strpos($update, 'F') !== false)) {
                        $process->stop(0);
                    }

                    $strlentoprint = strlen($update);

                    // If not enough dots printed on line then just print.
                    if ($strlentoprint < $remainingprintlen) {
                        echo $update;
                        $remainingprintlen = $remainingprintlen - $strlentoprint;
                    } else if ($strlentoprint == $remainingprintlen) {
                        $progresscount += $maxdotsonline;
                        echo $update ." " . $progresscount . PHP_EOL;
                        $remainingprintlen = $maxdotsonline;
                    } else {
                        while ($part = substr($update, 0, $remainingprintlen) > 0) {
                            $progresscount += $maxdotsonline;
                            echo $part . " " . $progresscount . PHP_EOL;
                            $update = substr($update, $remainingprintlen);
                            $remainingprintlen = $maxdotsonline;
                        }
                    }
                }
            } else {
                $exitcodes[$name] = $process->getExitCode();
                if ($stoponfail && ($exitcodes[$name] != 0)) {
                    foreach ($processes as $l => $p) {
                        $exitcodes[$l] = -1;
                        $process->stop(0);
                    }
                }
            }
        }
    }

    echo PHP_EOL;
    return $exitcodes;
}

/**
 * Loop though all processes and print combined o/p
 *
 * @param array $processes list of processes to loop though.
 * @param bool $verbose Show verbose output for each process.
 */
function print_each_process_info($processes, $verbose = false, $status = 0) {
    foreach ($processes as $name => $process) {
        echo "**************** [" . $name . "] ****************" . PHP_EOL;
        if ($verbose) {
            echo $process->getOutput();
            echo $process->getErrorOutput();

        } else if ($status) {
            // Only show failed o/p.
            $runno = str_replace(BEHAT_PARALLEL_SITE_NAME, '', $name);
            if ((1 << ($runno - 1)) & $status) {
                echo $process->getOutput();
                echo $process->getErrorOutput();
            } else {
                echo get_status_lines_from_run_op($process);
            }

        } else {
            echo get_status_lines_from_run_op($process);
        }
        echo PHP_EOL;
    }
}

/**
 * Extract status information from behat o/p and return.
 * @param Symfony\Component\Process\Process $process
 * @return string
 */
function get_status_lines_from_run_op(Symfony\Component\Process\Process $process) {
    $statusstr = '';
    $op = explode(PHP_EOL, $process->getOutput());
    foreach ($op as $line) {
        // Don't print progress .
        if (trim($line) && (strpos($line, '.') !== 0) && (strpos($line, 'Moodle ') !== 0) &&
            (strpos($line, 'Server OS ') !== 0) && (strpos($line, 'Started at ') !== 0) &&
            (strpos($line, 'Browser specific fixes ') !== 0)) {
            $statusstr .= $line . PHP_EOL;
        }
    }

    return $statusstr;
}

/**
 * Return last failed status of parallel runs.
 *
 * @param int $fromrun starting run.
 * @param int $torun end run.
 * @return int status of last failure.
 */
function get_last_failed_status($fromrun, $torun) {
    $lastfailedstatus = 0;

    for ($i = $fromrun; $i <= $torun; $i++) {
        $filepath = behat_command::get_last_failed_test_status_file_path($i);
        if (file_exists($filepath)) {
            if ($lastfailedstatus = file_get_contents($filepath)) {
                $lastfailedstatus = (int)$lastfailedstatus;
                break;
            }
        }
    }

    return $lastfailedstatus;
}