1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
3 * You can obtain one at http://mozilla.org/MPL/2.0/. */
7 const { Preferences } = ChromeUtils.import(
8 "resource://gre/modules/Preferences.jsm"
10 const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
11 const { XPCOMUtils } = ChromeUtils.import(
12 "resource://gre/modules/XPCOMUtils.jsm"
15 const { assert } = ChromeUtils.import("chrome://marionette/content/assert.js");
16 const { capture } = ChromeUtils.import(
17 "chrome://marionette/content/capture.js"
19 const { InvalidArgumentError } = ChromeUtils.import(
20 "chrome://marionette/content/error.js"
22 const { Log } = ChromeUtils.import("chrome://marionette/content/log.js");
24 XPCOMUtils.defineLazyGetter(this, "logger", Log.get);
26 this.EXPORTED_SYMBOLS = ["reftest"];
28 const XUL_NS = "http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul";
29 const PREF_E10S = "browser.tabs.remote.autostart";
31 const SCREENSHOT_MODE = {
44 const DEFAULT_REFTEST_WIDTH = 600;
45 const DEFAULT_REFTEST_HEIGHT = 600;
48 * Implements an fast runner for web-platform-tests format reftests
49 * c.f. http://web-platform-tests.org/writing-tests/reftests.html.
59 reftest.Runner = class {
62 this.canvasCache = new DefaultMap(undefined, () => new Map([[null, []]]));
63 this.windowUtils = null;
65 this.remote = Preferences.get(PREF_E10S);
69 * Setup the required environment for running reftests.
71 * This will open a non-browser window in which the tests will
72 * be loaded, and set up various caches for the reftest run.
74 * @param {Object.<Number>} urlCount
75 * Object holding a map of URL: number of times the URL
76 * will be opened during the reftest run, where that's
78 * @param {string} screenshotMode
79 * String enum representing when screenshots should be taken
81 setup(urlCount, screenshotMode) {
82 this.parentWindow = assert.open(this.driver.getCurrentWindow());
85 SCREENSHOT_MODE[screenshotMode] || SCREENSHOT_MODE.unexpected;
87 this.urlCount = Object.keys(urlCount || {}).reduce(
88 (map, key) => map.set(key, urlCount[key]),
93 async ensureWindow(timeout, width, height) {
94 logger.debug(`ensuring we have a window ${width}x${height}`);
96 if (this.reftestWin && !this.reftestWin.closed) {
97 let browserRect = this.reftestWin.gBrowser.getBoundingClientRect();
98 if (browserRect.width === width && browserRect.height === height) {
99 return this.reftestWin;
101 logger.debug(`current: ${browserRect.width}x${browserRect.height}`);
105 if (Services.appinfo.OS == "Android") {
106 logger.debug("Using current window");
107 reftestWin = this.parentWindow;
108 await this.driver.listener.get({
109 commandID: this.driver.listener.activeMessageId,
110 pageTimeout: timeout,
112 loadEventExpected: false,
115 logger.debug("Using separate window");
116 if (this.reftestWin && !this.reftestWin.closed) {
117 this.reftestWin.close();
119 reftestWin = await this.openWindow(width, height);
122 this.setupWindow(reftestWin, width, height);
123 this.windowUtils = reftestWin.windowUtils;
124 this.reftestWin = reftestWin;
126 let found = this.driver.findWindow([reftestWin], () => true);
127 await this.driver.setWindowHandle(found, true);
129 let browserRect = reftestWin.gBrowser.getBoundingClientRect();
130 logger.debug(`new: ${browserRect.width}x${browserRect.height}`);
135 async openWindow(width, height) {
136 assert.positiveInteger(width);
137 assert.positiveInteger(height);
139 let reftestWin = this.parentWindow.open(
140 "chrome://marionette/content/reftest.xul",
142 `chrome,height=${height},width=${width}`
145 await new Promise(resolve => {
146 reftestWin.addEventListener("load", resolve, { once: true });
151 setupWindow(reftestWin, width, height) {
153 if (Services.appinfo.OS === "Android") {
154 browser = reftestWin.document.getElementsByTagName("browser")[0];
155 browser.setAttribute("remote", "false");
157 browser = reftestWin.document.createElementNS(XUL_NS, "xul:browser");
158 browser.permanentKey = {};
159 browser.setAttribute("id", "browser");
160 browser.setAttribute("anonid", "initialBrowser");
161 browser.setAttribute("type", "content");
162 browser.setAttribute("primary", "true");
164 browser.setAttribute("remote", "true");
165 browser.setAttribute("remoteType", "web");
167 browser.setAttribute("remote", "false");
170 // Make sure the browser element is exactly the right size, no matter
171 // what size our window is
172 const windowStyle = `padding: 0px; margin: 0px; border:none;
173 min-width: ${width}px; min-height: ${height}px;
174 max-width: ${width}px; max-height: ${height}px`;
175 browser.setAttribute("style", windowStyle);
177 if (Services.appinfo.OS !== "Android") {
178 let doc = reftestWin.document.documentElement;
179 while (doc.firstChild) {
180 doc.firstChild.remove();
182 doc.appendChild(browser);
184 if (reftestWin.BrowserApp) {
185 reftestWin.BrowserApp = browser;
187 reftestWin.gBrowser = browser;
192 if (this.reftestWin) {
193 this.driver.closeChromeWindow();
195 this.reftestWin = null;
199 * Run a specific reftest.
201 * The assumed semantics are those of web-platform-tests where
202 * references form a tree and each test must meet all the conditions
203 * to reach one leaf node of the tree in order for the overall test
206 * @param {string} testUrl
207 * URL of the test itself.
208 * @param {Array.<Array>} references
209 * Array representing a tree of references to try.
211 * Each item in the array represents a single reference node and
212 * has the form <code>[referenceUrl, references, relation]</code>,
213 * where <var>referenceUrl</var> is a string to the URL, relation
214 * is either <code>==</code> or <code>!=</code> depending on the
215 * type of reftest, and references is another array containing
216 * items of the same form, representing further comparisons treated
217 * as AND with the current item. Sibling entries are treated as OR.
219 * For example with testUrl of T:
222 * references = [[A, [[B, [], ==]], ==]]
223 * Must have T == A AND A == B to pass
225 * references = [[A, [], ==], [B, [], !=]
226 * Must have T == A OR T != B
228 * references = [[A, [[B, [], ==], [C, [], ==]], ==], [D, [], ]]
229 * Must have (T == A AND A == B) OR (T == A AND A == C) OR (T == D)
232 * @param {string} expected
233 * Expected test outcome (e.g. <tt>PASS</tt>, <tt>FAIL</tt>).
234 * @param {number} timeout
235 * Test timeout in milliseconds.
238 * Result object with fields status, message and extra.
245 width = DEFAULT_REFTEST_WIDTH,
246 height = DEFAULT_REFTEST_HEIGHT
250 let timeoutPromise = new Promise(resolve => {
251 timeoutHandle = this.parentWindow.setTimeout(() => {
252 resolve({ status: STATUS.TIMEOUT, message: null, extra: {} });
256 let testRunner = (async () => {
259 result = await this.runTest(
269 status: STATUS.ERROR,
278 let result = await Promise.race([testRunner, timeoutPromise]);
279 this.parentWindow.clearTimeout(timeoutHandle);
280 if (result.status === STATUS.TIMEOUT) {
287 async runTest(testUrl, references, expected, timeout, width, height) {
288 let win = await this.ensureWindow(timeout, width, height);
290 function toBase64(screenshot) {
291 let dataURL = screenshot.canvas.toDataURL();
292 return dataURL.split(",")[1];
302 let screenshotData = [];
305 for (let i = references.length - 1; i >= 0; i--) {
306 let item = references[i];
307 stack.push([testUrl, ...item]);
312 while (stack.length && !done) {
313 let [lhsUrl, rhsUrl, references, relation, extras = {}] = stack.pop();
314 result.message += `Testing ${lhsUrl} ${relation} ${rhsUrl}\n`;
318 comparison = await this.compareUrls(
335 if (comparison.msg) {
336 result.message += `${comparison.msg}\n`;
338 if (comparison.error !== null) {
339 result.status = STATUS.ERROR;
340 result.message += String(comparison.error);
341 result.stack = comparison.error.stack;
344 function recordScreenshot() {
345 let encodedLHS = comparison.lhs ? toBase64(comparison.lhs) : "";
346 let encodedRHS = comparison.rhs ? toBase64(comparison.rhs) : "";
347 screenshotData.push([
348 { url: lhsUrl, screenshot: encodedLHS },
350 { url: rhsUrl, screenshot: encodedRHS },
354 if (this.screenshotMode === SCREENSHOT_MODE.always) {
358 if (comparison.passed) {
359 if (references.length) {
360 for (let i = references.length - 1; i >= 0; i--) {
361 let item = references[i];
362 stack.push([rhsUrl, ...item]);
365 // Reached a leaf node so all of one reference chain passed
366 result.status = STATUS.PASS;
368 this.screenshotMode <= SCREENSHOT_MODE.fail &&
369 expected != result.status
375 } else if (!stack.length || result.status == STATUS.ERROR) {
376 // If we don't have any alternatives to try then this will be
377 // the last iteration, so save the failing screenshots if required.
378 let isFail = this.screenshotMode === SCREENSHOT_MODE.fail;
379 let isUnexpected = this.screenshotMode === SCREENSHOT_MODE.unexpected;
380 if (isFail || (isUnexpected && expected != result.status)) {
385 // Return any reusable canvases to the pool
386 let cacheKey = width + "x" + height;
387 let canvasPool = this.canvasCache.get(cacheKey).get(null);
388 [comparison.lhs, comparison.rhs].map(screenshot => {
389 if (screenshot !== null && screenshot.reuseCanvas) {
390 canvasPool.push(screenshot.canvas);
394 `Canvas pool (${cacheKey}) is of length ${canvasPool.length}`
398 if (screenshotData.length) {
399 // For now the tbpl formatter only accepts one screenshot, so just
400 // return the last one we took.
401 let lastScreenshot = screenshotData[screenshotData.length - 1];
402 // eslint-disable-next-line camelcase
403 result.extra.reftest_screenshots = lastScreenshot;
409 async compareUrls(win, lhsUrl, rhsUrl, relation, timeout, extras) {
410 logger.info(`Testing ${lhsUrl} ${relation} ${rhsUrl}`);
412 // Take the reference screenshot first so that if we pause
413 // we see the test rendering
414 let rhs = await this.screenshot(win, rhsUrl, timeout);
415 let lhs = await this.screenshot(win, lhsUrl, timeout);
417 logger.debug(`lhs canvas size ${lhs.canvas.width}x${lhs.canvas.height}`);
418 logger.debug(`rhs canvas size ${rhs.canvas.width}x${rhs.canvas.height}`);
422 let pixelsDifferent = null;
423 let maxDifferences = {};
427 pixelsDifferent = this.windowUtils.compareCanvases(
437 if (error === null) {
438 passed = this.isAcceptableDifference(
439 maxDifferences.value,
447 `Found ${pixelsDifferent} pixels different, ` +
448 `maximum difference per channel ${maxDifferences.value}`;
455 throw new InvalidArgumentError(
456 "Reftest operator should be '==' or '!='"
460 return { lhs, rhs, passed, error, msg };
463 isAcceptableDifference(maxDifference, pixelsDifferent, allowed) {
465 logger.info(`No differences allowed`);
466 return pixelsDifferent === 0;
468 let [allowedDiff, allowedPixels] = allowed;
470 `Allowed ${allowedPixels.join("-")} pixels different, ` +
471 `maximum difference per channel ${allowedDiff.join("-")}`
474 (pixelsDifferent === 0 && allowedPixels[0] == 0) ||
475 (maxDifference === 0 && allowedDiff[0] == 0) ||
476 (maxDifference >= allowedDiff[0] &&
477 maxDifference <= allowedDiff[1] &&
478 (pixelsDifferent >= allowedPixels[0] ||
479 pixelsDifferent <= allowedPixels[1]))
484 const focusManager = Services.focus;
485 if (focusManager.activeWindow != win) {
486 focusManager.activeWindow = win;
488 this.driver.curBrowser.contentBrowser.focus();
491 async screenshot(win, url, timeout) {
492 // On windows the above doesn't *actually* set the window to be the
493 // reftest size; but *does* set the content area to be the right size;
494 // the window is given some extra borders that aren't explicable from CSS
495 let browserRect = win.gBrowser.getBoundingClientRect();
497 let remainingCount = this.urlCount.get(url) || 1;
498 let cache = remainingCount > 1;
499 let cacheKey = browserRect.width + "x" + browserRect.height;
501 `screenshot ${url} remainingCount: ` +
502 `${remainingCount} cache: ${cache} cacheKey: ${cacheKey}`
504 let reuseCanvas = false;
505 let sizedCache = this.canvasCache.get(cacheKey);
506 if (sizedCache.has(url)) {
507 logger.debug(`screenshot ${url} taken from cache`);
508 canvas = sizedCache.get(url);
510 sizedCache.delete(url);
513 let canvasPool = sizedCache.get(null);
514 if (canvasPool.length) {
515 logger.debug("reusing canvas from canvas pool");
516 canvas = canvasPool.pop();
518 logger.debug("using new canvas");
521 reuseCanvas = !cache;
523 let ctxInterface = win.CanvasRenderingContext2D;
525 ctxInterface.DRAWWINDOW_DRAW_CARET |
526 ctxInterface.DRAWWINDOW_DRAW_VIEW |
527 ctxInterface.DRAWWINDOW_USE_WIDGET_LAYERS;
531 0 <= browserRect.left &&
532 0 <= browserRect.top &&
533 win.innerWidth >= browserRect.width &&
534 win.innerHeight >= browserRect.height
537 logger.error(`Invalid window dimensions:
538 browserRect.left: ${browserRect.left}
539 browserRect.top: ${browserRect.top}
540 win.innerWidth: ${win.innerWidth}
541 browserRect.width: ${browserRect.width}
542 win.innerHeight: ${win.innerHeight}
543 browserRect.height: ${browserRect.height}`);
544 throw new Error("Window has incorrect dimensions");
547 url = new URL(url).href; // normalize the URL
548 logger.debug(`Starting load of ${url}`);
550 commandId: this.driver.listener.activeMessageId,
551 pageTimeout: timeout,
553 if (this.lastURL === url) {
554 logger.debug(`Refreshing page`);
555 await this.driver.listener.refresh(navigateOpts);
557 navigateOpts.url = url;
558 navigateOpts.loadEventExpected = false;
559 await this.driver.listener.get(navigateOpts);
563 this.ensureFocus(win);
564 await this.driver.listener.reftestWait(url, this.remote);
566 canvas = await capture.canvas(
568 win.docShell.browsingContext,
573 { canvas, flags, readback: true }
577 canvas.width !== browserRect.width ||
578 canvas.height !== browserRect.height
581 `Canvas dimensions changed to ${canvas.width}x${canvas.height}`
587 sizedCache.set(url, canvas);
589 this.urlCount.set(url, remainingCount - 1);
590 return { canvas, reuseCanvas };
594 class DefaultMap extends Map {
595 constructor(iterable, defaultFactory) {
597 this.defaultFactory = defaultFactory;
602 return super.get(key);
605 let v = this.defaultFactory();