added backend for puppeterr+firefox (fix #210)

Former-commit-id: 06e61799aa014062300461cc65860cabd19efc12

cli/back-ends/puppeteer-firefox.js

@@ -0,0 +1,129 @@
+/*
+ * Copyright 2010-2019 Gildas Lormeau
+ * contact : gildas.lormeau <at> gmail.com
+ * 
+ * This file is part of SingleFile.
+ *
+ *   The code in this file is free software: you can redistribute it and/or 
+ *   modify it under the terms of the GNU Affero General Public License 
+ *   (GNU AGPL) as published by the Free Software Foundation, either version 3
+ *   of the License, or (at your option) any later version.
+ * 
+ *   The code in this file is distributed in the hope that it will be useful, 
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
+ *   General Public License for more details.
+ *
+ *   As additional permission under GNU AGPL version 3 section 7, you may 
+ *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
+ *   AGPL normally required by section 4, provided you include this license 
+ *   notice and a URL through which recipients can access the Corresponding 
+ *   Source.
+ */
+
+/* global singlefile, require, exports */
+
+const puppeteer = require("puppeteer-firefox");
+const scripts = require("./common/scripts.js");
+
+const EXECUTION_CONTEXT_DESTROYED_ERROR = "Execution context was destroyed";
+const NETWORK_IDLE_STATE = "networkidle0";
+
+exports.getPageData = async options => {
+	let browser;
+	try {
+		browser = await puppeteer.launch(getBrowserOptions(options));
+		const page = await browser.newPage();
+		await setPageOptions(page, options);
+		return await getPageData(browser, page, options);
+	} finally {
+		if (browser && !options.browserDebug) {
+			await browser.close();
+		}
+	}
+};
+
+function getBrowserOptions(options) {
+	const browserOptions = {};
+	if (options.browserHeadless !== undefined) {
+		browserOptions.headless = options.browserHeadless && !options.browserDebug;
+	}
+	browserOptions.args = options.browserArgs ? JSON.parse(options.browserArgs) : [];
+	if (options.browserExecutablePath) {
+		browserOptions.executablePath = options.browserExecutablePath || "firefox";
+	}
+	return browserOptions;
+}
+
+async function setPageOptions(page, options) {
+	if (options.browserWidth && options.browserHeight) {
+		await page.setViewport({
+			width: options.browserWidth,
+			height: options.browserHeight
+		});
+	}
+	if ((options.browserBypassCSP === undefined || options.browserBypassCSP) && page.setBypassCSP) {
+		await page.setBypassCSP(true);
+	}
+}
+
+async function getPageData(browser, page, options) {
+	const injectedScript = await scripts.get(options);
+	await page.evaluateOnNewDocument(injectedScript);
+	if (options.browserDebug) {
+		await page.waitFor(3000);
+	}
+	await pageGoto(page, options);
+	try {
+		return await page.evaluate(async options => {
+			const pageData = await singlefile.lib.getPageData(options);
+			if (options.includeInfobar) {
+				await singlefile.common.ui.content.infobar.includeScript(pageData);
+			}
+			return pageData;
+		}, options);
+	} catch (error) {
+		if (error.message && error.message.includes(EXECUTION_CONTEXT_DESTROYED_ERROR)) {
+			const pageData = await handleJSRedirect(browser, options);
+			if (pageData) {
+				return pageData;
+			} else {
+				throw error;
+			}
+		} else {
+			throw error;
+		}
+	}
+}
+
+async function handleJSRedirect(browser, options) {
+	const pages = await browser.pages();
+	const page = pages[1] || pages[0];
+	await pageGoto(page, options);
+	const url = page.url();
+	if (url != options.url) {
+		options.url = url;
+		await browser.close();
+		return exports.getPageData(options);
+	}
+}
+
+async function pageGoto(page, options) {
+	try {
+		await page.goto(options.url, {
+			timeout: options.browserLoadMaxTime || 0,
+			waitUntil: options.browserWaitUntil || NETWORK_IDLE_STATE
+		});
+	} catch (error) {
+		if (error.name != "TimeoutError") {
+			if (error.message.includes("Unknown waitUntil condition")) {
+				await page.goto(options.url, {
+					timeout: options.browserLoadMaxTime || 0,
+					waitUntil: "load"
+				});
+			} else {
+				throw error;
+			}
+		}
+	}
+}

cli/single-file

@@ -70,26 +70,26 @@ const args = require("yargs")
 		"user-script-enabled": true
 	})
 	.options("back-end", { description: "Back-end to use" })
-	.choices("back-end", ["jsdom", "puppeteer", "webdriver-chromium", "webdriver-gecko"])
-	.options("browser-headless", { description: "Run the browser in headless mode (puppeteer, webdriver-gecko, webdriver-chromium)" })
+	.choices("back-end", ["jsdom", "puppeteer", "webdriver-chromium", "webdriver-gecko", "puppeteer-firefox"])
+	.options("browser-headless", { description: "Run the browser in headless mode (puppeteer, puppeteer-firefox, webdriver-gecko, webdriver-chromium)" })
 	.boolean("browser-headless")
-	.options("browser-executable-path", { description: "Path to chrome/chromium executable (puppeteer, webdriver-gecko, webdriver-chromium)" })
+	.options("browser-executable-path", { description: "Path to chrome/chromium executable (puppeteer, puppeteer-firefox, webdriver-gecko, webdriver-chromium)" })
 	.string("browser-executable-path")
 	.options("browser-width", { description: "Width of the browser viewport in pixels" })
 	.number("browser-width")
 	.options("browser-height", { description: "Height of the browser viewport in pixels" })
 	.number("browser-height")
-	.options("browser-load-max-time", { description: "Maximum delay of time to wait for page loading in ms (puppeteer, webdriver-gecko, webdriver-chromium)" })
+	.options("browser-load-max-time", { description: "Maximum delay of time to wait for page loading in ms (puppeteer, puppeteer-firefox, webdriver-gecko, webdriver-chromium)" })
 	.number("browser-load-max-time")
-	.options("browser-wait-until", { description: "When to consider the page is loaded (puppeteer, webdriver-gecko, webdriver-chromium)" })
+	.options("browser-wait-until", { description: "When to consider the page is loaded (puppeteer, puppeteer-firefox, webdriver-gecko, webdriver-chromium)" })
 	.choices("browser-wait-until", ["networkidle0", "networkidle2", "load", "domcontentloaded"])
-	.options("browser-debug", { description: "Enable debug mode (puppeteer, webdriver-gecko, webdriver-chromium)" })
+	.options("browser-debug", { description: "Enable debug mode (puppeteer, puppeteer-firefox, webdriver-gecko, webdriver-chromium)" })
 	.boolean("browser-debug")
 	.options("browser-extensions", { description: "List of extension paths separated by a space and relative to the 'cli' folder (webdriver-gecko, webdriver-chromium)" })
 	.array("browser-extensions")
 	.options("browser-scripts", { description: "List of script paths separated by a space and relative to the 'cli' folder. They will be executed in all the frames." })
 	.array("browser-scripts")
-	.options("browser-args", { description: "Arguments provided as a JSON array and passed to the browser (puppeteer, webdriver-gecko, webdriver-chromium)" })
+	.options("browser-args", { description: "Arguments provided as a JSON array and passed to the browser (puppeteer, puppeteer-firefox, webdriver-gecko, webdriver-chromium)" })
 	.string("browser-args")
 	.options("compress-CSS", { description: "Compress CSS stylesheets" })
 	.boolean("compress-CSS")
@@ -104,15 +104,15 @@ const args = require("yargs")
 	.boolean("group-duplicate-images")
 	.options("include-infobar", { description: "Include the infobar" })
 	.boolean("include-infobar")
-	.options("load-deferred-images", { description: "Load deferred (a.k.a. lazy-loaded) images (puppeteer, webdriver-gecko, webdriver-chromium)" })
+	.options("load-deferred-images", { description: "Load deferred (a.k.a. lazy-loaded) images (puppeteer, puppeteer-firefox, webdriver-gecko, webdriver-chromium)" })
 	.boolean("load-deferred-images")
-	.options("load-deferred-images-max-idle-time", { description: "Maximum delay of time to wait for deferred images in ms (puppeteer, webdriver-gecko, webdriver-chromium)" })
+	.options("load-deferred-images-max-idle-time", { description: "Maximum delay of time to wait for deferred images in ms (puppeteer, puppeteer-firefox, webdriver-gecko, webdriver-chromium)" })
 	.number("load-deferred-images-max-idle-time")
 	.options("max-resource-size-enabled", { description: "Enable removal of embedded resources exceeding a given size" })
 	.boolean("max-resource-size-enabled")
 	.options("max-resource-size", { description: "Maximum size of embedded resources in MB (i.e. images, stylesheets, scripts and iframes)" })
 	.number("max-resource-size")
-	.options("remove-frames", { description: "Remove frames (puppeteer, webdriver-gecko, webdriver-chromium)" })
+	.options("remove-frames", { description: "Remove frames (puppeteer, puppeteer-firefox, webdriver-gecko, webdriver-chromium)" })
 	.boolean("remove-frames")
 	.options("remove-hidden-elements", { description: "Remove HTML elements which are not displayed" })
 	.boolean("remove-hidden-elements")
@@ -134,7 +134,7 @@ const args = require("yargs")
 	.boolean("remove-alternative-medias")
 	.options("remove-alternative-images", { description: "Remove images for alternative sizes of screen" })
 	.boolean("remove-alternative-images")
-	.options("save-raw-page", { description: "Save the original page without interpreting it into the browser (puppeteer, webdriver-gecko, webdriver-chromium)" })
+	.options("save-raw-page", { description: "Save the original page without interpreting it into the browser (puppeteer, puppeteer-firefox, webdriver-gecko, webdriver-chromium)" })
 	.boolean("save-raw-page")
 	.options("user-agent", { description: "User-agent of the browser (puppeteer, webdriver-gecko, webdriver-chromium)" })
 	.string("user-agent")
@@ -147,6 +147,7 @@ const args = require("yargs")
 const backEnds = {
 	jsdom: "./back-ends/jsdom.js",
 	puppeteer: "./back-ends/puppeteer.js",
+	"puppeteer-firefox": "./back-ends/puppeteer-firefox.js",
 	"webdriver-chromium": "./back-ends/webdriver-chromium.js",
 	"webdriver-gecko": "./back-ends/webdriver-gecko.js"
 };