added support of "browserWaitUntil" option

cli/back-ends/extensions/network-idle/bg.js

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2010-2019 Gildas Lormeau
+ * contact : gildas.lormeau <at> gmail.com
+ * 
+ * This file is part of SingleFile.
+ *
+ *   The code in this file is free software: you can redistribute it and/or 
+ *   modify it under the terms of the GNU Affero General Public License 
+ *   (GNU AGPL) as published by the Free Software Foundation, either version 3
+ *   of the License, or (at your option) any later version.
+ * 
+ *   The code in this file is distributed in the hope that it will be useful, 
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
+ *   General Public License for more details.
+ *
+ *   As additional permission under GNU AGPL version 3 section 7, you may 
+ *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
+ *   AGPL normally required by section 4, provided you include this license 
+ *   notice and a URL through which recipients can access the Corresponding 
+ *   Source.
+ */
+
+/* global setTimeout, clearTimeout */
+
+const browserAPI = this.browser || this.chrome;
+
+const IDLE_DELAY = 1000;
+const watchDogs = [];
+let pendingRequests = new Set();
+
+browserAPI.webRequest.onSendHeaders.addListener(onRequest, { urls: ["<all_urls>"] }, []);
+browserAPI.webRequest.onResponseStarted.addListener(onResponse, { urls: ["<all_urls>"] }, []);
+browserAPI.webRequest.onErrorOccurred.addListener(onResponse, { urls: ["<all_urls>"] });
+
+function onRequest(details) {
+	if (details.tabId != -1) {
+		pendingRequests.add(details.requestId);
+		if (pendingRequests.size > 2) {
+			clearTimeout(watchDogs[2]);
+		}
+		clearTimeout(watchDogs[0]);
+	}
+}
+
+function onResponse(details) {
+	if (details.tabId != -1) {
+		pendingRequests.delete(details.requestId);
+		if (pendingRequests.size == 2) {
+			maybeIdle(2, details.tabId);
+		}
+		if (pendingRequests.size == 0) {
+			maybeIdle(0, details.tabId);
+		}
+	}
+}
+
+function maybeIdle(idleLevel, tabId) {
+	clearTimeout(watchDogs[idleLevel]);
+	watchDogs[idleLevel] = setTimeout(() => browserAPI.tabs.sendMessage(tabId, "network-idle-" + idleLevel), IDLE_DELAY);
+}

cli/back-ends/extensions/network-idle/content.js

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2010-2019 Gildas Lormeau
+ * contact : gildas.lormeau <at> gmail.com
+ * 
+ * This file is part of SingleFile.
+ *
+ *   The code in this file is free software: you can redistribute it and/or 
+ *   modify it under the terms of the GNU Affero General Public License 
+ *   (GNU AGPL) as published by the Free Software Foundation, either version 3
+ *   of the License, or (at your option) any later version.
+ * 
+ *   The code in this file is distributed in the hope that it will be useful, 
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
+ *   General Public License for more details.
+ *
+ *   As additional permission under GNU AGPL version 3 section 7, you may 
+ *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
+ *   AGPL normally required by section 4, provided you include this license 
+ *   notice and a URL through which recipients can access the Corresponding 
+ *   Source.
+ */
+
+/* global dispatchEvent, CustomEvent */
+
+const browserAPI = this.browser || this.chrome;
+
+browserAPI.runtime.onMessage.addListener(message => dispatchEvent(new CustomEvent("single-file-" + message)));

cli/back-ends/extensions/network-idle/manifest.json

@@ -0,0 +1,31 @@
+{
+	"name": "network-idle",
+	"version": "0.0.2",
+	"background": {
+		"scripts": [
+			"bg.js"
+		]
+	},
+	"content_scripts": [
+		{
+			"matches": [
+				"<all_urls>"
+			],
+			"run_at": "document_start",
+			"js": [
+				"content.js"
+			]
+		}
+	],
+	"permissions": [
+		"tabs",
+		"webRequest",
+		"<all_urls>"
+	],
+	"applications": {
+		"gecko": {
+			"id": "{caf9306a-8951-4f0c-beb0-bab690d00caf}"
+		}
+	},
+	"manifest_version": 2
+}

cli/back-ends/webdriver-firefox.js

@@ -58,7 +58,7 @@ exports.getPageData = async options => {
 	};
 	let driver;
 	try {
-		const builder = new Builder();
+		const builder = new Builder().withCapabilities({ "pageLoadStrategy": "none" });
 		const firefoxOptions = new firefox.Options();
 		if (options.browserHeadless === undefined || options.browserHeadless) {
 			firefoxOptions.headless();
@@ -66,22 +66,23 @@ exports.getPageData = async options => {
 		if (options.browserExecutablePath) {
 			firefoxOptions.setBinary(options.browserExecutablePath);
 		}
-		if (options.browserDisableWebSecurity === undefined || options.browserDisableWebSecurity || options.browserBypassCSP === undefined || options.browserBypassCSP || options.userAgent || options.enableMaff) {
-			const profile = new firefox.Profile();
-			if (options.browserDisableWebSecurity === undefined || options.browserDisableWebSecurity) {
-				profile.addExtension(require.resolve("./extensions/signed/disable_web_security-0.0.3-an+fx.xpi"));
-			}
-			if (options.browserBypassCSP === undefined || options.browserBypassCSP) {
-				profile.addExtension(require.resolve("./extensions/signed/bypass_csp-0.0.3-an+fx.xpi"));
-			}
-			if (options.userAgent) {
-				profile.setPreference("general.useragent.override", options.userAgent);
-			}
-			if (options.enableMaff) {
-				profile.addExtension(require.resolve("./extensions/signed/mozilla_archive_format_with_mht_and_faithful_save-5.2.1-fx+sm.xpi"));
-			}
-			firefoxOptions.setProfile(profile);
+		const profile = new firefox.Profile();
+		if (options.browserDisableWebSecurity === undefined || options.browserDisableWebSecurity) {
+			profile.addExtension(require.resolve("./extensions/signed/disable_web_security-0.0.3-an+fx.xpi"));
+		}
+		if (options.browserBypassCSP === undefined || options.browserBypassCSP) {
+			profile.addExtension(require.resolve("./extensions/signed/bypass_csp-0.0.3-an+fx.xpi"));
 		}
+		if (options.userAgent) {
+			profile.setPreference("general.useragent.override", options.userAgent);
+		}
+		if (options.enableMaff) {
+			profile.addExtension(require.resolve("./extensions/signed/mozilla_archive_format_with_mht_and_faithful_save-5.2.1-fx+sm.xpi"));
+		}
+		if (options.browserWaitUntil === undefined || options.browserWaitUntil == "networkidle0" || options.browserWaitUntil == "networkidle2") {
+			profile.addExtension(require.resolve("./extensions/signed/network_idle-0.0.2-an+fx.xpi"));
+		}
+		firefoxOptions.setProfile(profile);
 		builder.setFirefoxOptions(firefoxOptions);
 		driver = await builder.forBrowser("firefox").build();
 		driver.manage().setTimeouts({ script: null, pageLoad: null, implicit: null });
@@ -95,9 +96,17 @@ exports.getPageData = async options => {
 		}
 		let scripts = SCRIPTS.map(scriptPath => fs.readFileSync(require.resolve(scriptPath)).toString().replace(/\n(this)\.([^ ]+) = (this)\.([^ ]+) \|\|/g, "\nwindow.$2 = window.$4 ||")).join("\n");
 		scripts += "\nlazyLoader.getScriptContent = " + (function (path) { return (RESOLVED_CONTENTS)[path]; }).toString().replace("RESOLVED_CONTENTS", JSON.stringify(RESOLVED_CONTENTS)) + ";";
-		const loadPromise = driver.get(options.url);
-		driver.executeScript(scripts);
-		await loadPromise;
+		await driver.get(options.url);
+		await driver.wait(() => driver.executeScript("return location.href != \"about:blank\""));
+		if (options.browserWaitUntil === undefined || options.browserWaitUntil == "networkidle0") {
+			await driver.executeAsyncScript(scripts + "\naddEventListener(\"single-file-network-idle-0\", () => arguments[0](), true)");
+		} else if (options.browserWaitUntil == "networkidle2") {
+			await driver.executeAsyncScript(scripts + "\naddEventListener(\"single-file-network-idle-2\", () => arguments[0](), true)");
+		} else if (options.browserWaitUntil == "load") {
+			await driver.executeAsyncScript(scripts + "\nif (document.readyState == \"loading\") { document.addEventListener(\"load\", () => arguments[0]()) } else { arguments[0](); }");
+		} else {
+			await driver.executeScript(scripts);
+		}
 		if (!options.removeFrames) {
 			const windowHandles = await driver.getAllWindowHandles();
 			await Promise.all(windowHandles.map(async windowHandle => {