added urls-file option (fix #341)

Former-commit-id: fe7f118ae15b20a2cdd67b221bd3123c44d3902a

cli/back-ends/jsdom.js

@@ -29,6 +29,8 @@ const { JSDOM, VirtualConsole } = require("jsdom");
 const iconv = require("iconv-lite");
 const request = require("request-promise-native");
 
+exports.initialize = async () => { };
+
 exports.getPageData = async options => {
 	const pageContent = (await request({
 		method: "GET",
@@ -39,6 +41,54 @@ exports.getPageData = async options => {
 			"User-Agent": options.userAgent
 		}
 	})).body.toString();
+	let win;
+	try {
+		const dom = new JSDOM(pageContent, getBrowserOptions(options));
+		win = dom.window;
+		return await getPageData(win, options);
+	} finally {
+		if (win) {
+			win.close();
+		}
+	}
+};
+
+async function getPageData(win, options) {
+	const doc = win.document;
+	const scripts = await require("./common/scripts.js").get(options);
+	win.TextDecoder = class {
+		constructor(utfLabel) {
+			this.utfLabel = utfLabel;
+		}
+		decode(buffer) {
+			return iconv.decode(buffer, this.utfLabel);
+		}
+	};
+	win.crypto = {
+		subtle: {
+			digest: async function digestText(algo, text) {
+				const hash = crypto.createHash(algo.replace("-", "").toLowerCase());
+				hash.update(text, "utf-8");
+				return hash.digest();
+			}
+		}
+	};
+	win.Element.prototype.getBoundingClientRect = undefined;
+	win.getComputedStyle = () => { };
+	win.eval(scripts);
+	if (win.document.readyState == "loading" || win.document.readyState == "interactive") {
+		await new Promise(resolve => win.onload = resolve);
+	}
+	executeFrameScripts(doc, scripts);
+	options.removeHiddenElements = false;
+	const pageData = await win.singlefile.lib.getPageData(options, { fetch: url => fetchResource(url, options) }, doc, win);
+	if (options.includeInfobar) {
+		await win.singlefile.common.ui.content.infobar.includeScript(pageData);
+	}
+	return pageData;
+}
+
+function getBrowserOptions(options) {
 	const jsdomOptions = {
 		url: options.url,
 		virtualConsole: new VirtualConsole(),
@@ -53,47 +103,8 @@ exports.getPageData = async options => {
 			window.outerHeight = window.innerHeight = options.browserHeight;
 		};
 	}
-	const dom = new JSDOM(pageContent, jsdomOptions);
-	const win = dom.window;
-	const doc = win.document;
-	try {
-		const scripts = await require("./common/scripts.js").get(options);
-		win.TextDecoder = class {
-			constructor(utfLabel) {
-				this.utfLabel = utfLabel;
-			}
-			decode(buffer) {
-				return iconv.decode(buffer, this.utfLabel);
-			}
-		};
-		win.crypto = {
-			subtle: {
-				digest: async function digestText(algo, text) {
-					const hash = crypto.createHash(algo.replace("-", "").toLowerCase());
-					hash.update(text, "utf-8");
-					return hash.digest();
-				}
-			}
-		};
-		win.Element.prototype.getBoundingClientRect = undefined;
-		win.getComputedStyle = () => { };
-		win.eval(scripts);
-		if (win.document.readyState == "loading") {
-			await new Promise(resolve => win.document.onload = resolve);
-		}
-		executeFrameScripts(doc, scripts);
-		options.removeHiddenElements = false;
-		const pageData = await win.singlefile.lib.getPageData(options, { fetch: url => fetchResource(url, options) }, doc, win);
-		if (options.includeInfobar) {
-			await win.singlefile.common.ui.content.infobar.includeScript(pageData);
-		}
-		return pageData;
-	} finally {
-		if (win) {
-			win.close();
-		}
-	}
-};
+	return jsdomOptions;
+}
 
 function executeFrameScripts(doc, scripts) {
 	const frameElements = doc.querySelectorAll("iframe, frame");

cli/back-ends/puppeteer-firefox.js

@@ -29,15 +29,21 @@ const scripts = require("./common/scripts.js");
 const EXECUTION_CONTEXT_DESTROYED_ERROR = "Execution context was destroyed";
 const NETWORK_IDLE_STATE = "networkidle0";
 
+let browser, pendings = 0;
+
+exports.initialize = async options => {
+	browser = await puppeteer.launch(getBrowserOptions(options));
+};
+
 exports.getPageData = async options => {
-	let browser;
 	try {
-		browser = await puppeteer.launch(getBrowserOptions(options));
+		pendings++;
 		const page = await browser.newPage();
 		await setPageOptions(page, options);
 		return await getPageData(browser, page, options);
 	} finally {
-		if (browser && !options.browserDebug) {
+		pendings--;
+		if (!pendings && browser && !options.browserDebug) {
 			await browser.close();
 		}
 	}

cli/back-ends/puppeteer.js

@@ -29,15 +29,25 @@ const scripts = require("./common/scripts.js");
 const EXECUTION_CONTEXT_DESTROYED_ERROR = "Execution context was destroyed";
 const NETWORK_IDLE_STATE = "networkidle0";
 
+let browser, pendings = 0;
+
+exports.initialize = async options => {
+	browser = await puppeteer.launch(getBrowserOptions(options));
+};
+
 exports.getPageData = async options => {
-	let browser;
+	let page;
 	try {
-		browser = await puppeteer.launch(getBrowserOptions(options));
-		const page = await browser.newPage();
+		pendings++;
+		page = await browser.newPage();
 		await setPageOptions(page, options);
 		return await getPageData(browser, page, options);
 	} finally {
-		if (browser && !options.browserDebug) {
+		pendings--;
+		if (page) {
+			await page.close();
+		}
+		if (!pendings && browser && !options.browserDebug) {
 			await browser.close();
 		}
 	}

cli/back-ends/webdriver-chromium.js

@@ -28,102 +28,113 @@ const path = require("path");
 const chrome = require("selenium-webdriver/chrome");
 const { Builder } = require("selenium-webdriver");
 
+exports.initialize = async () => { };
+
 exports.getPageData = async options => {
 	let driver;
 	try {
 		const builder = new Builder();
-		const chromeOptions = new chrome.Options();
-		const optionHeadless = (options.browserHeadless === undefined || options.browserHeadless) && !options.browserDebug;
-		if (optionHeadless) {
-			chromeOptions.headless();
-		}
-		if (options.browserExecutablePath) {
-			chromeOptions.setChromeBinaryPath(options.browserExecutablePath);
-		}
-		if (options.webDriverExecutablePath) {
-			process.env["webdriver.chrome.driver"] = options.webDriverExecutablePath;
-		}
-		if (options.browserArgs) {
-			const args = JSON.parse(options.browserArgs);
-			args.forEach(argument => chromeOptions.addArguments(argument));
-		}
-		if (options.browserDisableWebSecurity === undefined || options.browserDisableWebSecurity) {
-			chromeOptions.addArguments("--disable-web-security");
-		}
-		chromeOptions.addArguments("--no-pings");
-		if (!optionHeadless) {
-			if (options.browserDebug) {
-				chromeOptions.addArguments("--auto-open-devtools-for-tabs");
-			}
-			const extensions = [];
-			if (options.browserBypassCSP === undefined || options.browserBypassCSP) {
-				extensions.push(encode(require.resolve("./extensions/signed/bypass_csp-0.0.3-an+fx.xpi")));
-			}
-			if (options.browserWaitUntil === undefined || options.browserWaitUntil == "networkidle0" || options.browserWaitUntil == "networkidle2") {
-				extensions.push(encode(require.resolve("./extensions/signed/network_idle-0.0.2-an+fx.xpi")));
-			}
-			if (options.browserExtensions && options.browserExtensions.length) {
-				options.browserExtensions.forEach(extensionPath => extensions.push(encode(path.resolve(__dirname, "..", extensionPath))));
-			}
-			chromeOptions.addExtensions(extensions);
-		}
-		if (options.userAgent) {
-			await chromeOptions.addArguments("--user-agent=" + JSON.stringify(options.userAgent));
+		builder.setChromeOptions(getBrowserOptions(options));
+		driver = builder.forBrowser("chrome").build();
+		return await getPageData(driver, options);
+	} finally {
+		if (driver && !options.browserDebug) {
+			driver.quit();
 		}
-		if (options.browserMobileEmulation) {
-			chromeOptions.setMobileEmulation({
-				deviceName: options.browserMobileEmulation
-			});
+	}
+};
+
+function getBrowserOptions(options) {
+	const chromeOptions = new chrome.Options();
+	const optionHeadless = (options.browserHeadless === undefined || options.browserHeadless) && !options.browserDebug;
+	if (optionHeadless) {
+		chromeOptions.headless();
+	}
+	if (options.browserExecutablePath) {
+		chromeOptions.setChromeBinaryPath(options.browserExecutablePath);
+	}
+	if (options.webDriverExecutablePath) {
+		process.env["webdriver.chrome.driver"] = options.webDriverExecutablePath;
+	}
+	if (options.browserArgs) {
+		const args = JSON.parse(options.browserArgs);
+		args.forEach(argument => chromeOptions.addArguments(argument));
+	}
+	if (options.browserDisableWebSecurity === undefined || options.browserDisableWebSecurity) {
+		chromeOptions.addArguments("--disable-web-security");
+	}
+	chromeOptions.addArguments("--no-pings");
+	if (!optionHeadless) {
+		if (options.browserDebug) {
+			chromeOptions.addArguments("--auto-open-devtools-for-tabs");
 		}
-		builder.setChromeOptions(chromeOptions);
-		driver = await builder.forBrowser("chrome").build();
-		driver.manage().setTimeouts({ script: options.browserLoadMaxTime, pageLoad: options.browserLoadMaxTime, implicit: options.browserLoadMaxTime });
-		if (options.browserWidth && options.browserHeight) {
-			const window = driver.manage().window();
-			if (window.setRect) {
-				window.setRect(options.browserHeight, options.browserWidth);
-			} else if (window.setSize) {
-				window.setSize(options.browserWidth, options.browserHeight);
-			}
+		const extensions = [];
+		if (options.browserBypassCSP === undefined || options.browserBypassCSP) {
+			extensions.push(encode(require.resolve("./extensions/signed/bypass_csp-0.0.3-an+fx.xpi")));
 		}
-		const scripts = await require("./common/scripts.js").get(options);
-		if (options.browserDebug) {
-			await driver.sleep(3000);
+		if (options.browserWaitUntil === undefined || options.browserWaitUntil == "networkidle0" || options.browserWaitUntil == "networkidle2") {
+			extensions.push(encode(require.resolve("./extensions/signed/network_idle-0.0.2-an+fx.xpi")));
 		}
-		await driver.get(options.url);
-		await driver.executeScript(scripts);
-		if (options.browserWaitUntil != "domcontentloaded") {
-			let scriptPromise;
-			if (!optionHeadless && (options.browserWaitUntil === undefined || options.browserWaitUntil == "networkidle0")) {
-				scriptPromise = driver.executeAsyncScript("addEventListener(\"single-file-network-idle-0\", () => arguments[0](), true)");
-			} else if (!optionHeadless && options.browserWaitUntil == "networkidle2") {
-				scriptPromise = driver.executeAsyncScript("addEventListener(\"single-file-network-idle-2\", () => arguments[0](), true)");
-			} else if (optionHeadless || options.browserWaitUntil == "load") {
-				scriptPromise = driver.executeAsyncScript("if (document.readyState == \"loading\" || document.readyState == \"interactive\") { document.addEventListener(\"load\", () => arguments[0]()) } else { arguments[0](); }");
-			}
-			let cancelTimeout;
-			const timeoutPromise = new Promise(resolve => {
-				const timeoutId = setTimeout(resolve, Math.max(0, options.browserLoadMaxTime - 5000));
-				cancelTimeout = () => {
-					clearTimeout(timeoutId);
-					resolve();
-				};
-			});
-			await Promise.race([scriptPromise, timeoutPromise]);
-			cancelTimeout();
+		if (options.browserExtensions && options.browserExtensions.length) {
+			options.browserExtensions.forEach(extensionPath => extensions.push(encode(path.resolve(__dirname, "..", extensionPath))));
 		}
-		const result = await driver.executeAsyncScript(getPageDataScript(), options);
-		if (result.error) {
-			throw result.error;
-		} else {
-			return result.pageData;
+		chromeOptions.addExtensions(extensions);
+	}
+	if (options.userAgent) {
+		chromeOptions.addArguments("--user-agent=" + JSON.stringify(options.userAgent));
+	}
+	if (options.browserMobileEmulation) {
+		chromeOptions.setMobileEmulation({
+			deviceName: options.browserMobileEmulation
+		});
+	}
+	return chromeOptions;
+}
+
+async function getPageData(driver, options) {
+	const optionHeadless = (options.browserHeadless === undefined || options.browserHeadless) && !options.browserDebug;
+	driver.manage().setTimeouts({ script: options.browserLoadMaxTime, pageLoad: options.browserLoadMaxTime, implicit: options.browserLoadMaxTime });
+	if (options.browserWidth && options.browserHeight) {
+		const window = driver.manage().window();
+		if (window.setRect) {
+			window.setRect(options.browserHeight, options.browserWidth);
+		} else if (window.setSize) {
+			window.setSize(options.browserWidth, options.browserHeight);
 		}
-	} finally {
-		if (driver && !options.browserDebug) {
-			driver.quit();
+	}
+	const scripts = await require("./common/scripts.js").get(options);
+	if (options.browserDebug) {
+		// await driver.sleep(3000);
+	}
+	await driver.get(options.url);
+	await driver.executeScript(scripts);
+	if (options.browserWaitUntil != "domcontentloaded") {
+		let scriptPromise;
+		if (!optionHeadless && (options.browserWaitUntil === undefined || options.browserWaitUntil == "networkidle0")) {
+			scriptPromise = driver.executeAsyncScript("addEventListener(\"single-file-network-idle-0\", () => arguments[0](), true)");
+		} else if (!optionHeadless && options.browserWaitUntil == "networkidle2") {
+			scriptPromise = driver.executeAsyncScript("addEventListener(\"single-file-network-idle-2\", () => arguments[0](), true)");
+		} else if (optionHeadless || options.browserWaitUntil == "load") {
+			scriptPromise = driver.executeAsyncScript("if (document.readyState == \"loading\" || document.readyState == \"interactive\") { addEventListener(\"load\", () => arguments[0]()) } else { arguments[0](); }");
 		}
+		let cancelTimeout;
+		const timeoutPromise = new Promise(resolve => {
+			const timeoutId = setTimeout(resolve, Math.max(0, options.browserLoadMaxTime - 5000));
+			cancelTimeout = () => {
+				clearTimeout(timeoutId);
+				resolve();
+			};
+		});
+		await Promise.race([scriptPromise, timeoutPromise]);
+		cancelTimeout();
 	}
-};
+	const result = await driver.executeAsyncScript(getPageDataScript(), options);
+	if (result.error) {
+		throw result.error;
+	} else {
+		return result.pageData;
+	}
+}
 
 function encode(file) {
 	return new Buffer.from(require("fs").readFileSync(file)).toString("base64");

cli/back-ends/webdriver-gecko.js

@@ -28,94 +28,15 @@ const path = require("path");
 const firefox = require("selenium-webdriver/firefox");
 const { Builder, By, Key } = require("selenium-webdriver");
 
+exports.initialize = async () => { };
+
 exports.getPageData = async options => {
 	let driver;
 	try {
 		const builder = new Builder().withCapabilities({ "pageLoadStrategy": "none" });
-		const firefoxOptions = new firefox.Options();
-		if ((options.browserHeadless === undefined || options.browserHeadless) && !options.browserDebug) {
-			firefoxOptions.headless();
-		}
-		if (options.browserExecutablePath) {
-			firefoxOptions.setBinary(options.browserExecutablePath);
-		}
-		if (options.webDriverExecutablePath) {
-			process.env["webdriver.gecko.driver"] = options.webDriverExecutablePath;
-		}
-		const extensions = [];
-		if (options.browserDisableWebSecurity === undefined || options.browserDisableWebSecurity) {
-			extensions.push(require.resolve("./extensions/signed/disable_web_security-0.0.3-an+fx.xpi"));
-		}
-		if (options.browserBypassCSP === undefined || options.browserBypassCSP) {
-			extensions.push(require.resolve("./extensions/signed/bypass_csp-0.0.3-an+fx.xpi"));
-		}
-		if (options.browserWaitUntil === undefined || options.browserWaitUntil == "networkidle0" || options.browserWaitUntil == "networkidle2") {
-			extensions.push(require.resolve("./extensions/signed/network_idle-0.0.2-an+fx.xpi"));
-		}
-		if (options.browserExtensions && options.browserExtensions.length) {
-			options.browserExtensions.forEach(extensionPath => extensions.push(path.resolve(__dirname, "..", extensionPath)));
-		}
-		if (options.browserArgs) {
-			const args = JSON.parse(options.browserArgs);
-			args.forEach(argument => firefoxOptions.addArguments(argument));
-		}
-		if (extensions.length) {
-			firefoxOptions.addExtensions(extensions);
-		}
-		if (options.userAgent) {
-			firefoxOptions.setPreference("general.useragent.override", options.userAgent);
-		}
-		builder.setFirefoxOptions(firefoxOptions);
-		driver = await builder.forBrowser("firefox").build();
-		driver.manage().setTimeouts({ script: options.browserLoadMaxTime, pageLoad: options.browserLoadMaxTime, implicit: options.browserLoadMaxTime });
-		if (options.browserWidth && options.browserHeight) {
-			const window = driver.manage().window();
-			if (window.setRect) {
-				window.setRect(options.browserHeight, options.browserWidth);
-			} else if (window.setSize) {
-				window.setSize(options.browserWidth, options.browserHeight);
-			}
-		}
-		let scripts = await require("./common/scripts.js").get(options);
-		if (options.browserDebug) {
-			await driver.findElement(By.css("html")).sendKeys(Key.SHIFT + Key.F5);
-			await driver.sleep(3000);
-		}
-		await driver.get(options.url);
-		while (await driver.getCurrentUrl() == "about:blank") {
-			// do nothing
-		}
-		scripts = scripts.replace(/\n(this)\.([^ ]+) = (this)\.([^ ]+) \|\|/g, "\nwindow.$2 = window.$4 ||");
-		await driver.executeScript(scripts);
-		if (options.browserWaitUntil != "domcontentloaded") {
-			let scriptPromise;
-			if (options.browserWaitUntil === undefined || options.browserWaitUntil == "networkidle0") {
-				scriptPromise = driver.executeAsyncScript("addEventListener(\"single-file-network-idle-0\", () => arguments[0](), true)");
-			} else if (options.browserWaitUntil == "networkidle2") {
-				scriptPromise = driver.executeAsyncScript("addEventListener(\"single-file-network-idle-2\", () => arguments[0](), true)");
-			} else if (options.browserWaitUntil == "load") {
-				scriptPromise = driver.executeAsyncScript("if (document.readyState == \"loading\" || document.readyState == \"interactive\") { document.addEventListener(\"load\", () => arguments[0]()) } else { arguments[0](); }");
-			}
-			let cancelTimeout;
-			const timeoutPromise = new Promise(resolve => {
-				const timeoutId = setTimeout(resolve, Math.max(0, options.browserLoadMaxTime - 5000));
-				cancelTimeout = () => {
-					clearTimeout(timeoutId);
-					resolve();
-				};
-			});
-			await Promise.race([scriptPromise, timeoutPromise]);
-			cancelTimeout();
-		}
-		if (!options.removeFrames) {
-			await executeScriptInFrames(driver, scripts);
-		}
-		const result = await driver.executeAsyncScript(getPageDataScript(), options);
-		if (result.error) {
-			throw result.error;
-		} else {
-			return result.pageData;
-		}
+		builder.setFirefoxOptions(getBrowserOptions(options));
+		driver = builder.forBrowser("firefox").build();
+		return await getPageData(driver, options);
 	} finally {
 		if (driver && !options.browserDebug) {
 			driver.quit();
@@ -123,6 +44,94 @@ exports.getPageData = async options => {
 	}
 };
 
+function getBrowserOptions(options) {
+	const firefoxOptions = new firefox.Options();
+	if ((options.browserHeadless === undefined || options.browserHeadless) && !options.browserDebug) {
+		firefoxOptions.headless();
+	}
+	if (options.browserExecutablePath) {
+		firefoxOptions.setBinary(options.browserExecutablePath);
+	}
+	if (options.webDriverExecutablePath) {
+		process.env["webdriver.gecko.driver"] = options.webDriverExecutablePath;
+	}
+	const extensions = [];
+	if (options.browserDisableWebSecurity === undefined || options.browserDisableWebSecurity) {
+		extensions.push(require.resolve("./extensions/signed/disable_web_security-0.0.3-an+fx.xpi"));
+	}
+	if (options.browserBypassCSP === undefined || options.browserBypassCSP) {
+		extensions.push(require.resolve("./extensions/signed/bypass_csp-0.0.3-an+fx.xpi"));
+	}
+	if (options.browserWaitUntil === undefined || options.browserWaitUntil == "networkidle0" || options.browserWaitUntil == "networkidle2") {
+		extensions.push(require.resolve("./extensions/signed/network_idle-0.0.2-an+fx.xpi"));
+	}
+	if (options.browserExtensions && options.browserExtensions.length) {
+		options.browserExtensions.forEach(extensionPath => extensions.push(path.resolve(__dirname, "..", extensionPath)));
+	}
+	if (extensions.length) {
+		firefoxOptions.addExtensions(extensions);
+	}
+	if (options.browserArgs) {
+		const args = JSON.parse(options.browserArgs);
+		args.forEach(argument => firefoxOptions.addArguments(argument));
+	}
+	if (options.userAgent) {
+		firefoxOptions.setPreference("general.useragent.override", options.userAgent);
+	}
+}
+
+async function getPageData(driver, options) {
+	driver.manage().setTimeouts({ script: options.browserLoadMaxTime, pageLoad: options.browserLoadMaxTime, implicit: options.browserLoadMaxTime });
+	if (options.browserWidth && options.browserHeight) {
+		const window = driver.manage().window();
+		if (window.setRect) {
+			window.setRect(options.browserHeight, options.browserWidth);
+		} else if (window.setSize) {
+			window.setSize(options.browserWidth, options.browserHeight);
+		}
+	}
+	let scripts = await require("./common/scripts.js").get(options);
+	if (options.browserDebug) {
+		await driver.findElement(By.css("html")).sendKeys(Key.SHIFT + Key.F5);
+		await driver.sleep(3000);
+	}
+	await driver.get(options.url);
+	while (await driver.getCurrentUrl() == "about:blank") {
+		// do nothing
+	}
+	scripts = scripts.replace(/\n(this)\.([^ ]+) = (this)\.([^ ]+) \|\|/g, "\nwindow.$2 = window.$4 ||");
+	await driver.executeScript(scripts);
+	if (options.browserWaitUntil != "domcontentloaded") {
+		let scriptPromise;
+		if (options.browserWaitUntil === undefined || options.browserWaitUntil == "networkidle0") {
+			scriptPromise = driver.executeAsyncScript("addEventListener(\"single-file-network-idle-0\", () => arguments[0](), true)");
+		} else if (options.browserWaitUntil == "networkidle2") {
+			scriptPromise = driver.executeAsyncScript("addEventListener(\"single-file-network-idle-2\", () => arguments[0](), true)");
+		} else if (options.browserWaitUntil == "load") {
+			scriptPromise = driver.executeAsyncScript("if (document.readyState == \"loading\" || document.readyState == \"interactive\") { addEventListener(\"load\", () => arguments[0]()) } else { arguments[0](); }");
+		}
+		let cancelTimeout;
+		const timeoutPromise = new Promise(resolve => {
+			const timeoutId = setTimeout(resolve, Math.max(0, options.browserLoadMaxTime - 5000));
+			cancelTimeout = () => {
+				clearTimeout(timeoutId);
+				resolve();
+			};
+		});
+		await Promise.race([scriptPromise, timeoutPromise]);
+		cancelTimeout();
+	}
+	if (!options.removeFrames) {
+		await executeScriptInFrames(driver, scripts);
+	}
+	const result = await driver.executeAsyncScript(getPageDataScript(), options);
+	if (result.error) {
+		throw result.error;
+	} else {
+		return result.pageData;
+	}
+}
+
 async function executeScriptInFrames(driver, scripts) {
 	let finished = false, indexFrame = 0;
 	while (!finished) {

cli/single-file

@@ -28,7 +28,7 @@
 const fileUrl = require("file-url");
 const args = require("yargs")
 	.wrap(null)
-	.command("$0 <url> [output]", "Save a page into a single HTML file.", yargs => {
+	.command("$0 [url] [output]", "Save a page into a single HTML file.", yargs => {
 		yargs.positional("url", { description: "URL or path on the filesystem of the page to save", type: "string" });
 		yargs.positional("output", { description: "Output filename", type: "string" });
 	})
@@ -52,6 +52,7 @@ const args = require("yargs")
 		"include-infobar": false,
 		"load-deferred-images": true,
 		"load-deferred-images-max-idle-time": 1500,
+		"maxParallelWorkers": 8,
 		"max-resource-size-enabled": false,
 		"max-resource-size": 10,
 		"remove-hidden-elements": true,
@@ -108,6 +109,8 @@ const args = require("yargs")
 	.boolean("load-deferred-images")
 	.options("load-deferred-images-max-idle-time", { description: "Maximum delay of time to wait for deferred images in ms (puppeteer, puppeteer-firefox, webdriver-gecko, webdriver-chromium)" })
 	.number("load-deferred-images-max-idle-time")
+	.options("max-parallel-workers", { description: "Maximum number of browsers launched in parallel when processing a list of URLs (cf --urls-file)" })
+	.number("max-parallel-workers")
 	.options("max-resource-size-enabled", { description: "Enable removal of embedded resources exceeding a given size" })
 	.boolean("max-resource-size-enabled")
 	.options("max-resource-size", { description: "Maximum size of embedded resources in MB (i.e. images, stylesheets, scripts and iframes)" })
@@ -136,6 +139,8 @@ const args = require("yargs")
 	.boolean("remove-alternative-images")
 	.options("save-raw-page", { description: "Save the original page without interpreting it into the browser (puppeteer, puppeteer-firefox, webdriver-gecko, webdriver-chromium)" })
 	.boolean("save-raw-page")
+	.options("urls-file", { description: "Path to a text file containing a list of URLs (separated by a newline) to save" })
+	.string("urls-file")
 	.options("user-agent", { description: "User-agent of the browser (puppeteer, webdriver-gecko, webdriver-chromium)" })
 	.string("user-agent")
 	.options("user-script-enabled", { description: "Enable the event API allowing to execute scripts before the page is saved" })
@@ -144,6 +149,7 @@ const args = require("yargs")
 	.string("web-driver-executable-path")
 	.argv;
 
+const fs = require("fs");
 const backEnds = {
 	jsdom: "./back-ends/jsdom.js",
 	puppeteer: "./back-ends/puppeteer.js",
@@ -153,18 +159,60 @@ const backEnds = {
 };
 args.compressCSS = args.compressCss;
 args.compressHTML = args.compressHtml;
-if (!/^(https?|file):\/\//.test(args.url)) {
+if (args.url && !/^(https?|file):\/\//.test(args.url)) {
 	args.url = fileUrl(args.url);
 }
 args.browserScripts = args.browserScripts.map(path => require.resolve(path));
-require(backEnds[args.backEnd]).getPageData(args).then(pageData => {
+const backend = require(backEnds[args.backEnd]);
+backend.initialize(args).then(() => {
+	if (args.urlsFile) {
+		const urls = fs.readFileSync(args.urlsFile).toString().split("\n").map(url => url.trim()).filter(url => url);
+		for (let workerIndex = 0; workerIndex < args.maxParallelWorkers; workerIndex++) {
+			workerCapturePage(args, urls, workerIndex);
+		}
+	} else {
+		capturePage(args);
+	}
+});
+
+async function workerCapturePage(args, urls, workerIndex, depth = 0) {
+	const url = urls[workerIndex + (depth * args.maxParallelWorkers)];
+	if (url) {
+		args = JSON.parse(JSON.stringify(args));
+		args.url = url;
+		args.output = null;
+		await capturePage(args);
+		await workerCapturePage(args, urls, workerIndex, depth + 1);
+	}
+}
+
+async function capturePage(args) {
+	const pageData = await backend.getPageData(args);
 	if (args.output) {
-		require("fs").writeFileSync(args.output, pageData.content);
+		fs.writeFileSync(getFilename(args.output), pageData.content);
 	} else {
-		if (args.filenameTemplate) {
-			require("fs").writeFileSync(pageData.filename, pageData.content);
+		if (args.filenameTemplate && pageData.filename) {
+			fs.writeFileSync(getFilename(pageData.filename), pageData.content);
 		} else {
 			console.log(pageData.content); // eslint-disable-line no-console			
 		}
 	}
-});
+}
+
+function getFilename(filename, index = 1) {
+	let newFilename = filename;
+	if (index > 1) {
+		const regExpMatchExtension = /(\.[^.]+)$/;
+		const matchExtension = newFilename.match(regExpMatchExtension);
+		if (matchExtension && matchExtension[1]) {
+			newFilename = newFilename.replace(regExpMatchExtension, " - " + index + matchExtension[1]);
+		} else {
+			newFilename += " - " + index;
+		}
+	}
+	if (fs.existsSync(newFilename)) {
+		return getFilename(filename, index + 1);
+	} else {
+		return newFilename;
+	}
+}