SingleFile/cli/single-file

#!/usr/bin/env node

/*
 * Copyright 2010-2020 Gildas Lormeau
 * contact : gildas.lormeau <at> gmail.com
 *
 * This file is part of SingleFile.
 *
 *   The code in this file is free software: you can redistribute it and/or
 *   modify it under the terms of the GNU Affero General Public License
 *   (GNU AGPL) as published by the Free Software Foundation, either version 3
 *   of the License, or (at your option) any later version.
 *
 *   The code in this file is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
 *   General Public License for more details.
 *
 *   As additional permission under GNU AGPL version 3 section 7, you may
 *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU
 *   AGPL normally required by section 4, provided you include this license
 *   notice and a URL through which recipients can access the Corresponding
 *   Source.
 */

/* global require, URL */

const fileUrl = require("file-url");
const args = require("yargs")
	.wrap(null)
	.command("$0 [url] [output]", "Save a page into a single HTML file.", yargs => {
		yargs.positional("url", { description: "URL or path on the filesystem of the page to save", type: "string" });
		yargs.positional("output", { description: "Output filename", type: "string" });
	})
	.default({
		"back-end": "puppeteer",
		"browser-headless": true,
		"browser-executable-path": "",
		"browser-width": 1280,
		"browser-height": 720,
		"browser-load-max-time": 60000,
		"browser-wait-until": "networkidle0",
		"browser-wait-until-fallback": true,
		"browser-debug": false,
		"browser-extensions": [],
		"browser-scripts": [],
		"browser-args": "",
		"compress-CSS": false,
		"compress-HTML": true,
		"filename-template": "{page-title} ({date-iso} {time-locale}).html",
		"filename-replacement-character": "_",
		"group-duplicate-images": true,
		"include-infobar": false,
		"load-deferred-images": true,
		"load-deferred-images-max-idle-time": 1500,
		"maxParallelWorkers": 8,
		"max-resource-size-enabled": false,
		"max-resource-size": 10,
		"remove-hidden-elements": true,
		"remove-unused-styles": true,
		"remove-unused-fonts": true,
		"remove-frames": false,
		"remove-imports": true,
		"remove-scripts": true,
		"remove-audio-src": true,
		"remove-video-src": true,
		"remove-alternative-fonts": true,
		"remove-alternative-medias": true,
		"remove-alternative-images": true,
		"save-raw-page": false,
		"web-driver-executable-path": "",
		"user-script-enabled": true,
		"include-BOM": false,
		"crawl-links": false,
		"crawl-inner-links-only": true,
		"crawl-max-depth": 1,
		"url-rewrite-rules": []
	})
	.options("back-end", { description: "Back-end to use" })
	.choices("back-end", ["jsdom", "puppeteer", "webdriver-chromium", "webdriver-gecko", "puppeteer-firefox"])
	.options("browser-headless", { description: "Run the browser in headless mode (puppeteer, puppeteer-firefox, webdriver-gecko, webdriver-chromium)" })
	.boolean("browser-headless")
	.options("browser-executable-path", { description: "Path to chrome/chromium executable (puppeteer, puppeteer-firefox, webdriver-gecko, webdriver-chromium)" })
	.string("browser-executable-path")
	.options("browser-width", { description: "Width of the browser viewport in pixels" })
	.number("browser-width")
	.options("browser-height", { description: "Height of the browser viewport in pixels" })
	.number("browser-height")
	.options("browser-load-max-time", { description: "Maximum delay of time to wait for page loading in ms (puppeteer, puppeteer-firefox, webdriver-gecko, webdriver-chromium)" })
	.number("browser-load-max-time")
	.options("browser-wait-until", { description: "When to consider the page is loaded (puppeteer, puppeteer-firefox, webdriver-gecko, webdriver-chromium)" })
	.choices("browser-wait-until", ["networkidle0", "networkidle2", "load", "domcontentloaded"])
	.options("browser-wait-until-fallback", { description: "Retry with the next value of --browser-wait-until when a timeout error is thrown" })
	.boolean("browser-wait-until-fallback")
	.options("browser-debug", { description: "Enable debug mode (puppeteer, puppeteer-firefox, webdriver-gecko, webdriver-chromium)" })
	.boolean("browser-debug")
	.options("browser-extensions", { description: "List of extension paths separated by a space and relative to the 'cli' folder (webdriver-gecko, webdriver-chromium)" })
	.array("browser-extensions")
	.options("browser-scripts", { description: "List of script paths separated by a space and relative to the 'cli' folder. They will be executed in all the frames." })
	.array("browser-scripts")
	.options("browser-args", { description: "Arguments provided as a JSON array and passed to the browser (puppeteer, puppeteer-firefox, webdriver-gecko, webdriver-chromium)" })
	.string("browser-args")
	.options("compress-CSS", { description: "Compress CSS stylesheets" })
	.boolean("compress-CSS")
	.options("compress-HTML", { description: "Compress HTML content" })
	.boolean("compress-HTML")
	.options("crawl-links", { description: "Crawl and save pages found via inner links" })
	.boolean("crawl-links")
	.options("crawl-inner-links-only", { description: "Crawl pages found via inner links only if they are hosted on the same domain" })
	.boolean("crawl-inner-links-only")
	.options("crawl-max-depth", { description: "Max depth when crawl pages found via inner links" })
	.number("crawl-max-depth")
	.options("error-file")
	.string("error-file")
	.options("filename-template", { description: "Template used to generate the output filename (see help page of the extension for more info)" })
	.string("filename-template")
	.options("filename-replacement-character", { description: "The character used for replacing invalid characters in filenames" })
	.string("filename-replacement-character")
	.string("filename-template")
	.options("group-duplicate-images", { description: "Group duplicate images into CSS custom properties" })
	.boolean("group-duplicate-images")
	.options("include-BOM", { description: "Include the UTF-8 BOM into the HTML page" })
	.boolean("include-BOM")
	.options("include-infobar", { description: "Include the infobar" })
	.boolean("include-infobar")
	.options("load-deferred-images", { description: "Load deferred (a.k.a. lazy-loaded) images (puppeteer, puppeteer-firefox, webdriver-gecko, webdriver-chromium)" })
	.boolean("load-deferred-images")
	.options("load-deferred-images-max-idle-time", { description: "Maximum delay of time to wait for deferred images in ms (puppeteer, puppeteer-firefox, webdriver-gecko, webdriver-chromium)" })
	.number("load-deferred-images-max-idle-time")
	.options("max-parallel-workers", { description: "Maximum number of browsers launched in parallel when processing a list of URLs (cf --urls-file)" })
	.number("max-parallel-workers")
	.options("max-resource-size-enabled", { description: "Enable removal of embedded resources exceeding a given size" })
	.boolean("max-resource-size-enabled")
	.options("max-resource-size", { description: "Maximum size of embedded resources in MB (i.e. images, stylesheets, scripts and iframes)" })
	.number("max-resource-size")
	.options("remove-frames", { description: "Remove frames (puppeteer, puppeteer-firefox, webdriver-gecko, webdriver-chromium)" })
	.boolean("remove-frames")
	.options("remove-hidden-elements", { description: "Remove HTML elements which are not displayed" })
	.boolean("remove-hidden-elements")
	.options("remove-unused-styles", { description: "Remove unused CSS rules and unneeded declarations" })
	.boolean("remove-unused-styles")
	.options("remove-unused-fonts", { description: "Remove unused CSS font rules" })
	.boolean("remove-unused-fonts")
	.options("remove-imports", { description: "Remove HTML imports" })
	.boolean("remove-imports")
	.options("remove-scripts", { description: "Remove JavaScript scripts" })
	.boolean("remove-scripts")
	.options("remove-audio-src", { description: "Remove source of audio elements" })
	.boolean("remove-audio-src")
	.options("remove-video-src", { description: "Remove source of video elements" })
	.boolean("remove-video-src")
	.options("remove-alternative-fonts", { description: "Remove alternative fonts to the ones displayed" })
	.boolean("remove-alternative-fonts")
	.options("remove-alternative-medias", { description: "Remove alternative CSS stylesheets" })
	.boolean("remove-alternative-medias")
	.options("remove-alternative-images", { description: "Remove images for alternative sizes of screen" })
	.boolean("remove-alternative-images")
	.options("save-raw-page", { description: "Save the original page without interpreting it into the browser (puppeteer, puppeteer-firefox, webdriver-gecko, webdriver-chromium)" })
	.boolean("save-raw-page")
	.options("url-rewrite-rules", { description: "List of rewrite rules used to rewrite URLs" })
	.array("url-rewrite-rules")
	.options("urls-file", { description: "Path to a text file containing a list of URLs (separated by a newline) to save" })
	.string("urls-file")
	.options("user-agent", { description: "User-agent of the browser (puppeteer, webdriver-gecko, webdriver-chromium)" })
	.string("user-agent")
	.options("user-script-enabled", { description: "Enable the event API allowing to execute scripts before the page is saved" })
	.boolean("user-script-enabled")
	.options("web-driver-executable-path", { description: "Path to Selenium WebDriver executable (webdriver-gecko, webdriver-chromium)" })
	.string("web-driver-executable-path")
	.argv;

const fs = require("fs");
const backEnds = {
	jsdom: "./back-ends/jsdom.js",
	puppeteer: "./back-ends/puppeteer.js",
	"puppeteer-firefox": "./back-ends/puppeteer-firefox.js",
	"webdriver-chromium": "./back-ends/webdriver-chromium.js",
	"webdriver-gecko": "./back-ends/webdriver-gecko.js"
};
args.compressCSS = args.compressCss;
args.compressHTML = args.compressHtml;
args.includeBOM = args.includeBom;
if (args.url && !/^(https?|file):\/\//.test(args.url)) {
	args.url = fileUrl(args.url);
}
args.retrieveLinks = true;
args.browserScripts = args.browserScripts.map(path => require.resolve(path));
const backend = require(backEnds[args.backEnd]);
backend.initialize(args).then(() => {
	let tasks;
	if (args.urlsFile) {
		tasks = fs.readFileSync(args.urlsFile).toString().split("\n")
			.map(url => ({ url: rewriteURL(url, args.urlRewriteRules), depth: 0 }))
			.filter(task => task.url);
	} else {
		tasks = [{ url: rewriteURL(args.url, args.urlRewriteRules), depth: 0 }];
	}
	return runTasks(tasks, args);
}).then(() => {
	if (!args.browserDebug) {
		return backend.closeBrowser();
	}
});

async function runTasks(tasks, options) {
	const availableTasks = tasks.filter(task => !task.status).length;
	const processingTasks = tasks.filter(task => task.status == "processing").length;
	const promisesTasks = [];
	for (let workerIndex = 0; workerIndex < Math.min(availableTasks, options.maxParallelWorkers - processingTasks); workerIndex++) {
		promisesTasks.push(runNextTask(tasks, options));
	}
	await Promise.all(promisesTasks);
}

async function runNextTask(tasks, options) {
	const task = tasks.find(task => !task.status);
	if (task) {
		options = JSON.parse(JSON.stringify(options));
		options.url = task.url;
		options.output = null;
		task.status = "processing";
		const pageData = await capturePage(options);
		task.status = "processed";
		if (pageData && options.crawlLinks) {
			pageData.links = pageData.links
				.map(urlLink => rewriteURL(urlLink, options.urlRewriteRules))
				.filter(urlLink => !tasks.find(task => task.url == urlLink));
			if (options.crawlInnerLinksOnly) {
				const urlHost = getHostURL(options.url);
				pageData.links = pageData.links.filter(urlLink => urlLink.startsWith(urlHost));
			}
			if (task.depth < options.crawlMaxDepth) {
				tasks.splice(tasks.length, 0, ...pageData.links.map(url => ({ url, depth: task.depth + 1 })));
			}
		}
		await runTasks(tasks, options);
	}
}

function rewriteURL(url, rewriteRules) {
	url = url.trim();
	rewriteRules.forEach(rewriteRule => {
		const parts = rewriteRule.split(/ +/);
		if (parts.length == 2) {
			url = url.replace(new RegExp(parts[0]), parts[1]).trim();
		}
	});
	return url;
}

function getHostURL(url) {
	url = new URL(url);
	return url.protocol + "//" + (url.username ? url.username + (url.password || "") + "@" : "") + url.hostname;
}

async function capturePage(options) {
	try {
		const pageData = await backend.getPageData(options);
		if (options.output) {
			fs.writeFileSync(getFilename(options.output), pageData.content);
		} else {
			if (options.filenameTemplate && pageData.filename) {
				fs.writeFileSync(getFilename(pageData.filename), pageData.content);
			} else {
				console.log(pageData.content); // eslint-disable-line no-console
			}
		}
		return pageData;
	} catch (error) {
		const message = "URL: " + options.url + "\nStack: " + error.stack + "\n";
		if (options.errorFile) {
			fs.writeFileSync(options.errorFile, message, { flag: "a" });
		} else {
			console.error(message); // eslint-disable-line no-console
		}
	}
}

function getFilename(filename, index = 1) {
	let newFilename = filename;
	if (index > 1) {
		const regExpMatchExtension = /(\.[^.]+)$/;
		const matchExtension = newFilename.match(regExpMatchExtension);
		if (matchExtension && matchExtension[1]) {
			newFilename = newFilename.replace(regExpMatchExtension, " - " + index + matchExtension[1]);
		} else {
			newFilename += " - " + index;
		}
	}
	if (fs.existsSync(newFilename)) {
		return getFilename(filename, index + 1);
	} else {
		return newFilename;
	}
}