Merge pull request #11 from gildas-lormeau/master

upd

_locales/de/messages.json

@@ -19,6 +19,10 @@
 		"message": "Annotieren und Speichern der Webseite...",
 		"description": "Menu entry: 'Annotate and save the page...'"
 	},
+	"menuSaveWithProfile": {
+		"message": "Speichern der Webseite mit Profil",
+		"description": "Menu entry: 'Save with profile'"
+	},
 	"menuViewPendingSaves": {
 		"message": "Anzeigen der aktuellen Speichern...",
 		"description": "Menu entry: 'View pending saves...'"
@@ -370,7 +374,7 @@
 	"optionAutoSaveExternalSave": {
 		"message": "Speichern der Seite mit SingleFile Companion",
 		"description": "Options page label: 'save the page with SingleFile Companion'"
-	},	
+	},
 	"optionsEditorSubTitle": {
 		"message": "Annotation editor",
 		"description": "Options sub-title: 'Annotation editor'"

_locales/en/messages.json

@@ -15,6 +15,10 @@
 		"message": "Annotate and save the page...",
 		"description": "Menu entry: 'Annotate and save the page...'"
 	},
+	"menuSaveWithProfile": {
+		"message": "Save page with profile",
+		"description": "Menu entry: 'Save with profile'"
+	},
 	"menuSaveSelectedLinks": {
 		"message": "Save selected links",
 		"description": "Menu entry: 'Save selected links'"

_locales/es/messages.json

@@ -19,6 +19,10 @@
 		"message": "Anotar y guardar la página",
 		"description": "Menu entry: 'Annotate and save the page...'"
 	},
+	"menuSaveWithProfile": {
+		"message": "Guardar página con el perfil",
+		"description": "Menu entry: 'Save with profile'"
+	},
 	"menuViewPendingSaves": {
 		"message": "View pending saves...",
 		"description": "Menu entry: 'View pending saves...'"
@@ -370,7 +374,7 @@
 	"optionAutoSaveExternalSave": {
 		"message": "guardar la página con SingleFile Companion",
 		"description": "Options page label: 'save the page with SingleFile Companion'"
-	},	
+	},
 	"optionsEditorSubTitle": {
 		"message": "Annotation editor",
 		"description": "Options sub-title: 'Annotation editor'"

_locales/fr/messages.json

@@ -19,6 +19,10 @@
 		"message": "Annoter et sauver la page...",
 		"description": "Menu entry: 'Annotate and save the page...'"
 	},
+	"menuSaveWithProfile": {
+		"message": "Sauver la page avec le profil",
+		"description": "Menu entry: 'Save with profile'"
+	},
 	"menuViewPendingSaves": {
 		"message": "Afficher les sauvegardes en cours...",
 		"description": "Menu entry: 'View pending saves...'"

_locales/ja/messages.json

@@ -19,6 +19,10 @@
 		"message": "ページに注釈を付けて保存する...",
 		"description": "Menu entry: 'Annotate and save the page...'"
 	},
+	"menuSaveWithProfile": {
+		"message": "Save page with profile",
+		"description": "Menu entry: 'Save with profile'"
+	},
 	"menuViewPendingSaves": {
 		"message": "保留中の保存を表示...",
 		"description": "Menu entry: 'View pending saves...'"

_locales/pl/messages.json

@@ -19,6 +19,10 @@
 		"message": "Adnotuj i zapisz stronę...",
 		"description": "Menu entry: 'Annotate and save the page...'"
 	},
+	"menuSaveWithProfile": {
+		"message": "Zapisz stronę z profilem",
+		"description": "Menu entry: 'Save with profile'"
+	},
 	"menuViewPendingSaves": {
 		"message": "Zobacz oczekujące zapisy...",
 		"description": "Menu entry: 'View pending saves...'"

_locales/ru/messages.json

@@ -19,6 +19,10 @@
 		"message": "Аннотировать и сохранить страницу...",
 		"description": "Menu entry: 'Annotate and save the page...'"
 	},
+	"menuSaveWithProfile": {
+		"message": "Save page with profile",
+		"description": "Menu entry: 'Save with profile'"
+	},
 	"menuViewPendingSaves": {
 		"message": "Просмотр отложенных сохранений...",
 		"description": "Menu entry: 'View pending saves...'"

_locales/uk/messages.json

@@ -19,6 +19,10 @@
 		"message": "Анотувати і зберегти сторінку...",
 		"description": "Menu entry: 'Annotate and save the page...'"
 	},
+	"menuSaveWithProfile": {
+		"message": "Save page with profile",
+		"description": "Menu entry: 'Save with profile'"
+	},
 	"menuViewPendingSaves": {
 		"message": "View pending saves...",
 		"description": "Menu entry: 'View pending saves...'"

_locales/zh_CN/messages.json

@@ -12,13 +12,17 @@
 		"description": "Menu entry: 'Save selected links'"
 	},
 	"menuEditPage": {
-		"message": "Annotate the page...",
+		"message": "标注页面...",
 		"description": "Menu entry: 'Annotate the page...'"
 	},
 	"menuEditAndSavePage": {
 		"message": "标注并保存该页面...",
 		"description": "Menu entry: 'Annotate and save the page...'"
 	},
+	"menuSaveWithProfile": {
+		"message": "使用指定的配置文件保存页面",
+		"description": "Menu entry: 'Save with profile'"
+	},
 	"menuViewPendingSaves": {
 		"message": "查看挂起的保存项...",
 		"description": "Menu entry: 'View pending saves...'"
@@ -368,11 +372,11 @@
 		"description": "Options page label: 'period (s)'"
 	},
 	"optionAutoSaveExternalSave": {
-		"message": "save the page with SingleFile Companion",
+		"message": "使用 SingleFile Companion 保存页面",
 		"description": "Options page label: 'save the page with SingleFile Companion'"
 	},	
 	"optionsEditorSubTitle": {
-		"message": "Annotation editor",
+		"message": "标注编辑器",
 		"description": "Options sub-title: 'Annotation editor'"
 	},
 	"optionOpenEditor": {
@@ -388,7 +392,7 @@
 		"description": "Title of the button 'apply the system theme when formatting a page'"
 	},
 	"optionWarnUnsavedPage": {
-		"message": "warn if leaving page with unsaved changes",
+		"message": "离开标注编辑器时提醒保存已修改的内容",
 		"description": "Title of the button 'warn if leaving page with unsaved changes'"
 	},
 	"optionsMiscSubTitle": {
@@ -420,7 +424,7 @@
 		"description": "Options page label: 'save to clipboard'"
 	},
 	"optionAddProof": {
-		"message": "添加证明文件存在指纹",
+		"message": "添加证明文件存在的指纹",
 		"description": "Options page label: 'add proof of existence'"
 	},
 	"optionsAddProofConfirm": {
@@ -520,7 +524,7 @@
 		"description": "Tooltip 'Delete the profile' in the options page"
 	},
 	"profileAddPrompt": {
-		"message": "请为这个新的配置文件输入名称",
+		"message": "请输入这个新的配置文件名",
 		"description": "Popup text 'Enter a name for this new profile' in the options page"
 	},
 	"profileDeleteConfirm": {

_locales/zh_TW/messages.json

@@ -12,13 +12,17 @@
 		"description": "Menu entry: 'Save selected links'"
 	},
 	"menuEditPage": {
-		"message": "Annotate the page...",
+		"message": "標註頁面...",
 		"description": "Menu entry: 'Annotate the page...'"
 	},
 	"menuEditAndSavePage": {
 		"message": "標註並保存該頁面...",
 		"description": "Menu entry: 'Annotate and save the page...'"
 	},
+	"menuSaveWithProfile": {
+		"message": "使用指定的配置文件保存頁面",
+		"description": "Menu entry: 'Save with profile'"
+	},
 	"menuViewPendingSaves": {
 		"message": "查看掛起的保存項...",
 		"description": "Menu entry: 'View pending saves...'"
@@ -368,11 +372,11 @@
 		"description": "Options page label: 'period (s)'"
 	},
 	"optionAutoSaveExternalSave": {
-		"message": "save the page with SingleFile Companion",
+		"message": "使用 SingleFile Companion 保存頁面",
 		"description": "Options page label: 'save the page with SingleFile Companion'"
-	},
+	},	
 	"optionsEditorSubTitle": {
-		"message": "Annotation editor",
+		"message": "標註編輯器",
 		"description": "Options sub-title: 'Annotation editor'"
 	},
 	"optionOpenEditor": {
@@ -388,7 +392,7 @@
 		"description": "Title of the button 'apply the system theme when formatting a page'"
 	},
 	"optionWarnUnsavedPage": {
-		"message": "warn if leaving page with unsaved changes",
+		"message": "離開標註編輯器時提醒保存已修改的內容",
 		"description": "Title of the button 'warn if leaving page with unsaved changes'"
 	},
 	"optionsMiscSubTitle": {
@@ -420,7 +424,7 @@
 		"description": "Options page label: 'save to clipboard'"
 	},
 	"optionAddProof": {
-		"message": "添加證明文件存在指紋",
+		"message": "添加證明文件存在的指紋",
 		"description": "Options page label: 'add proof of existence'"
 	},
 	"optionsAddProofConfirm": {
@@ -520,7 +524,7 @@
 		"description": "Tooltip 'Delete the profile' in the options page"
 	},
 	"profileAddPrompt": {
-		"message": "請為這個新的配置文件輸入名稱",
+		"message": "請輸入這個新的配置文件名",
 		"description": "Popup text 'Enter a name for this new profile' in the options page"
 	},
 	"profileDeleteConfirm": {

cli/README.MD

@@ -10,25 +10,37 @@ SingleFile can be launched from the command line by running it into a (headless)
 
 - Install [Node.js](https://nodejs.org)
 
-- Unzip the [master archive](https://github.com/gildas-lormeau/SingleFile/archive/master.zip) somewhere on your disk in an empty folder and go into the `SingleFile-master/cli` directory.
+- There are 3 ways to download the code of SingleFile, choose the one you prefer (`npm` is installed with Node.js):
 
-  `unzip master.zip .`
+  - Download and unzip manually the [master archive](https://github.com/gildas-lormeau/SingleFile/archive/master.zip) provided by Github      
+
+    `unzip master.zip .`
   
-  `cd SingleFile-master`
+    `cd SingleFile-master`
   
-  `cd cli`
+    `cd cli`
+      
+    `npm install`
+    
+  - Download with `npm`
+    
+    `npm install 'gildas-lormeau/SingleFile#master'`
+       
+    `cd node-modules`
+       
+    `cd single-file`
+       
+    `cd cli`  
   
-- As an alternative to decompressing the master archive, you can clone the repository if `git` is installed on your machine and go into the `SingleFile/cli` directory.
+  - Download with `git`
 
-  `git clone --depth 1 --recursive https://github.com/gildas-lormeau/SingleFile.git`
+    `git clone --depth 1 --recursive https://github.com/gildas-lormeau/SingleFile.git`
   
-  `cd SingleFile`
+    `cd SingleFile`
   
-  `cd cli`
-
-- Install dependencies with npm (installed with Node.js).
-
-  `npm install`
+    `cd cli`
+       
+    `npm install`        
   
 - Make `single-file` executable (Linux/Unix/BSD etc.).
 
@@ -50,7 +62,7 @@ SingleFile can be launched from the command line by running it into a (headless)
 
   - Dump the processed content of https://www.wikipedia.org into the console
 
-  `single-file https://www.wikipedia.org`
+  `single-file https://www.wikipedia.org` --filename-template=""
 
   - Save https://www.wikipedia.org into `wikipedia.html`
 

cli/single-file

@@ -23,175 +23,26 @@
  *   Source.
  */
 
-/* global require, URL */
-
-const VALID_URL_TEST = /^(https?|file):\/\//;
+/* global require */
 
 const fileUrl = require("file-url");
 const fs = require("fs");
-const options = require("./args");
-
-const backEnds = {
-	jsdom: "./back-ends/jsdom.js",
-	puppeteer: "./back-ends/puppeteer.js",
-	"puppeteer-firefox": "./back-ends/puppeteer-firefox.js",
-	"webdriver-chromium": "./back-ends/webdriver-chromium.js",
-	"webdriver-gecko": "./back-ends/webdriver-gecko.js"
-};
-if (options.url && !VALID_URL_TEST.test(options.url)) {
-	options.url = fileUrl(options.url);
-}
-options.retrieveLinks = true;
-options.browserScripts = options.browserScripts.map(path => require.resolve(path));
-const backend = require(backEnds[options.backEnd]);
-run(options);
+run(require("./args"));
 
 async function run(options) {
-	await backend.initialize(options);
-	let tasks;
-	if (options.urlsFile) {
-		tasks = fs.readFileSync(options.urlsFile).toString().split("\n")
-			.map(url => createTask(url));
-	} else {
-		tasks = [createTask(options.url)];
-	}
-	tasks = tasks.filter(task => task);
-	await runTasks(tasks);
-	if (options.crawlReplaceURLs) {
-		tasks.forEach(task => {
-			try {
-				let pageContent = fs.readFileSync(task.filename).toString();
-				tasks.forEach(otherTask => {
-					pageContent = pageContent.replace(new RegExp(escapeRegExp("\"" + otherTask.originalUrl + "\""), "gi"), "\"" + otherTask.filename + "\"");
-					pageContent = pageContent.replace(new RegExp(escapeRegExp("'" + otherTask.originalUrl + "'"), "gi"), "'" + otherTask.filename + "'");
-					const filename = otherTask.filename.replace(/ /g, "%20");
-					pageContent = pageContent.replace(new RegExp(escapeRegExp("=" + otherTask.originalUrl + " "), "gi"), "=" + filename + " ");
-					pageContent = pageContent.replace(new RegExp(escapeRegExp("=" + otherTask.originalUrl + ">"), "gi"), "=" + filename + ">");
-				});
-				fs.writeFileSync(task.filename, pageContent);
-			} catch (error) {
-				// ignored
-			}
-		});
-	}
-	if (!options.browserDebug) {
-		return backend.closeBrowser();
+	const singlefile = await require("./singlefile-cli-api")(options);
+	let urls;
+	if (options.url && !singlefile.VALID_URL_TEST.test(options.url)) {
+		options.url = fileUrl(options.url);
 	}
-}
-
-async function runTasks(tasks) {
-	const availableTasks = tasks.filter(task => !task.status).length;
-	const processingTasks = tasks.filter(task => task.status == "processing").length;
-	const promisesTasks = [];
-	for (let workerIndex = 0; workerIndex < Math.min(availableTasks, options.maxParallelWorkers - processingTasks); workerIndex++) {
-		promisesTasks.push(runNextTask(tasks));
-	}
-	return Promise.all(promisesTasks);
-}
-
-async function runNextTask(tasks) {
-	const task = tasks.find(task => !task.status);
-	if (task) {
-		let taskOptions = JSON.parse(JSON.stringify(options));
-		taskOptions.url = task.url;
-		task.status = "processing";
-		const pageData = await capturePage(taskOptions);
-		task.status = "processed";
-		if (pageData) {
-			task.filename = pageData.filename;
-			if (options.crawlLinks && testMaxDepth(task)) {
-				let newTasks = pageData.links
-					.map(urlLink => createTask(urlLink, task, tasks[0]))
-					.filter(task => task &&
-						testMaxDepth(task) &&
-						!tasks.find(otherTask => otherTask.url == task.url) &&
-						(!options.crawlInnerLinksOnly || task.isInnerLink));
-				tasks.splice(tasks.length, 0, ...newTasks);
-			}
-		}
-		await runTasks(tasks);
-	}
-}
-
-function testMaxDepth(task) {
-	return (options.crawlMaxDepth == 0 || task.depth < options.crawlMaxDepth) &&
-		(options.crawlExternalLinksMaxDepth == 0 || task.externalLinkDepth < options.crawlExternalLinksMaxDepth);
-}
-
-function createTask(url, parentTask, rootTask) {
-	url = parentTask ? rewriteURL(url) : url;
-	if (VALID_URL_TEST.test(url)) {
-		const isInnerLink = rootTask && url.startsWith(getHostURL(rootTask.url));
-		return {
-			url,
-			isInnerLink,
-			originalUrl: url,
-			depth: parentTask ? parentTask.depth + 1 : 0,
-			externalLinkDepth: isInnerLink ? -1 : parentTask ? parentTask.externalLinkDepth + 1 : -1
-		};
-	}
-}
-
-function rewriteURL(url) {
-	url = url.trim();
-	if (options.crawlRemoveURLFragment) {
-		url = url.replace(/^(.*?)#.*$/, "$1");
-	}
-	options.crawlRewriteRules.forEach(rewriteRule => {
-		const parts = rewriteRule.trim().split(/ +/);
-		if (parts.length) {
-			url = url.replace(new RegExp(parts[0]), parts[1] || "").trim();
-		}
-	});
-	return url;
-}
-
-function getHostURL(url) {
-	url = new URL(url);
-	return url.protocol + "//" + (url.username ? url.username + (url.password || "") + "@" : "") + url.hostname;
-}
-
-async function capturePage(options) {
-	try {
-		const pageData = await backend.getPageData(options);
-		if (options.output) {
-			fs.writeFileSync(getFilename(options.output), pageData.content);
-		} else {
-			if (options.filenameTemplate && pageData.filename) {
-				fs.writeFileSync(getFilename(pageData.filename), pageData.content);
-			} else {
-				console.log(pageData.content); // eslint-disable-line no-console
-			}
-		}
-		return pageData;
-	} catch (error) {
-		const message = "URL: " + options.url + "\nStack: " + error.stack + "\n";
-		if (options.errorFile) {
-			fs.writeFileSync(options.errorFile, message, { flag: "a" });
-		} else {
-			console.error(message); // eslint-disable-line no-console
-		}
-	}
-}
-
-function getFilename(filename, index = 1) {
-	let newFilename = filename;
-	if (index > 1) {
-		const regExpMatchExtension = /(\.[^.]+)$/;
-		const matchExtension = newFilename.match(regExpMatchExtension);
-		if (matchExtension && matchExtension[1]) {
-			newFilename = newFilename.replace(regExpMatchExtension, " - " + index + matchExtension[1]);
-		} else {
-			newFilename += " - " + index;
-		}
-	}
-	if (fs.existsSync(newFilename)) {
-		return getFilename(filename, index + 1);
+	if (options.urlsFile) {
+		urls = fs.readFileSync(options.urlsFile).toString().split("\n");
 	} else {
-		return newFilename;
+		urls = [options.url];
 	}
-}
+	options.retrieveLinks = true;
+	options.browserScripts = options.browserScripts.map(path => require.resolve(path));
 
-function escapeRegExp(string) {
-	return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+	await singlefile.capture(urls);
+	await singlefile.finish();
 }

cli/singlefile-cli-api.js

@@ -0,0 +1,207 @@
+#!/usr/bin/env node
+
+/*
+ * Copyright 2010-2020 Gildas Lormeau
+ * contact : gildas.lormeau <at> gmail.com
+ *
+ * This file is part of SingleFile.
+ *
+ *   The code in this file is free software: you can redistribute it and/or
+ *   modify it under the terms of the GNU Affero General Public License
+ *   (GNU AGPL) as published by the Free Software Foundation, either version 3
+ *   of the License, or (at your option) any later version.
+ *
+ *   The code in this file is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+ *   General Public License for more details.
+ *
+ *   As additional permission under GNU AGPL version 3 section 7, you may
+ *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU
+ *   AGPL normally required by section 4, provided you include this license
+ *   notice and a URL through which recipients can access the Corresponding
+ *   Source.
+ */
+
+/* global require, module, URL */
+
+const fs = require("fs");
+const VALID_URL_TEST = /^(https?|file):\/\//;
+
+const backEnds = {
+	jsdom: "./back-ends/jsdom.js",
+	puppeteer: "./back-ends/puppeteer.js",
+	"puppeteer-firefox": "./back-ends/puppeteer-firefox.js",
+	"webdriver-chromium": "./back-ends/webdriver-chromium.js",
+	"webdriver-gecko": "./back-ends/webdriver-gecko.js"
+};
+
+let backend, tasks = [], maxParallelWorkers = 8;
+module.exports = initialize;
+
+async function initialize(options) {
+	maxParallelWorkers = options.maxParallelWorkers;
+	backend = require(backEnds[options.backEnd]);
+	await backend.initialize(options);
+	return {
+		capture: urls => capture(urls, options),
+		finish: () => finish(options),
+		VALID_URL_TEST
+	};
+}
+
+async function capture(urls, options) {
+	let newTasks;
+	newTasks = urls.map(url => createTask(url, options));
+	newTasks = newTasks.filter(task => task);
+	if (newTasks.length) {
+		tasks = tasks.concat(newTasks);
+		await runTasks();
+	}
+}
+
+async function finish(options) {
+	const promiseTasks = tasks.map(task => task.promise);
+	await Promise.all(promiseTasks);
+	if (options.crawlReplaceURLs) {
+		tasks.forEach(task => {
+			try {
+				let pageContent = fs.readFileSync(task.filename).toString();
+				tasks.forEach(otherTask => {
+					pageContent = pageContent.replace(new RegExp(escapeRegExp("\"" + otherTask.originalUrl + "\""), "gi"), "\"" + otherTask.filename + "\"");
+					pageContent = pageContent.replace(new RegExp(escapeRegExp("'" + otherTask.originalUrl + "'"), "gi"), "'" + otherTask.filename + "'");
+					const filename = otherTask.filename.replace(/ /g, "%20");
+					pageContent = pageContent.replace(new RegExp(escapeRegExp("=" + otherTask.originalUrl + " "), "gi"), "=" + filename + " ");
+					pageContent = pageContent.replace(new RegExp(escapeRegExp("=" + otherTask.originalUrl + ">"), "gi"), "=" + filename + ">");
+				});
+				fs.writeFileSync(task.filename, pageContent);
+			} catch (error) {
+				// ignored
+			}
+		});
+	}
+	if (!options.browserDebug) {
+		return backend.closeBrowser();
+	}
+}
+
+async function runTasks() {
+	const availableTasks = tasks.filter(task => !task.status).length;
+	const processingTasks = tasks.filter(task => task.status == "processing").length;
+	const promisesTasks = [];
+	for (let workerIndex = 0; workerIndex < Math.min(availableTasks, maxParallelWorkers - processingTasks); workerIndex++) {
+		promisesTasks.push(runNextTask());
+	}
+	return Promise.all(promisesTasks);
+}
+
+async function runNextTask() {
+	const task = tasks.find(task => !task.status);
+	if (task) {
+		const options = task.options;
+		let taskOptions = JSON.parse(JSON.stringify(options));
+		taskOptions.url = task.url;
+		task.status = "processing";
+		task.promise = capturePage(taskOptions);
+		const pageData = await task.promise;
+		task.status = "processed";
+		if (pageData) {
+			task.filename = pageData.filename;
+			if (options.crawlLinks && testMaxDepth(task)) {
+				let newTasks = pageData.links
+					.map(urlLink => createTask(urlLink, options, task, tasks[0]))
+					.filter(task => task &&
+						testMaxDepth(task) &&
+						!tasks.find(otherTask => otherTask.url == task.url) &&
+						(!options.crawlInnerLinksOnly || task.isInnerLink));
+				tasks.splice(tasks.length, 0, ...newTasks);
+			}
+		}
+		await runTasks();
+	}
+}
+
+function testMaxDepth(task) {
+	const options = task.options;
+	return (options.crawlMaxDepth == 0 || task.depth < options.crawlMaxDepth) &&
+		(options.crawlExternalLinksMaxDepth == 0 || task.externalLinkDepth < options.crawlExternalLinksMaxDepth);
+}
+
+function createTask(url, options, parentTask, rootTask) {
+	url = parentTask ? rewriteURL(url, options.crawlRemoveURLFragment, options.crawlRewriteRules) : url;
+	if (VALID_URL_TEST.test(url)) {
+		const isInnerLink = rootTask && url.startsWith(getHostURL(rootTask.url));
+		return {
+			url,
+			isInnerLink,
+			originalUrl: url,
+			depth: parentTask ? parentTask.depth + 1 : 0,
+			externalLinkDepth: isInnerLink ? -1 : parentTask ? parentTask.externalLinkDepth + 1 : -1,
+			options
+		};
+	}
+}
+
+function rewriteURL(url, crawlRemoveURLFragment, crawlRewriteRules) {
+	url = url.trim();
+	if (crawlRemoveURLFragment) {
+		url = url.replace(/^(.*?)#.*$/, "$1");
+	}
+	crawlRewriteRules.forEach(rewriteRule => {
+		const parts = rewriteRule.trim().split(/ +/);
+		if (parts.length) {
+			url = url.replace(new RegExp(parts[0]), parts[1] || "").trim();
+		}
+	});
+	return url;
+}
+
+function getHostURL(url) {
+	url = new URL(url);
+	return url.protocol + "//" + (url.username ? url.username + (url.password || "") + "@" : "") + url.hostname;
+}
+
+async function capturePage(options) {
+	try {
+		const pageData = await backend.getPageData(options);
+		if (options.output) {
+			fs.writeFileSync(getFilename(options.output), pageData.content);
+		} else {
+			if (options.filenameTemplate && pageData.filename) {
+				fs.writeFileSync(getFilename(pageData.filename), pageData.content);
+			} else {
+				console.log(pageData.content); // eslint-disable-line no-console
+			}
+		}
+		return pageData;
+	} catch (error) {
+		const message = "URL: " + options.url + "\nStack: " + error.stack + "\n";
+		if (options.errorFile) {
+			fs.writeFileSync(options.errorFile, message, { flag: "a" });
+		} else {
+			console.error(message); // eslint-disable-line no-console
+		}
+	}
+}
+
+function getFilename(filename, index = 1) {
+	let newFilename = filename;
+	if (index > 1) {
+		const regExpMatchExtension = /(\.[^.]+)$/;
+		const matchExtension = newFilename.match(regExpMatchExtension);
+		if (matchExtension && matchExtension[1]) {
+			newFilename = newFilename.replace(regExpMatchExtension, " - " + index + matchExtension[1]);
+		} else {
+			newFilename += " - " + index;
+		}
+	}
+	if (fs.existsSync(newFilename)) {
+		return getFilename(filename, index + 1);
+	} else {
+		return newFilename;
+	}
+}
+
+function escapeRegExp(string) {
+	return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}

extension/ui/bg/ui-menus.js

@@ -29,9 +29,11 @@ singlefile.extension.ui.bg.menus = (() => {
 	const BROWSER_MENUS_API_SUPPORTED = menus && menus.onClicked && menus.create && menus.update && menus.removeAll;
 	const MENU_ID_SAVE_PAGE = "save-page";
 	const MENU_ID_EDIT_AND_SAVE_PAGE = "edit-and-save-page";
-	const MENU_ID_SAVE_SELECTED_LINKS = "save-selectec-links";
+	const MENU_ID_SAVE_WITH_PROFILE = "save-with-profile";
+	const MENU_ID_SAVE_SELECTED_LINKS = "save-selected-links";
 	const MENU_ID_VIEW_PENDINGS = "view-pendings";
 	const MENU_ID_SELECT_PROFILE = "select-profile";
+	const MENU_ID_SAVE_WITH_PROFILE_PREFIX = "wasve-with-profile-";
 	const MENU_ID_SELECT_PROFILE_PREFIX = "select-profile-";
 	const MENU_ID_ASSOCIATE_WITH_PROFILE = "associate-with-profile";
 	const MENU_ID_ASSOCIATE_WITH_PROFILE_PREFIX = "associate-with-profile-";
@@ -52,6 +54,7 @@ singlefile.extension.ui.bg.menus = (() => {
 	const MENU_CREATE_DOMAIN_RULE_MESSAGE = browser.i18n.getMessage("menuCreateDomainRule");
 	const MENU_UPDATE_RULE_MESSAGE = browser.i18n.getMessage("menuUpdateRule");
 	const MENU_SAVE_PAGE_MESSAGE = browser.i18n.getMessage("menuSavePage");
+	const MENU_SAVE_WITH_PROFILE = browser.i18n.getMessage("menuSaveWithProfile");
 	const MENU_SAVE_SELECTED_LINKS = browser.i18n.getMessage("menuSaveSelectedLinks");
 	const MENU_EDIT_PAGE_MESSAGE = browser.i18n.getMessage("menuEditPage");
 	const MENU_EDIT_AND_SAVE_PAGE_MESSAGE = browser.i18n.getMessage("menuEditAndSavePage");
@@ -72,6 +75,7 @@ singlefile.extension.ui.bg.menus = (() => {
 	const MENU_TOP_VISIBLE_ENTRIES = [
 		MENU_ID_EDIT_AND_SAVE_PAGE,
 		MENU_ID_SAVE_SELECTED_LINKS,
+		MENU_ID_SAVE_WITH_PROFILE,
 		MENU_ID_VIEW_PENDINGS,
 		MENU_ID_SAVE_SELECTED,
 		MENU_ID_SAVE_FRAME,
@@ -144,6 +148,13 @@ singlefile.extension.ui.bg.menus = (() => {
 				contexts: options.contextMenuEnabled ? defaultContextsDisabled.concat(["selection"]) : defaultContextsDisabled,
 				title: MENU_SAVE_SELECTED_LINKS
 			});
+			if (Object.keys(profiles).length > 1) {
+				menus.create({
+					id: MENU_ID_SAVE_WITH_PROFILE,
+					contexts: defaultContexts,
+					title: MENU_SAVE_WITH_PROFILE
+				});
+			}
 			if (options.contextMenuEnabled) {
 				menus.create({
 					id: "separator-1",
@@ -214,6 +225,12 @@ singlefile.extension.ui.bg.menus = (() => {
 					title: MENU_SELECT_PROFILE_MESSAGE,
 					contexts: defaultContexts,
 				});
+				menus.create({
+					id: MENU_ID_SAVE_WITH_PROFILE_PREFIX + "default",
+					contexts: defaultContexts,
+					title: PROFILE_DEFAULT_SETTINGS_MESSAGE,
+					parentId: MENU_ID_SAVE_WITH_PROFILE
+				});
 				const defaultProfileId = MENU_ID_SELECT_PROFILE_PREFIX + "default";
 				const defaultProfileChecked = !tabsData.profileName || tabsData.profileName == config.DEFAULT_PROFILE_NAME;
 				menus.create({
@@ -261,7 +278,14 @@ singlefile.extension.ui.bg.menus = (() => {
 				profileIndexes = new Map();
 				Object.keys(profiles).forEach((profileName, profileIndex) => {
 					if (profileName != config.DEFAULT_PROFILE_NAME) {
-						let profileId = MENU_ID_SELECT_PROFILE_PREFIX + profileIndex;
+						let profileId = MENU_ID_SAVE_WITH_PROFILE_PREFIX + profileIndex;
+						menus.create({
+							id: profileId,
+							contexts: defaultContexts,
+							title: profileName,
+							parentId: MENU_ID_SAVE_WITH_PROFILE
+						});
+						profileId = MENU_ID_SELECT_PROFILE_PREFIX + profileIndex;
 						let profileChecked = tabsData.profileName == profileName;
 						menus.create({
 							id: profileId,
@@ -429,6 +453,18 @@ singlefile.extension.ui.bg.menus = (() => {
 					await tabsData.set(allTabsData);
 					refreshExternalComponents(tab);
 				}
+				if (event.menuItemId.startsWith(MENU_ID_SAVE_WITH_PROFILE_PREFIX)) {
+					const profiles = await config.getProfiles();
+					const profileId = event.menuItemId.split(MENU_ID_SAVE_WITH_PROFILE_PREFIX)[1];
+					let profileName;
+					if (profileId == "default") {
+						profileName = config.DEFAULT_PROFILE_NAME;
+					} else {
+						const profileIndex = Number(profileId);
+						profileName = Object.keys(profiles)[profileIndex];
+					}
+					business.saveTabs([tab], profiles[profileName]);
+				}
 				if (event.menuItemId.startsWith(MENU_ID_SELECT_PROFILE_PREFIX)) {
 					const [profiles, allTabsData] = await Promise.all([config.getProfiles(), tabsData.get()]);
 					const profileId = event.menuItemId.split(MENU_ID_SELECT_PROFILE_PREFIX)[1];

lib/single-file/single-file-core.js

@@ -345,19 +345,20 @@ this.singlefile.lib.core = this.singlefile.lib.core || (() => {
 				const [resourceURL, asBinary] = JSON.parse(requestKey);
 				const resourceRequests = this.requests.get(requestKey);
 				try {
+					const currentIndexResource = indexResource;
+					indexResource = indexResource + 1;
 					const content = await util.getContent(resourceURL, {
 						asBinary,
 						maxResourceSize: options.maxResourceSize,
 						maxResourceSizeEnabled: options.maxResourceSizeEnabled,
 						frameId: options.windowId
 					});
-					indexResource = indexResource + 1;
 					onloadListener({ url: resourceURL });
 					if (!this.cancelled) {
 						resourceRequests.forEach(callbacks => {
 							const duplicateCallbacks = this.duplicates.get(requestKey);
 							const duplicate = duplicateCallbacks && duplicateCallbacks.length > 1 && duplicateCallbacks.includes(callbacks);
-							callbacks.resolve({ content: content.data, indexResource, duplicate });
+							callbacks.resolve({ content: content.data, indexResource: currentIndexResource, duplicate });
 						});
 					}
 				} catch (error) {

lib/single-file/single-file-helper.js

@@ -31,6 +31,7 @@ this.singlefile.lib.helper = this.singlefile.lib.helper || (() => {
 	const ON_AFTER_CAPTURE_EVENT_NAME = "single-file-on-after-capture";
 	const REMOVED_CONTENT_ATTRIBUTE_NAME = "data-single-file-removed-content";
 	const HIDDEN_CONTENT_ATTRIBUTE_NAME = "data-single-file-hidden-content";
+	const KEPT_CONTENT_ATTRIBUTE_NAME = "data-single-file-kept-content";
 	const HIDDEN_FRAME_ATTRIBUTE_NAME = "data-single-file-hidden-frame";
 	const PRESERVED_SPACE_ELEMENT_ATTRIBUTE_NAME = "data-single-file-preserved-space-element";
 	const SHADOW_ROOT_ATTRIBUTE_NAME = "data-single-file-shadow-root-element";
@@ -46,7 +47,7 @@ this.singlefile.lib.helper = this.singlefile.lib.helper || (() => {
 	const SELECTED_CONTENT_ATTRIBUTE_NAME = "data-single-file-selected-content";
 	const ASYNC_SCRIPT_ATTRIBUTE_NAME = "data-single-file-async-script";
 	const FLOW_ELEMENTS_SELECTOR = "*:not(base):not(link):not(meta):not(noscript):not(script):not(style):not(template):not(title)";
-	const IGNORED_REMOVED_TAG_NAMES = ["NOSCRIPT", "DISABLED-NOSCRIPT", "META", "LINK", "STYLE", "TITLE", "TEMPLATE", "SOURCE", "OBJECT", "SCRIPT", "HEAD"];
+	const KEPT_TAG_NAMES = ["NOSCRIPT", "DISABLED-NOSCRIPT", "META", "LINK", "STYLE", "TITLE", "TEMPLATE", "SOURCE", "OBJECT", "SCRIPT", "HEAD"];
 	const REGEXP_SIMPLE_QUOTES_STRING = /^'(.*?)'$/;
 	const REGEXP_DOUBLE_QUOTES_STRING = /^"(.*?)"$/;
 	const FONT_WEIGHTS = {
@@ -151,19 +152,18 @@ this.singlefile.lib.helper = this.singlefile.lib.helper || (() => {
 	function getElementsInfo(win, doc, element, options, data = { usedFonts: new Map(), canvases: [], images: [], posters: [], shadowRoots: [], imports: [], markedElements: [] }, ascendantHidden) {
 		const elements = Array.from(element.childNodes).filter(node => node instanceof win.HTMLElement);
 		elements.forEach(element => {
-			let elementHidden, computedStyle;
+			let elementHidden, elementKept, computedStyle;
 			if (!options.autoSaveExternalSave && (options.removeHiddenElements || options.removeUnusedFonts || options.compressHTML)) {
 				computedStyle = win.getComputedStyle(element);
 				if (options.removeHiddenElements) {
-					if (ascendantHidden) {
-						Array.from(element.childNodes).filter(node => node instanceof win.HTMLElement).forEach(element => {
-							if (!IGNORED_REMOVED_TAG_NAMES.includes(element.tagName)) {
-								element.setAttribute(REMOVED_CONTENT_ATTRIBUTE_NAME, "");
-								data.markedElements.push(element);
-							}
-						});
+					elementKept = (ascendantHidden || element.closest("html > head")) && KEPT_TAG_NAMES.includes(element.tagName);
+					if (!elementKept) {
+						elementHidden = ascendantHidden || testHiddenElement(element, computedStyle);
+						if (elementHidden) {
+							element.setAttribute(HIDDEN_CONTENT_ATTRIBUTE_NAME, "");
+							data.markedElements.push(element);
+						}
 					}
-					elementHidden = ascendantHidden || testHiddenElement(element, computedStyle, data.markedElements);
 				}
 				if (!elementHidden) {
 					if (options.compressHTML && computedStyle) {
@@ -196,6 +196,15 @@ this.singlefile.lib.helper = this.singlefile.lib.helper || (() => {
 				}
 			}
 			getElementsInfo(win, doc, element, options, data, elementHidden);
+			if (!options.autoSaveExternalSave && options.removeHiddenElements && ascendantHidden) {
+				if (elementKept || element.getAttribute(KEPT_CONTENT_ATTRIBUTE_NAME) == "") {
+					element.parentElement.setAttribute(KEPT_CONTENT_ATTRIBUTE_NAME, "");
+					data.markedElements.push(element);
+				} else if (elementHidden) {
+					element.setAttribute(REMOVED_CONTENT_ATTRIBUTE_NAME, "");
+					data.markedElements.push(element);
+				}
+			}
 		});
 		return data;
 	}
@@ -316,19 +325,14 @@ this.singlefile.lib.helper = this.singlefile.lib.helper || (() => {
 		return removeQuotes(singlefile.lib.vendor.cssUnescape.process(fontFamilyName.trim())).toLowerCase();
 	}
 
-	function testHiddenElement(element, computedStyle, markedElements) {
+	function testHiddenElement(element, computedStyle) {
 		let hidden = false;
 		if (computedStyle) {
 			const display = computedStyle.getPropertyValue("display");
 			const opacity = computedStyle.getPropertyValue("opacity");
 			const visibility = computedStyle.getPropertyValue("visibility");
 			hidden = display == "none";
-			if (hidden) {
-				if (element.style.getPropertyValue("display") != "none" && !IGNORED_REMOVED_TAG_NAMES.includes(element.tagName)) {
-					element.setAttribute(HIDDEN_CONTENT_ATTRIBUTE_NAME, "");
-					markedElements.push(element);
-				}
-			} else if ((opacity == "0" || visibility == "hidden") && element.getBoundingClientRect) {
+			if (!hidden && (opacity == "0" || visibility == "hidden") && element.getBoundingClientRect) {
 				const boundingRect = element.getBoundingClientRect();
 				hidden = !boundingRect.width && !boundingRect.height;
 			}
@@ -355,6 +359,7 @@ this.singlefile.lib.helper = this.singlefile.lib.helper || (() => {
 		markedElements.forEach(element => {
 			element.removeAttribute(REMOVED_CONTENT_ATTRIBUTE_NAME);
 			element.removeAttribute(HIDDEN_CONTENT_ATTRIBUTE_NAME);
+			element.removeAttribute(KEPT_CONTENT_ATTRIBUTE_NAME);
 			element.removeAttribute(HIDDEN_FRAME_ATTRIBUTE_NAME);
 			element.removeAttribute(PRESERVED_SPACE_ELEMENT_ATTRIBUTE_NAME);
 			element.removeAttribute(IMAGE_ATTRIBUTE_NAME);

manifest.json

@@ -8,7 +8,7 @@
 		"64": "extension/ui/resources/icon_64.png",
 		"128": "extension/ui/resources/icon_128.png"
 	},
-	"version": "1.17.61",
+	"version": "1.18.0",
 	"description": "__MSG_extensionDescription__",
 	"content_scripts": [
 		{

cli/package.json → package.json

@@ -1,9 +1,13 @@
 {
 	"name": "single-file",
-	"version": "0.0.1",
+	"version": "0.1.0",
 	"description": "SingleFile",
 	"author": "Gildas Lormeau",
 	"license": "AGPL-3.0-or-later",
+	"main": "cli/singlefile-cli-api.js",
+	"bin": {
+		"single-file": "./cli/single-file"
+	},
 	"dependencies": {
 		"file-url": "^3.0.0",
 		"iconv-lite": "^0.5.2",
@@ -14,4 +18,4 @@
 		"strong-data-uri": "^1.0.6",
 		"yargs": "^15.4.1"
 	}
-}
+}