Merge pull request #13 from gildas-lormeau/master

upd

README.MD

@@ -14,7 +14,6 @@ SingleFile is a Web Extension (and a CLI tool) compatible with Chrome, Firefox (
  - [Integration with user scripts](#integration-with-user-scripts)
  - [SingleFileZ](#singlefilez)
  - [File format comparison](#file-format-comparison)
- - [Statistics (Firefox)](#statistics-firefox)
  - [Integration with WebKit](#integration-with-webkit)
  - [Privacy policy](#privacy-policy)
  - [Contributors](#contributors)
@@ -129,9 +128,6 @@ More info here: https://github.com/gildas-lormeau/SingleFileZ
 | **    | only in Chromium-based browsers and Internet Explorer                                                            |
 | ***   | only in Safari                                                                                                   |
 | \**** | an option must be enabled in the extension                                                                       |
-
-## Statistics (Firefox)
-See https://addons.mozilla.org/firefox/addon/single-file/statistics/?last=90
  
 ## Integration with WebKit
 Here is a Swift application (for macOS) made by [@captaindavepdx](https://github.com/captaindavepdx) that illustrates how to use SingleFile with WebKit: https://github.com/captaindavepdx/SingleFileMacOS

_locales/de/messages.json

@@ -395,6 +395,26 @@
 		"message": "open pages saved with SingleFile in the annotation editor",
 		"description": "Options page label: 'open pages saved with SingleFile in the annotation editor'"
 	},
+	"optionDefaultEditorMode": {
+		"message": "Default-Modus",
+		"description": "Options page label: 'default mode'"
+	},
+	"optionDefaultEditorModeNormal": {
+		"message": "Normal",
+		"description": "Options page label: 'default mode > normal'"
+	},
+	"optionDefaultEditorModeEdit": {
+		"message": "Editieren der Seite",
+		"description": "Options page label: 'default mode > edit the page'"
+	},
+	"optionDefaultEditorModeFormat": {
+		"message": "Formatieren der Seite",
+		"description": "Options page label: 'default mode > format the page'"
+	},
+	"optionDefaultEditorModeCut": {
+		"message": "Entfernen von Elementen",
+		"description": "Options page label: 'default mode > remove elements'"
+	},
 	"optionApplySystemTheme": {
 		"message": "apply the system theme when formatting a page",
 		"description": "Title of the button 'apply the system theme when formatting a page'"
@@ -607,6 +627,10 @@
 		"message": "Wiederherstellen aller entfernten Elemente",
 		"description": "Title of the button 'Restore all removed elements' in the editor"
 	},
+	"editorRedoCutPage": {
+		"message": "Entfernen des zuletzt wiederhergestellten Elements",
+		"description": "Title of the button 'Remove last restored element' in the editor"
+	},
 	"editorSavePage": {
 		"message": "Speichern der Webseite",
 		"description": "Title of the button 'Save the page' in the editor"

_locales/en/messages.json

@@ -395,6 +395,26 @@
 		"message": "open pages saved with SingleFile in the annotation editor",
 		"description": "Options page label: 'open pages saved with SingleFile in the annotation editor'"
 	},
+	"optionDefaultEditorMode": {
+		"message": "default mode",
+		"description": "Options page label: 'default mode'"
+	},
+	"optionDefaultEditorModeNormal": {
+		"message": "normal",
+		"description": "Options page label: 'default mode > normal'"
+	},
+	"optionDefaultEditorModeEdit": {
+		"message": "edit the page",
+		"description": "Options page label: 'default mode > edit the page'"
+	},
+	"optionDefaultEditorModeFormat": {
+		"message": "format the page",
+		"description": "Options page label: 'default mode > format the page'"
+	},
+	"optionDefaultEditorModeCut": {
+		"message": "remove elements",
+		"description": "Options page label: 'default mode > remove elements'"
+	},
 	"optionApplySystemTheme": {
 		"message": "apply the system theme when formatting a page",
 		"description": "Title of the button 'apply the system theme when formatting a page'"
@@ -607,6 +627,10 @@
 		"message": "Restore all removed elements",
 		"description": "Title of the button 'Restore all removed elements' in the editor"
 	},
+	"editorRedoCutPage": {
+		"message": "Remove last restored element",
+		"description": "Title of the button 'Remove last restored element' in the editor"
+	},
 	"editorSavePage": {
 		"message": "Save the page",
 		"description": "Title of the button 'Save the page' in the editor"

_locales/es/messages.json

@@ -395,6 +395,26 @@
 		"message": "open pages saved with SingleFile in the annotation editor",
 		"description": "Options page label: 'open pages saved with SingleFile in the annotation editor'"
 	},
+	"optionDefaultEditorMode": {
+		"message": "modo por defecto",
+		"description": "Options page label: 'default mode'"
+	},
+	"optionDefaultEditorModeNormal": {
+		"message": "normal",
+		"description": "Options page label: 'default mode > normal'"
+	},
+	"optionDefaultEditorModeEdit": {
+		"message": "editar la página",
+		"description": "Options page label: 'default mode > edit the page'"
+	},
+	"optionDefaultEditorModeFormat": {
+		"message": "formatear la página",
+		"description": "Options page label: 'default mode > format the page'"
+	},
+	"optionDefaultEditorModeCut": {
+		"message": "eliminar elementos",
+		"description": "Options page label: 'default mode > remove elements'"
+	},
 	"optionApplySystemTheme": {
 		"message": "apply the system theme when formatting a page",
 		"description": "Title of the button 'apply the system theme when formatting a page'"
@@ -607,6 +627,10 @@
 		"message": "Restore all removed elements",
 		"description": "Title of the button 'Restore all removed elements' in the editor"
 	},
+	"editorRedoCutPage": {
+		"message": "Remove last restored element",
+		"description": "Title of the button 'Remove last restored element' in the editor"
+	},
 	"editorSavePage": {
 		"message": "Save the page",
 		"description": "Title of the button 'Save the page' in the editor"

_locales/fr/messages.json

@@ -395,6 +395,26 @@
 		"message": "ouvrir les pages sauvées avec SingleFile dans l'éditeur d'annotations",
 		"description": "Options page label: 'open pages saved with SingleFile in the annotation editor'"
 	},
+	"optionDefaultEditorMode": {
+		"message": "mode par défaut",
+		"description": "Options page label: 'default mode'"
+	},
+	"optionDefaultEditorModeNormal": {
+		"message": "normal",
+		"description": "Options page label: 'default mode > normal'"
+	},
+	"optionDefaultEditorModeEdit": {
+		"message": "éditer la page",
+		"description": "Options page label: 'default mode > edit the page'"
+	},
+	"optionDefaultEditorModeFormat": {
+		"message": "formater la page",
+		"description": "Options page label: 'default mode > format the page'"
+	},
+	"optionDefaultEditorModeCut": {
+		"message": "supprimer des éléments",
+		"description": "Options page label: 'default mode > remove elements'"
+	},
 	"optionApplySystemTheme": {
 		"message": "appliquer le thème système lors du formattage d'une page",
 		"description": "Title of the button 'apply the system theme when formatting a page'"
@@ -607,6 +627,10 @@
 		"message": "Restaurer tous les élements supprimés",
 		"description": "Title of the button 'Restore all removed elements' in the editor"
 	},
+	"editorRedoCutPage": {
+		"message": "Supprimer le dernier élement restauré",
+		"description": "Title of the button 'Remove last restored element' in the editor"
+	},
 	"editorSavePage": {
 		"message": "Sauver la page",
 		"description": "Title of the button 'Save the page' in the editor"

_locales/ja/messages.json

@@ -395,6 +395,26 @@
 		"message": "annotation editor で、SingleFile で保存されたページを開く",
 		"description": "Options page label: 'open pages saved with SingleFile in the annotation editor'"
 	},
+	"optionDefaultEditorMode": {
+		"message": "default mode",
+		"description": "Options page label: 'default mode'"
+	},
+	"optionDefaultEditorModeNormal": {
+		"message": "normal",
+		"description": "Options page label: 'default mode > normal'"
+	},
+	"optionDefaultEditorModeEdit": {
+		"message": "edit the page",
+		"description": "Options page label: 'default mode > edit the page'"
+	},
+	"optionDefaultEditorModeFormat": {
+		"message": "format the page",
+		"description": "Options page label: 'default mode > format the page'"
+	},
+	"optionDefaultEditorModeCut": {
+		"message": "remove elements",
+		"description": "Options page label: 'default mode > remove elements'"
+	},
 	"optionApplySystemTheme": {
 		"message": "ページの書式設定時にシステムテーマを適用する",
 		"description": "Title of the button 'apply the system theme when formatting a page'"
@@ -607,6 +627,10 @@
 		"message": "削除されたすべての要素を復元する",
 		"description": "Title of the button 'Restore all removed elements' in the editor"
 	},
+	"editorRedoCutPage": {
+		"message": "Remove last restored element",
+		"description": "Title of the button 'Remove last restored element' in the editor"
+	},
 	"editorSavePage": {
 		"message": "ページを保存する",
 		"description": "Title of the button 'Save the page' in the editor"

_locales/pl/messages.json

@@ -4,7 +4,7 @@
 		"description": "Description of the extension."
 	},
     "commandSaveTab": {
-        "message": "Save the current tab or the selected content",
+        "message": "Zapisz bieżącą kartę lub wybraną zawartość",
         "description": "Command (Ctrl+Shift+Y): 'Save the current tab or the selected content'"
     },
     "commandSaveAllTabs": {
@@ -395,6 +395,26 @@
 		"message": "otwieraj strony zapisane z SingleFile w edytorze adnotacji",
 		"description": "Options page label: 'open pages saved with SingleFile in the annotation editor'"
 	},
+	"optionDefaultEditorMode": {
+		"message": "tryb domyślny",
+		"description": "Options page label: 'default mode'"
+	},
+	"optionDefaultEditorModeNormal": {
+		"message": "normalny",
+		"description": "Options page label: 'default mode > normal'"
+	},
+	"optionDefaultEditorModeEdit": {
+		"message": "edytuj stronę",
+		"description": "Options page label: 'default mode > edit the page'"
+	},
+	"optionDefaultEditorModeFormat": {
+		"message": "formatuj stronę",
+		"description": "Options page label: 'default mode > format the page'"
+	},
+	"optionDefaultEditorModeCut": {
+		"message": "usuń elementy",
+		"description": "Options page label: 'default mode > remove elements'"
+	},
 	"optionApplySystemTheme": {
 		"message": "zastosuj motyw systemowy podczas formatowania strony",
 		"description": "Title of the button 'apply the system theme when formatting a page'"
@@ -607,6 +627,10 @@
 		"message": "Przywróć wszystkie usunięte elementy",
 		"description": "Title of the button 'Restore all removed elements' in the editor"
 	},
+	"editorRedoCutPage": {
+		"message": "Usuń ostatni przywrócony element",
+		"description": "Title of the button 'Remove last restored element' in the editor"
+	},
 	"editorSavePage": {
 		"message": "Zapisz stronę",
 		"description": "Title of the button 'Save the page' in the editor"

_locales/ru/messages.json

@@ -395,6 +395,26 @@
 		"message": "сохранять открытые страницы из редактора аннотаций SingleFile",
 		"description": "Options page label: 'open pages saved with SingleFile in the annotation editor'"
 	},
+	"optionDefaultEditorMode": {
+		"message": "default mode",
+		"description": "Options page label: 'default mode'"
+	},
+	"optionDefaultEditorModeNormal": {
+		"message": "normal",
+		"description": "Options page label: 'default mode > normal'"
+	},
+	"optionDefaultEditorModeEdit": {
+		"message": "edit the page",
+		"description": "Options page label: 'default mode > edit the page'"
+	},
+	"optionDefaultEditorModeFormat": {
+		"message": "format the page",
+		"description": "Options page label: 'default mode > format the page'"
+	},
+	"optionDefaultEditorModeCut": {
+		"message": "remove elements",
+		"description": "Options page label: 'default mode > remove elements'"
+	},
 	"optionApplySystemTheme": {
 		"message": "использовать системную тему при форматировании страницы",
 		"description": "Title of the button 'apply the system theme when formatting a page'"
@@ -607,6 +627,10 @@
 		"message": "Восстановить все удалённые элементы",
 		"description": "Title of the button 'Restore all removed elements' in the editor"
 	},
+	"editorRedoCutPage": {
+		"message": "Remove last restored element",
+		"description": "Title of the button 'Remove last restored element' in the editor"
+	},
 	"editorSavePage": {
 		"message": "Сохранить страницу",
 		"description": "Title of the button 'Save the page' in the editor"

_locales/uk/messages.json

@@ -395,6 +395,26 @@
 		"message": "open pages saved with SingleFile in the annotation editor",
 		"description": "Options page label: 'open pages saved with SingleFile in the annotation editor'"
 	},
+	"optionDefaultEditorMode": {
+		"message": "default mode",
+		"description": "Options page label: 'default mode'"
+	},
+	"optionDefaultEditorModeNormal": {
+		"message": "normal",
+		"description": "Options page label: 'default mode > normal'"
+	},
+	"optionDefaultEditorModeEdit": {
+		"message": "edit the page",
+		"description": "Options page label: 'default mode > edit the page'"
+	},
+	"optionDefaultEditorModeFormat": {
+		"message": "format the page",
+		"description": "Options page label: 'default mode > format the page'"
+	},
+	"optionDefaultEditorModeCut": {
+		"message": "remove elements",
+		"description": "Options page label: 'default mode > remove elements'"
+	},
 	"optionApplySystemTheme": {
 		"message": "apply the system theme when formatting a page",
 		"description": "Title of the button 'apply the system theme when formatting a page'"
@@ -607,6 +627,10 @@
 		"message": "Restore all removed elements",
 		"description": "Title of the button 'Restore all removed elements' in the editor"
 	},
+	"editorRedoCutPage": {
+		"message": "Remove last restored element",
+		"description": "Title of the button 'Remove last restored element' in the editor"
+	},
 	"editorSavePage": {
 		"message": "Зберегти сторінку",
 		"description": "Title of the button 'Save the page' in the editor"

_locales/zh_CN/messages.json

@@ -3,14 +3,14 @@
 		"message": "将一个完整的页面保存到单个 HTML 文件中",
 		"description": "Description of the extension."
 	},
-    "commandSaveTab": {
-        "message": "Save the current tab or the selected content",
-        "description": "Command (Ctrl+Shift+Y): 'Save the current tab or the selected content'"
-    },
-    "commandSaveAllTabs": {
-        "message": "保存所有标签页",
-        "description": "Command (Ctrl+Shift+U): 'Save all tabs'"
-    },
+	"commandSaveTab": {
+		"message": "保存所选标签页或选中部分",
+		"description": "Command (Ctrl+Shift+Y): 'Save the current tab or the selected content'"
+	},
+	"commandSaveAllTabs": {
+		"message": "保存所有标签页",
+		"description": "Command (Ctrl+Shift+U): 'Save all tabs'"
+	},
 	"menuSavePage": {
 		"message": "使用 SingleFile 保存页面",
 		"description": "Menu entry: 'Save page with SingleFile'"
@@ -382,7 +382,7 @@
 	"optionAutoSaveExternalSave": {
 		"message": "使用 SingleFile Companion 保存页面",
 		"description": "Options page label: 'save the page with SingleFile Companion'"
-	},	
+	},
 	"optionsEditorSubTitle": {
 		"message": "标注编辑器",
 		"description": "Options sub-title: 'Annotation editor'"
@@ -395,6 +395,26 @@
 		"message": "在标注编辑器中打开用 SingleFile 保存的页面",
 		"description": "Options page label: 'open pages saved with SingleFile in the annotation editor'"
 	},
+	"optionDefaultEditorMode": {
+		"message": "default mode",
+		"description": "Options page label: 'default mode'"
+	},
+	"optionDefaultEditorModeNormal": {
+		"message": "normal",
+		"description": "Options page label: 'default mode > normal'"
+	},
+	"optionDefaultEditorModeEdit": {
+		"message": "edit the page",
+		"description": "Options page label: 'default mode > edit the page'"
+	},
+	"optionDefaultEditorModeFormat": {
+		"message": "format the page",
+		"description": "Options page label: 'default mode > format the page'"
+	},
+	"optionDefaultEditorModeCut": {
+		"message": "remove elements",
+		"description": "Options page label: 'default mode > remove elements'"
+	},
 	"optionApplySystemTheme": {
 		"message": "在标注编辑器中打开页面时应用系统主题样式",
 		"description": "Title of the button 'apply the system theme when formatting a page'"
@@ -607,6 +627,10 @@
 		"message": "还原所有被移除的元素",
 		"description": "Title of the button 'Restore all removed elements' in the editor"
 	},
+	"editorRedoCutPage": {
+		"message": "Remove last restored element",
+		"description": "Title of the button 'Remove last restored element' in the editor"
+	},
 	"editorSavePage": {
 		"message": "保存该页面",
 		"description": "Title of the button 'Save the page' in the editor"

_locales/zh_TW/messages.json

@@ -4,7 +4,7 @@
 		"description": "Description of the extension."
 	},
 	"commandSaveTab": {
-		"message": "Save the current tab or the selected content",
+		"message": "保存所選標籤頁或選中部分",
 		"description": "Command (Ctrl+Shift+Y): 'Save the current tab or the selected content'"
 	},
 	"commandSaveAllTabs": {
@@ -395,6 +395,26 @@
 		"message": "在標註編輯器中打開用 SingleFile 保存的頁面",
 		"description": "Options page label: 'open pages saved with SingleFile in the annotation editor'"
 	},
+	"optionDefaultEditorMode": {
+		"message": "default mode",
+		"description": "Options page label: 'default mode'"
+	},
+	"optionDefaultEditorModeNormal": {
+		"message": "normal",
+		"description": "Options page label: 'default mode > normal'"
+	},
+	"optionDefaultEditorModeEdit": {
+		"message": "edit the page",
+		"description": "Options page label: 'default mode > edit the page'"
+	},
+	"optionDefaultEditorModeFormat": {
+		"message": "format the page",
+		"description": "Options page label: 'default mode > format the page'"
+	},
+	"optionDefaultEditorModeCut": {
+		"message": "remove elements",
+		"description": "Options page label: 'default mode > remove elements'"
+	},
 	"optionApplySystemTheme": {
 		"message": "在標註編輯器中打開頁面時應用系統主題樣式",
 		"description": "Title of the button 'apply the system theme when formatting a page'"
@@ -607,6 +627,10 @@
 		"message": "還原所有被移除的元素",
 		"description": "Title of the button 'Restore all removed elements' in the editor"
 	},
+	"editorRedoCutPage": {
+		"message": "Remove last restored element",
+		"description": "Title of the button 'Remove last restored element' in the editor"
+	},
 	"editorSavePage": {
 		"message": "保存該頁面",
 		"description": "Title of the button 'Save the page' in the editor"

cli/README.MD

@@ -18,9 +18,9 @@ SingleFile can be launched from the command line by running it into a (headless)
   
     `cd SingleFile-master`
   
-    `cd cli`
-      
     `npm install`
+  
+    `cd cli`    
     
   - Download with `npm`
     
@@ -37,10 +37,10 @@ SingleFile can be launched from the command line by running it into a (headless)
     `git clone --depth 1 --recursive https://github.com/gildas-lormeau/SingleFile.git`
   
     `cd SingleFile`
+    
+    `npm install`
   
-    `cd cli`
-       
-    `npm install`        
+    `cd cli`           
   
 - Make `single-file` executable (Linux/Unix/BSD etc.).
 
@@ -62,7 +62,7 @@ SingleFile can be launched from the command line by running it into a (headless)
 
   - Dump the processed content of https://www.wikipedia.org into the console
 
-  `single-file https://www.wikipedia.org --filename-template=""`
+  `single-file https://www.wikipedia.org --dump-content`
 
   - Save https://www.wikipedia.org into `wikipedia.html` in the current folder
 
@@ -104,6 +104,8 @@ SingleFile can be launched from the command line by running it into a (headless)
 
   `docker run singlefile "https://www.wikipedia.org" > wikipedia.html`
 
+- An alternative docker file can be found here https://github.com/screenbreak/SingleFile-dockerized. It allows you to save pages from the command line interface or through an HTTP server.
+
 ## License
 
 SingleFile is licensed under AGPL. Code derived from third-party projects is licensed under MIT. Please contact me at gildas.lormeau <at> gmail.com if you are interested in licensing the SingleFile code for a commercial service or product.

cli/args.js

@@ -42,11 +42,14 @@ const args = require("yargs")
 		"browser-extensions": [],
 		"browser-scripts": [],
 		"browser-args": "",
+		"browser-start-minimized": false,
 		"compress-CSS": false,
 		"compress-HTML": true,
+		"dump-content": false,
 		"filename-template": "{page-title} ({date-iso} {time-locale}).html",
 		"filename-replacement-character": "_",
 		"group-duplicate-images": true,
+		"http-header": [],
 		"include-infobar": false,
 		"load-deferred-images": true,
 		"load-deferred-images-max-idle-time": 1500,
@@ -78,7 +81,7 @@ const args = require("yargs")
 		"crawl-rewrite-rules": []
 	})
 	.options("back-end", { description: "Back-end to use" })
-	.choices("back-end", ["jsdom", "puppeteer", "webdriver-chromium", "webdriver-gecko"])
+	.choices("back-end", ["jsdom", "puppeteer", "webdriver-chromium", "webdriver-gecko", "puppeteer-firefox", "playwright-firefox", "playwright-chromium"])
 	.options("browser-headless", { description: "Run the browser in headless mode (puppeteer, webdriver-gecko, webdriver-chromium)" })
 	.boolean("browser-headless")
 	.options("browser-executable-path", { description: "Path to chrome/chromium executable (puppeteer, webdriver-gecko, webdriver-chromium)" })
@@ -101,6 +104,8 @@ const args = require("yargs")
 	.array("browser-scripts")
 	.options("browser-args", { description: "Arguments provided as a JSON array and passed to the browser (puppeteer, webdriver-gecko, webdriver-chromium)" })
 	.string("browser-args")
+	.options("browser-start-minimized", { description: "Minimize the browser (puppeteer)" })
+	.boolean("browser-start-minimized")
 	.options("compress-CSS", { description: "Compress CSS stylesheets" })
 	.boolean("compress-CSS")
 	.options("compress-HTML", { description: "Compress HTML content" })
@@ -109,8 +114,14 @@ const args = require("yargs")
 	.boolean("crawl-links")
 	.options("crawl-inner-links-only", { description: "Crawl pages found via inner links only if they are hosted on the same domain" })
 	.boolean("crawl-inner-links-only")
+	.options("crawl-load-session", { description: "Name of the file of the session to load (previously saved with --crawl-save-session or --crawl-sync-session)" })
+	.string("crawl-load-session")
 	.options("crawl-remove-url-fragment", { description: "Remove URL fragments found in links" })
 	.boolean("crawl-remove-url-fragment")
+	.options("crawl-save-session", { description: "Name of the file where to save the state of the session" })
+	.string("crawl-save-session")
+	.options("crawl-sync-session", { description: "Name of the file where to load and save the state of the session" })
+	.string("crawl-sync-session")
 	.options("crawl-max-depth", { description: "Max depth when crawling pages found in internal and external links (0: infinite)" })
 	.number("crawl-max-depth")
 	.options("crawl-external-links-max-depth", { description: "Max depth when crawling pages found in external links (0: infinite)" })
@@ -119,15 +130,19 @@ const args = require("yargs")
 	.boolean("crawl-replace-urls")
 	.options("crawl-rewrite-rules", { description: "List of rewrite rules used to rewrite URLs of internal and external links" })
 	.array("crawl-rewrite-rules")
+	.options("dump-content", { description: "Dump the content of the processed page in the console" })
+	.boolean("dump-content")
 	.options("error-file")
 	.string("error-file")
 	.options("filename-template", { description: "Template used to generate the output filename (see help page of the extension for more info)" })
 	.string("filename-template")
 	.options("filename-replacement-character", { description: "The character used for replacing invalid characters in filenames" })
 	.string("filename-replacement-character")
-	.string("filename-template")
+	.string("filename-replacement-character")
 	.options("group-duplicate-images", { description: "Group duplicate images into CSS custom properties" })
 	.boolean("group-duplicate-images")
+	.options("http-header", { description: "Extra HTTP header (puppeteer, jsdom)" })
+	.array("http-header")
 	.options("include-BOM", { description: "Include the UTF-8 BOM into the HTML page" })
 	.boolean("include-BOM")
 	.options("include-infobar", { description: "Include the infobar" })
@@ -177,9 +192,25 @@ const args = require("yargs")
 	.options("web-driver-executable-path", { description: "Path to Selenium WebDriver executable (webdriver-gecko, webdriver-chromium)" })
 	.string("web-driver-executable-path")
 	.argv;
+if (args.dumpContent) {
+	args.filenameTemplate = "";
+}
 args.compressCSS = args.compressCss;
 args.compressHTML = args.compressHtml;
 args.includeBOM = args.includeBom;
 args.crawlReplaceURLs = args.crawlReplaceUrls;
 args.crawlRemoveURLFragment = args.crawlRemoveUrlFragment;
+const headers = args.httpHeader;
+delete args.httpHeader;
+args.httpHeaders = {};
+headers.forEach(header => {
+	const matchedHeader = header.match(/^(.*?):(.*)$/);
+	if (matchedHeader.length == 3) {
+		args.httpHeaders[matchedHeader[1].trim()] = matchedHeader[2].trimLeft();
+	}
+});
+Object.keys(args).filter(optionName => optionName.includes("-"))
+	.forEach(optionName => delete args[optionName]);
+delete args["$0"];
+delete args["_"];
 module.exports = args;

cli/back-ends/common/scripts.js

@@ -25,7 +25,7 @@
 
 const fs = require("fs");
 
-const SCRIPTS = [		
+const SCRIPTS = [
 	"lib/single-file/processors/hooks/content/content-hooks.js",
 	"lib/single-file/processors/hooks/content/content-hooks-web.js",
 	"lib/single-file/processors/hooks/content/content-hooks-frames.js",
@@ -59,9 +59,9 @@ const INDEX_SCRIPTS = [
 ];
 
 const WEB_SCRIPTS = [
-	"lib/single-file/processors/hooks/content/content-hooks-web.js",
-	"lib/single-file/processors/hooks/content/content-hooks-frames-web.js",
-	"common/ui/content/content-infobar-web.js"
+	"/lib/single-file/processors/hooks/content/content-hooks-web.js",
+	"/lib/single-file/processors/hooks/content/content-hooks-frames-web.js",
+	"/common/ui/content/content-infobar-web.js"
 ];
 
 exports.get = async options => {

cli/back-ends/jsdom.js

@@ -21,29 +21,20 @@
  *   Source.
  */
 
-/* global require, exports */
+/* global require, exports, Buffer */
 
 const crypto = require("crypto");
 
-const { JSDOM, VirtualConsole } = require("jsdom");
+const jsdom = require("jsdom");
+const { JSDOM, VirtualConsole } = jsdom;
 const iconv = require("iconv-lite");
-const request = require("request-promise-native");
 
 exports.initialize = async () => { };
 
 exports.getPageData = async options => {
-	const pageContent = (await request({
-		method: "GET",
-		uri: options.url,
-		resolveWithFullResponse: true,
-		encoding: null,
-		headers: {
-			"User-Agent": options.userAgent
-		}
-	})).body.toString();
 	let win;
 	try {
-		const dom = new JSDOM(pageContent, getBrowserOptions(options));
+		const dom = await JSDOM.fromURL(options.url, getBrowserOptions(options));
 		win = dom.window;
 		return await getPageData(win, options);
 	} finally {
@@ -63,7 +54,7 @@ async function getPageData(win, options) {
 			this.utfLabel = utfLabel;
 		}
 		decode(buffer) {
-			return iconv.decode(buffer, this.utfLabel);
+			return iconv.decode(Buffer.from(buffer), this.utfLabel);
 		}
 	};
 	win.crypto = {
@@ -83,21 +74,55 @@ async function getPageData(win, options) {
 	}
 	executeFrameScripts(doc, scripts);
 	options.removeHiddenElements = false;
+	options.loadDeferredImages = false;
 	const pageData = await win.singlefile.lib.getPageData(options, { fetch: url => fetchResource(url, options) }, doc, win);
 	if (options.includeInfobar) {
 		await win.singlefile.common.ui.content.infobar.includeScript(pageData);
 	}
 	return pageData;
+
+	async function fetchResource(resourceURL) {
+		return new Promise((resolve, reject) => {
+			const xhrRequest = new win.XMLHttpRequest();
+			xhrRequest.withCredentials = true;
+			xhrRequest.responseType = "arraybuffer";
+			xhrRequest.onerror = event => reject(new Error(event.detail));
+			xhrRequest.onreadystatechange = () => {
+				if (xhrRequest.readyState == win.XMLHttpRequest.DONE) {
+					resolve({
+						arrayBuffer: async () => new Uint8Array(xhrRequest.response).buffer,
+						headers: {
+							get: headerName => xhrRequest.getResponseHeader(headerName)
+						},
+						status: xhrRequest.status
+					});
+				}
+			};
+			xhrRequest.open("GET", resourceURL, true);
+			xhrRequest.send();
+		});
+	}
 }
 
 function getBrowserOptions(options) {
+	class ResourceLoader extends jsdom.ResourceLoader {
+		_getRequestOptions(fetchOptions) {
+			const requestOptions = super._getRequestOptions(fetchOptions);
+			if (options.httpHeaders) {
+				requestOptions.headers = Object.assign(requestOptions.headers, options.httpHeaders);
+			}
+			return requestOptions;
+		}
+	}
+	const resourceLoader = new ResourceLoader({
+		userAgent: options.userAgent
+	});
 	const jsdomOptions = {
-		url: options.url,
 		virtualConsole: new VirtualConsole(),
 		userAgent: options.userAgent,
 		pretendToBeVisual: true,
 		runScripts: "outside-only",
-		resources: "usable"
+		resources: resourceLoader
 	};
 	if (options.browserWidth && options.browserHeight) {
 		jsdomOptions.beforeParse = function (window) {
@@ -119,23 +144,4 @@ function executeFrameScripts(doc, scripts) {
 			// ignored
 		}
 	});
-}
-
-async function fetchResource(resourceURL, options) {
-	const response = await request({
-		method: "GET",
-		uri: resourceURL,
-		resolveWithFullResponse: true,
-		encoding: null,
-		headers: {
-			"User-Agent": options.userAgent
-		}
-	});
-	return {
-		status: response.statusCode,
-		headers: {
-			get: name => response.headers[name]
-		},
-		arrayBuffer: async () => response.body
-	};
 }

cli/back-ends/playwright-chromium.js

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2010-2020 Gildas Lormeau
+ * contact : gildas.lormeau <at> gmail.com
+ * 
+ * This file is part of SingleFile.
+ *
+ *   The code in this file is free software: you can redistribute it and/or 
+ *   modify it under the terms of the GNU Affero General Public License 
+ *   (GNU AGPL) as published by the Free Software Foundation, either version 3
+ *   of the License, or (at your option) any later version.
+ * 
+ *   The code in this file is distributed in the hope that it will be useful, 
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
+ *   General Public License for more details.
+ *
+ *   As additional permission under GNU AGPL version 3 section 7, you may 
+ *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
+ *   AGPL normally required by section 4, provided you include this license 
+ *   notice and a URL through which recipients can access the Corresponding 
+ *   Source.
+ */
+
+/* global singlefile, require, exports */
+
+const playwright = require("playwright").chromium;
+const scripts = require("./common/scripts.js");
+
+const NETWORK_IDLE_STATE = "networkidle";
+
+let browser;
+
+exports.initialize = async options => {
+	browser = await playwright.launch(getBrowserOptions(options));
+};
+
+exports.getPageData = async options => {
+	let page;
+	try {
+		page = await browser.newPage({
+			bypassCSP: options.browserBypassCSP === undefined || options.browserBypassCSP
+		});
+		await setPageOptions(page, options);
+		return await getPageData(page, options);
+	} finally {
+		if (page) {
+			await page.close();
+		}
+	}
+};
+
+exports.closeBrowser = () => {
+	if (browser) {
+		return browser.close();
+	}
+};
+
+function getBrowserOptions(options) {
+	const browserOptions = {};
+	if (options.browserHeadless !== undefined) {
+		browserOptions.headless = options.browserHeadless && !options.browserDebug;
+	}
+	browserOptions.args = options.browserArgs ? JSON.parse(options.browserArgs) : [];
+	if (options.browserExecutablePath) {
+		browserOptions.executablePath = options.browserExecutablePath || "chrome";
+	}
+	return browserOptions;
+}
+
+async function setPageOptions(page, options) {
+	if (options.browserWidth && options.browserHeight) {
+		await page.setViewportSize({
+			width: options.browserWidth,
+			height: options.browserHeight
+		});
+	}	
+	if (options.httpHeaders) {
+		page.setExtraHTTPHeaders(options.httpHeaders);
+	}
+}
+
+async function getPageData(page, options) {
+	const injectedScript = await scripts.get(options);
+	await page.addInitScript(injectedScript);
+	if (options.browserDebug) {
+		await page.waitForTimeout(3000);
+	}
+	await page.goto(options.url, {
+		timeout: options.browserLoadMaxTime || 0,
+		waitUntil: options.browserWaitUntil && options.browserWaitUntil.startsWith("networkidle") ? NETWORK_IDLE_STATE : options.browserWaitUntil || NETWORK_IDLE_STATE
+	});
+	return await page.evaluate(async options => {
+		const pageData = await singlefile.lib.getPageData(options);
+		if (options.includeInfobar) {
+			await singlefile.common.ui.content.infobar.includeScript(pageData);
+		}
+		return pageData;
+	}, options);
+}

cli/back-ends/playwright-firefox.js

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2010-2020 Gildas Lormeau
+ * contact : gildas.lormeau <at> gmail.com
+ * 
+ * This file is part of SingleFile.
+ *
+ *   The code in this file is free software: you can redistribute it and/or 
+ *   modify it under the terms of the GNU Affero General Public License 
+ *   (GNU AGPL) as published by the Free Software Foundation, either version 3
+ *   of the License, or (at your option) any later version.
+ * 
+ *   The code in this file is distributed in the hope that it will be useful, 
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
+ *   General Public License for more details.
+ *
+ *   As additional permission under GNU AGPL version 3 section 7, you may 
+ *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
+ *   AGPL normally required by section 4, provided you include this license 
+ *   notice and a URL through which recipients can access the Corresponding 
+ *   Source.
+ */
+
+/* global singlefile, require, exports */
+
+const playwright = require("playwright").firefox;
+const scripts = require("./common/scripts.js");
+
+const NETWORK_IDLE_STATE = "networkidle";
+
+let browser;
+
+exports.initialize = async options => {
+	browser = await playwright.launch(getBrowserOptions(options));
+};
+
+exports.getPageData = async options => {
+	let page;
+	try {
+		page = await browser.newPage({
+			bypassCSP: options.browserBypassCSP === undefined || options.browserBypassCSP
+		});
+		await setPageOptions(page, options);
+		return await getPageData(page, options);
+	} finally {
+		if (page) {
+			await page.close();
+		}
+	}
+};
+
+exports.closeBrowser = () => {
+	if (browser) {
+		return browser.close();
+	}
+};
+
+function getBrowserOptions(options) {
+	const browserOptions = {};
+	if (options.browserHeadless !== undefined) {
+		browserOptions.headless = options.browserHeadless && !options.browserDebug;
+	}
+	browserOptions.args = options.browserArgs ? JSON.parse(options.browserArgs) : [];
+	if (options.browserExecutablePath) {
+		browserOptions.executablePath = options.browserExecutablePath || "firefox";
+	}
+	return browserOptions;
+}
+
+async function setPageOptions(page, options) {
+	if (options.browserWidth && options.browserHeight) {
+		await page.setViewportSize({
+			width: options.browserWidth,
+			height: options.browserHeight
+		});
+	}
+	if (options.httpHeaders) {
+		page.setExtraHTTPHeaders(options.httpHeaders);
+	}
+}
+
+async function getPageData(page, options) {
+	const injectedScript = await scripts.get(options);
+	await page.addInitScript(injectedScript);
+	if (options.browserDebug) {
+		await page.waitForTimeout(3000);
+	}
+	await page.goto(options.url, {
+		timeout: options.browserLoadMaxTime || 0,
+		waitUntil: options.browserWaitUntil && options.browserWaitUntil.startsWith("networkidle") ? NETWORK_IDLE_STATE : options.browserWaitUntil || NETWORK_IDLE_STATE
+	});
+	return await page.evaluate(async options => {
+		const pageData = await singlefile.lib.getPageData(options);
+		if (options.includeInfobar) {
+			await singlefile.common.ui.content.infobar.includeScript(pageData);
+		}
+		return pageData;
+	}, options);
+}

cli/back-ends/puppeteer-firefox.js

@@ -81,16 +81,24 @@ async function setPageOptions(page, options) {
 			// ignored
 		}
 	}
+	if (options.httpHeaders) {
+		try {
+			await page.setExtraHTTPHeaders(options.httpHeaders);
+		} catch (error) {
+			// ignored
+		}
+	}
 }
 
 async function getPageData(browser, page, options) {
 	const injectedScript = await scripts.get(options);
 	await page.evaluateOnNewDocument(injectedScript);
 	if (options.browserDebug) {
-		await page.waitFor(3000);
+		await page.waitForTimeout(3000);
 	}
 	await pageGoto(page, options);
 	try {
+		await page.evaluate(injectedScript);
 		return await page.evaluate(async options => {
 			const pageData = await singlefile.lib.getPageData(options);
 			if (options.includeInfobar) {

cli/back-ends/puppeteer.js

@@ -65,7 +65,7 @@ function getBrowserOptions(options = {}) {
 		browserOptions.args.push("--disable-web-security");
 	}
 	browserOptions.args.push("--no-pings");
-	if (options.browserDebug) {
+	if (!options.browserStartMinimized && options.browserDebug) {
 		browserOptions.args.push("--auto-open-devtools-for-tabs");
 	}
 	if (options.browserWidth && options.browserHeight) {
@@ -88,13 +88,21 @@ async function setPageOptions(page, options) {
 	if (options.browserBypassCSP === undefined || options.browserBypassCSP) {
 		await page.setBypassCSP(true);
 	}
+	if (options.httpHeaders) {
+		page.setExtraHTTPHeaders(options.httpHeaders);
+	}
+	if (options.browserStartMinimized) {
+		const session = await page.target().createCDPSession();
+		const { windowId } = await session.send("Browser.getWindowForTarget");
+		await session.send("Browser.setWindowBounds", { windowId, bounds: { windowState: "minimized" } });
+	}
 }
 
 async function getPageData(browser, page, options) {
 	const injectedScript = await scripts.get(options);
 	await page.evaluateOnNewDocument(injectedScript);
 	if (options.browserDebug) {
-		await page.waitFor(3000);
+		await page.waitForTimeout(3000);
 	}
 	try {
 		await pageGoto(page, options);

cli/dockerfile

@@ -9,5 +9,5 @@ RUN set -x \
     
 WORKDIR /usr/src/app/SingleFile/cli
 
-ENTRYPOINT ["./single-file", "--browser-executable-path", "/usr/bin/chromium-browser", "--filename-template=''", "--browser-args", "[\"--no-sandbox\"]"]
+ENTRYPOINT ["./single-file", "--browser-executable-path", "/usr/bin/chromium-browser", "--dump-content", "--browser-args", "[\"--no-sandbox\"]"]
 CMD ["https://github.com/Zenika/alpine-chrome"]

cli/single-file

@@ -27,10 +27,11 @@
 
 const fileUrl = require("file-url");
 const fs = require("fs");
-run(require("./args"));
+run(require("./args"))
+	.catch(error => console.error(error.message || error)); // eslint-disable-line no-console	
 
 async function run(options) {
-	const singlefile = await require("./singlefile-cli-api")(options);
+	const singlefile = await require("./single-file-cli-api")(options);
 	let urls;
 	if (options.url && !singlefile.VALID_URL_TEST.test(options.url)) {
 		options.url = fileUrl(options.url);

cli/singlefile-cli-api.js → cli/single-file-cli-api.js

@@ -26,21 +26,38 @@
 const fs = require("fs");
 const VALID_URL_TEST = /^(https?|file):\/\//;
 
+const STATE_PROCESSING = "processing";
+const STATE_PROCESSED = "processed";
+
 const backEnds = {
 	jsdom: "./back-ends/jsdom.js",
 	puppeteer: "./back-ends/puppeteer.js",
 	"puppeteer-firefox": "./back-ends/puppeteer-firefox.js",
 	"webdriver-chromium": "./back-ends/webdriver-chromium.js",
-	"webdriver-gecko": "./back-ends/webdriver-gecko.js"
+	"webdriver-gecko": "./back-ends/webdriver-gecko.js",
+	"playwright-firefox": "./back-ends/playwright-firefox.js",
+	"playwright-chromium": "./back-ends/playwright-chromium.js"
 };
 
-let backend, tasks = [], maxParallelWorkers = 8;
+let backend, tasks = [], maxParallelWorkers = 8, sessionFilename;
 module.exports = initialize;
 
 async function initialize(options) {
 	maxParallelWorkers = options.maxParallelWorkers;
 	backend = require(backEnds[options.backEnd]);
 	await backend.initialize(options);
+	if (options.crawlSyncSession || options.crawlLoadSession) {
+		try {
+			tasks = JSON.parse(fs.readFileSync(options.crawlSyncSession || options.crawlLoadSession).toString());
+		} catch (error) {
+			if (options.crawlLoadSession) {
+				throw error;
+			}
+		}
+	}
+	if (options.crawlSyncSession || options.crawlSaveSession) {
+		sessionFilename = options.crawlSyncSession || options.crawlSaveSession;
+	}
 	return {
 		capture: urls => capture(urls, options),
 		finish: () => finish(options),
@@ -50,12 +67,14 @@ async function initialize(options) {
 
 async function capture(urls, options) {
 	let newTasks;
+	const taskUrls = tasks.map(task => task.url);
 	newTasks = urls.map(url => createTask(url, options));
-	newTasks = newTasks.filter(task => task);
+	newTasks = newTasks.filter(task => task && !taskUrls.includes(task.url));
 	if (newTasks.length) {
 		tasks = tasks.concat(newTasks);
-		await runTasks();
+		saveTasks();
 	}
+	await runTasks();
 }
 
 async function finish(options) {
@@ -66,11 +85,13 @@ async function finish(options) {
 			try {
 				let pageContent = fs.readFileSync(task.filename).toString();
 				tasks.forEach(otherTask => {
-					pageContent = pageContent.replace(new RegExp(escapeRegExp("\"" + otherTask.originalUrl + "\""), "gi"), "\"" + otherTask.filename + "\"");
-					pageContent = pageContent.replace(new RegExp(escapeRegExp("'" + otherTask.originalUrl + "'"), "gi"), "'" + otherTask.filename + "'");
-					const filename = otherTask.filename.replace(/ /g, "%20");
-					pageContent = pageContent.replace(new RegExp(escapeRegExp("=" + otherTask.originalUrl + " "), "gi"), "=" + filename + " ");
-					pageContent = pageContent.replace(new RegExp(escapeRegExp("=" + otherTask.originalUrl + ">"), "gi"), "=" + filename + ">");
+					if (otherTask.filename) {
+						pageContent = pageContent.replace(new RegExp(escapeRegExp("\"" + otherTask.originalUrl + "\""), "gi"), "\"" + otherTask.filename + "\"");
+						pageContent = pageContent.replace(new RegExp(escapeRegExp("'" + otherTask.originalUrl + "'"), "gi"), "'" + otherTask.filename + "'");
+						const filename = otherTask.filename.replace(/ /g, "%20");
+						pageContent = pageContent.replace(new RegExp(escapeRegExp("=" + otherTask.originalUrl + " "), "gi"), "=" + filename + " ");
+						pageContent = pageContent.replace(new RegExp(escapeRegExp("=" + otherTask.originalUrl + ">"), "gi"), "=" + filename + ">");
+					}
 				});
 				fs.writeFileSync(task.filename, pageContent);
 			} catch (error) {
@@ -85,7 +106,7 @@ async function finish(options) {
 
 async function runTasks() {
 	const availableTasks = tasks.filter(task => !task.status).length;
-	const processingTasks = tasks.filter(task => task.status == "processing").length;
+	const processingTasks = tasks.filter(task => task.status == STATE_PROCESSING).length;
 	const promisesTasks = [];
 	for (let workerIndex = 0; workerIndex < Math.min(availableTasks, maxParallelWorkers - processingTasks); workerIndex++) {
 		promisesTasks.push(runNextTask());
@@ -99,10 +120,11 @@ async function runNextTask() {
 		const options = task.options;
 		let taskOptions = JSON.parse(JSON.stringify(options));
 		taskOptions.url = task.url;
-		task.status = "processing";
+		task.status = STATE_PROCESSING;
+		saveTasks();
 		task.promise = capturePage(taskOptions);
 		const pageData = await task.promise;
-		task.status = "processed";
+		task.status = STATE_PROCESSED;
 		if (pageData) {
 			task.filename = pageData.filename;
 			if (options.crawlLinks && testMaxDepth(task)) {
@@ -115,13 +137,14 @@ async function runNextTask() {
 				tasks.splice(tasks.length, 0, ...newTasks);
 			}
 		}
+		saveTasks();
 		await runTasks();
 	}
 }
 
 function testMaxDepth(task) {
 	const options = task.options;
-	return (options.crawlMaxDepth == 0 || task.depth < options.crawlMaxDepth) &&
+	return (options.crawlMaxDepth == 0 || task.depth <= options.crawlMaxDepth) &&
 		(options.crawlExternalLinksMaxDepth == 0 || task.externalLinkDepth < options.crawlExternalLinksMaxDepth);
 }
 
@@ -140,6 +163,18 @@ function createTask(url, options, parentTask, rootTask) {
 	}
 }
 
+function saveTasks() {
+	if (sessionFilename) {
+		fs.writeFileSync(sessionFilename, JSON.stringify(
+			tasks.map(task => Object.assign({}, task, {
+				status: task.status == STATE_PROCESSING ? undefined : task.status,
+				promise: undefined,
+				options: task.status && task.status == STATE_PROCESSED ? undefined : task.options
+			}))
+		));
+	}
+}
+
 function rewriteURL(url, crawlRemoveURLFragment, crawlRewriteRules) {
 	url = url.trim();
 	if (crawlRemoveURLFragment) {

common/ui/content/content-infobar-web.js

@@ -29,6 +29,7 @@
 	const LINK_ICON = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABIAAAASCAYAAABWzo5XAAAABmJLR0QABQDuAACS38mlAAAACXBIWXMAACfuAAAn7gExzuVDAAAAB3RJTUUH4ggCDDcMnYqGGAAAATtJREFUOMvNk19LwlAYxp+zhOoqpxJ1la3patFVINk/oRDBLuyreiPFMmcj/QQRSOOwpEINDCpwRr7d1HBMc4sufO7Oe877e5/zcA4wbWLDi8urGr2+vXsOFfJZdnPboDtuueoRcQEH6RQDgNBP8bxcpfvmA0QxPHF6u/MMInLVHFDP7kMUwyjks2xU8+ZGkgGAbtSp1e5gRhBc+0KQHHSjTg2TY0tVEItF/wYqV6+pYXKoiox0atvjOuQXYnILqiJj/ztceXUlGEirGGRyC0pCciDDmfm6mlYxiFtNKAkJmb0dV2OxpFGxpNFE0NmFTtxqQpbiHsgojQX1bBuyFMfR4S7zk+PYjE5PcizI0xD+6685jubnZvH41MJwgL+p233B8tKiF7SeXMPnYIB+/8OXg2hERO44wzC1+gJYGGpVbtoqiAAAAABJRU5ErkJggg==";
 	const IMAGE_ICON = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAABIUlEQVQ4y+2TsarCMBSGvxTBRdqiUZAWOrhJB9EXcPKFfCvfQYfulUKHDqXg4CYUJSioYO4mSDX3ttzt3n87fMlHTpIjlsulxpDZbEYYhgghSNOUOI5Ny2mZYBAELBYLer0eAJ7ncTweKYri4x7LJJRS0u12n7XrukgpjSc0CpVSXK/XZ32/31FKNW85z3PW6zXT6RSAJEnIsqy5UGvNZrNhu90CcDqd+C6tT6J+v//2Th+PB2VZ1hN2Oh3G4zGTyQTbtl/YbrdjtVpxu91+Ljyfz0RRhG3bzOfzF+Y4TvNXvlwuaK2pE4tfzr/wzwsty0IIURlL0998KxRCMBqN8H2/wlzXJQxD2u12vVkeDoeUZUkURRU+GAw4HA7s9/sK+wK6CWHasQ/S/wAAAABJRU5ErkJggg==";
 	const SINGLEFILE_COMMENT = "SingleFile";
+	const SINGLE_FILE_UI_ELEMENT_CLASS = "single-file-ui-element";
 
 	const browser = this.browser;
 
@@ -79,6 +80,7 @@
 				infoData = saveDate;
 			}
 			infobarElement = createElement(INFOBAR_TAGNAME, document.body);
+			infobarElement.className = SINGLE_FILE_UI_ELEMENT_CLASS;
 			setProperty(infobarElement, "background-color", "#f9f9f9");
 			setProperty(infobarElement, "display", "flex");
 			setProperty(infobarElement, "position", "fixed");

extension/core/bg/business.js

@@ -30,6 +30,8 @@ singlefile.extension.core.bg.business = (() => {
 	const ERROR_CONNECTION_LOST_GECKO = "Message manager disconnected";
 	const INJECT_SCRIPTS_STEP = 1;
 	const EXECUTE_SCRIPTS_STEP = 2;
+	const TASK_PENDING_STATE = "pending";
+	const TASK_PROCESSING_STATE = "processing";
 
 	const extensionScriptFiles = [
 		"common/index.js",
@@ -65,7 +67,7 @@ singlefile.extension.core.bg.business = (() => {
 		onSaveEnd: taskId => {
 			const taskInfo = tasks.find(taskInfo => taskInfo.id == taskId);
 			if (taskInfo) {
-				taskInfo.resolve();
+				taskInfo.done();
 			}
 		},
 		onInit: tab => cancelTab(tab.id),
@@ -93,8 +95,12 @@ singlefile.extension.core.bg.business = (() => {
 			Object.keys(options).forEach(key => tabOptions[key] = options[key]);
 			tabOptions.autoClose = true;
 			tabOptions.extensionScriptFiles = extensionScriptFiles;
-			tasks.push({ id: currentTaskId, status: "pending", tab: { url }, options: tabOptions, method: "content.save" });
-			currentTaskId++;
+			addTask({
+				tab: { url },
+				status: TASK_PENDING_STATE,
+				options: tabOptions,
+				method: "content.save"
+			});
 		}));
 		runTasks();
 	}
@@ -113,18 +119,25 @@ singlefile.extension.core.bg.business = (() => {
 			tabOptions.extensionScriptFiles = extensionScriptFiles;
 			if (options.autoSave) {
 				if (autosave.isEnabled(tab)) {
-					const taskInfo = { id: currentTaskId, status: "processing", tab, options: tabOptions, method: "content.autosave" };
-					tasks.push(taskInfo);
+					const taskInfo = addTask({
+						status: TASK_PROCESSING_STATE,
+						tab,
+						options: tabOptions,
+						method: "content.autosave"
+					});
 					runTask(taskInfo);
-					currentTaskId++;
 				}
 			} else {
 				ui.onStart(tabId, INJECT_SCRIPTS_STEP);
 				const scriptsInjected = await singlefile.extension.injectScript(tabId, tabOptions);
 				if (scriptsInjected) {
 					ui.onStart(tabId, EXECUTE_SCRIPTS_STEP);
-					tasks.push({ id: currentTaskId, status: "pending", tab, options: tabOptions, method: "content.save" });
-					currentTaskId++;
+					addTask({
+						status: TASK_PENDING_STATE,
+						tab,
+						options: tabOptions,
+						method: "content.save"
+					});
 				} else {
 					ui.onForbiddenDomain(tab);
 				}
@@ -133,6 +146,23 @@ singlefile.extension.core.bg.business = (() => {
 		runTasks();
 	}
 
+	function addTask(info) {
+		const taskInfo = {
+			id: currentTaskId,
+			status: info.status,
+			tab: info.tab,
+			options: info.options,
+			method: info.method,
+			done: function () {
+				tasks.splice(tasks.findIndex(taskInfo => taskInfo.id == this.id), 1);
+				runTasks();
+			}
+		};
+		tasks.push(taskInfo);
+		currentTaskId++;
+		return taskInfo;
+	}
+
 	function openEditor(tab) {
 		singlefile.extension.core.bg.tabs.sendMessage(tab.id, { method: "content.openEditor" });
 	}
@@ -144,64 +174,57 @@ singlefile.extension.core.bg.business = (() => {
 	}
 
 	function runTasks() {
-		const processingCount = tasks.filter(taskInfo => taskInfo.status == "processing").length;
+		const processingCount = tasks.filter(taskInfo => taskInfo.status == TASK_PROCESSING_STATE).length;
 		for (let index = 0; index < Math.min(tasks.length - processingCount, (maxParallelWorkers - processingCount)); index++) {
-			const taskInfo = tasks.find(taskInfo => taskInfo.status == "pending");
+			const taskInfo = tasks.find(taskInfo => taskInfo.status == TASK_PENDING_STATE);
 			if (taskInfo) {
 				runTask(taskInfo);
 			}
 		}
 	}
 
-	function runTask(taskInfo) {
+	async function runTask(taskInfo) {
 		const ui = singlefile.extension.ui.bg.main;
 		const tabs = singlefile.extension.core.bg.tabs;
 		const taskId = taskInfo.id;
-		return new Promise(async (resolve, reject) => {
-			taskInfo.status = "processing";
-			taskInfo.resolve = () => {
-				tasks.splice(tasks.findIndex(taskInfo => taskInfo.id == taskId), 1);
-				resolve();
-				runTasks();
-			};
-			taskInfo.reject = error => {
-				tasks.splice(tasks.findIndex(taskInfo => taskInfo.id == taskId), 1);
-				reject(error);
-				runTasks();
-			};
-			if (!taskInfo.tab.id) {
-				let scriptsInjected;
-				try {
-					const tab = await tabs.createAndWait({ url: taskInfo.tab.url, active: false });
-					taskInfo.tab.id = taskInfo.options.tabId = tab.id;
-					taskInfo.tab.index = taskInfo.options.tabIndex = tab.index;
-					ui.onStart(taskInfo.tab.id, INJECT_SCRIPTS_STEP);
-					scriptsInjected = await singlefile.extension.injectScript(taskInfo.tab.id, taskInfo.options);
-				} catch (tabId) {
-					taskInfo.tab.id = tabId;
-				}
-				if (scriptsInjected) {
-					ui.onStart(taskInfo.tab.id, EXECUTE_SCRIPTS_STEP);
-				} else {
-					taskInfo.reject();
-					return;
-				}
+		taskInfo.status = TASK_PROCESSING_STATE;
+		if (!taskInfo.tab.id) {
+			let scriptsInjected;
+			try {
+				const tab = await tabs.createAndWait({ url: taskInfo.tab.url, active: false });
+				taskInfo.tab.id = taskInfo.options.tabId = tab.id;
+				taskInfo.tab.index = taskInfo.options.tabIndex = tab.index;
+				ui.onStart(taskInfo.tab.id, INJECT_SCRIPTS_STEP);
+				scriptsInjected = await singlefile.extension.injectScript(taskInfo.tab.id, taskInfo.options);
+			} catch (tabId) {
+				taskInfo.tab.id = tabId;
 			}
-			taskInfo.options.taskId = taskId;
-			tabs.sendMessage(taskInfo.tab.id, { method: taskInfo.method, options: taskInfo.options })
-				.then(() => {
-					if (taskInfo.options.autoClose && !taskInfo.cancelled) {
-						tabs.remove(taskInfo.tab.id);
-					}
-				})
-				.catch(error => {
-					if (error && (!error.message || (error.message != ERROR_CONNECTION_LOST_CHROMIUM && error.message != ERROR_CONNECTION_ERROR_CHROMIUM && error.message != ERROR_CONNECTION_LOST_GECKO))) {
-						console.log(error); // eslint-disable-line no-console
-						ui.onError(taskInfo.tab.id);
-						taskInfo.reject(error);
-					}
-				});
-		});
+			if (scriptsInjected) {
+				ui.onStart(taskInfo.tab.id, EXECUTE_SCRIPTS_STEP);
+			} else {
+				taskInfo.done();
+				return;
+			}
+		}
+		taskInfo.options.taskId = taskId;
+		try {
+			await tabs.sendMessage(taskInfo.tab.id, { method: taskInfo.method, options: taskInfo.options });
+			if (taskInfo.options.autoClose && !taskInfo.cancelled) {
+				tabs.remove(taskInfo.tab.id);
+			}
+		} catch (error) {
+			if (error && (!error.message || !isIgnoredError(error))) {
+				console.log(error); // eslint-disable-line no-console
+				ui.onError(taskInfo.tab.id);
+				taskInfo.done();
+			}
+		}
+	}
+
+	function isIgnoredError(error) {
+		return error.message == ERROR_CONNECTION_LOST_CHROMIUM ||
+			error.message == ERROR_CONNECTION_ERROR_CHROMIUM ||
+			error.message == ERROR_CONNECTION_LOST_GECKO;
 	}
 
 	function cancelTab(tabId) {
@@ -210,7 +233,6 @@ singlefile.extension.core.bg.business = (() => {
 
 	function cancelTask(taskInfo) {
 		const tabId = taskInfo.tab.id;
-		const taskId = taskInfo.id;
 		taskInfo.cancelled = true;
 		singlefile.extension.core.bg.tabs.sendMessage(tabId, { method: "content.cancelSave", resetZoomLevel: taskInfo.options.loadDeferredImagesKeepZoomLevel });
 		if (taskInfo.cancel) {
@@ -220,10 +242,7 @@ singlefile.extension.core.bg.business = (() => {
 			singlefile.extension.ui.bg.main.onEnd(tabId, true);
 		}
 		singlefile.extension.ui.bg.main.onCancelled(taskInfo.tab);
-		tasks.splice(tasks.findIndex(taskInfo => taskInfo.id == taskId), 1);
-		if (taskInfo.resolve) {
-			taskInfo.resolve();
-		}
+		taskInfo.done();
 	}
 
 	function mapTaskInfo(taskInfo) {

extension/core/bg/config.js

@@ -44,7 +44,7 @@ singlefile.extension.core.bg.config = (() => {
 		loadDeferredImagesBlockCookies: false,
 		loadDeferredImagesBlockStorage: false,
 		loadDeferredImagesKeepZoomLevel: false,
-		filenameTemplate: "{page-title} ({date-iso} {time-locale}).html",
+		filenameTemplate: "{page-title} ({date-locale} {time-locale}).html",
 		infobarTemplate: "",
 		includeInfobar: false,
 		confirmInfobarContent: false,
@@ -67,6 +67,7 @@ singlefile.extension.core.bg.config = (() => {
 		displayInfobar: true,
 		displayStats: false,
 		backgroundSave: true,
+		defaultEditorMode: "normal",
 		applySystemTheme: true,
 		autoSaveDelay: 1,
 		autoSaveLoad: false,

extension/core/bg/downloads.js

@@ -115,7 +115,9 @@ singlefile.extension.core.bg.downloads = (() => {
 					compressHTML: message.compressHTML,
 					bookmarkId: message.bookmarkId,
 					replaceBookmarkURL: message.replaceBookmarkURL,
-					applySystemTheme: message.applySystemTheme
+					applySystemTheme: message.applySystemTheme,
+					defaultEditorMode: message.defaultEditorMode,
+					includeInfobar: message.includeInfobar
 				});
 			} else {
 				if (message.saveToClipboard) {
@@ -145,7 +147,8 @@ singlefile.extension.core.bg.downloads = (() => {
 					confirmFilename: message.confirmFilename,
 					incognito,
 					filenameConflictAction: message.filenameConflictAction,
-					filenameReplacementCharacter: message.filenameReplacementCharacter
+					filenameReplacementCharacter: message.filenameReplacementCharacter,
+					includeInfobar: message.includeInfobar
 				});
 			}
 			singlefile.extension.ui.bg.main.onEnd(tabId);

extension/core/bg/tabs.js

@@ -149,7 +149,14 @@ singlefile.extension.core.bg.tabs = (() => {
 
 	async function onTabUpdated(tabId, changeInfo) {
 		if (changeInfo.status == "complete") {
-			setTimeout(() => browser.tabs.sendMessage(tabId, { method: "content.maybeInit" }), DELAY_MAYBE_INIT);
+			setTimeout(async () => {
+				try {
+					await browser.tabs.sendMessage(tabId, { method: "content.maybeInit" });
+				}
+				catch (error) {
+					// ignored
+				}
+			}, DELAY_MAYBE_INIT);
 			const tab = await browser.tabs.get(tabId);
 			if (singlefile.extension.core.bg.editor.isEditor(tab)) {
 				const tabsData = await singlefile.extension.core.bg.tabsData.get(tab.id);

extension/core/content/content-download.js

@@ -56,7 +56,9 @@ this.singlefile.extension.core.content.download = this.singlefile.extension.core
 					backgroundSave: options.backgroundSave,
 					bookmarkId: options.bookmarkId,
 					replaceBookmarkURL: options.replaceBookmarkURL,
-					applySystemTheme: options.applySystemTheme
+					applySystemTheme: options.applySystemTheme,
+					defaultEditorMode: options.defaultEditorMode,
+					includeInfobar: options.includeInfobar
 				};
 				message.truncated = pageData.content.length > MAX_CONTENT_SIZE;
 				if (message.truncated) {

extension/lib/readability/Readability-readerable.js

@@ -1,5 +1,4 @@
 /* eslint-env es6:false */
-/* globals exports */
 /*
  * Copyright (c) 2010 Arc90 Inc
  *
@@ -95,6 +94,6 @@ function isProbablyReaderable(doc, isVisible) {
   });
 }
 
-if (typeof exports === "object") {
-  exports.isProbablyReaderable = isProbablyReaderable;
+if (typeof module === "object") {
+  module.exports = isProbablyReaderable;
 }

extension/lib/readability/Readability.js

@@ -50,6 +50,10 @@ function Readability(doc, options) {
   this._charThreshold = options.charThreshold || this.DEFAULT_CHAR_THRESHOLD;
   this._classesToPreserve = this.CLASSES_TO_PRESERVE.concat(options.classesToPreserve || []);
   this._keepClasses = !!options.keepClasses;
+  this._serializer = options.serializer || function(el) {
+    return el.innerHTML;
+  };
+  this._disableJSONLD = !!options.disableJSONLD;
 
   // Start with all flags set
   this._flags = this.FLAG_STRIP_UNLIKELYS |
@@ -131,8 +135,14 @@ Readability.prototype = {
     prevLink: /(prev|earl|old|new|<|«)/i,
     whitespace: /^\s*$/,
     hasContent: /\S$/,
+    srcsetUrl: /(\S+)(\s+[\d.]+[xw])?(\s*(?:,|$))/g,
+    b64DataUrl: /^data:\s*([^\s;,]+)\s*;\s*base64\s*,/i,
+    // See: https://schema.org/Article
+    jsonLdArticleTypes: /^Article|AdvertiserContentArticle|NewsArticle|AnalysisNewsArticle|AskPublicNewsArticle|BackgroundNewsArticle|OpinionNewsArticle|ReportageNewsArticle|ReviewNewsArticle|Report|SatiricalArticle|ScholarlyArticle|MedicalScholarlyArticle|SocialMediaPosting|BlogPosting|LiveBlogPosting|DiscussionForumPosting|TechArticle|APIReference$/
   },
 
+  UNLIKELY_ROLES: [ "menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog" ],
+
   DIV_TO_P_ELEMS: [ "A", "BLOCKQUOTE", "DL", "DIV", "IMG", "OL", "P", "PRE", "TABLE", "UL", "SELECT" ],
 
   ALTER_TO_DIV_EXCEPTIONS: ["DIV", "ARTICLE", "SECTION", "P"],
@@ -155,6 +165,15 @@ Readability.prototype = {
   // These are the classes that readability sets itself.
   CLASSES_TO_PRESERVE: [ "page" ],
 
+  // These are the list of HTML entities that need to be escaped.
+  HTML_ESCAPE_MAP: {
+    "lt": "<",
+    "gt": ">",
+    "amp": "&",
+    "quot": '"',
+    "apos": "'",
+  },
+
   /**
    * Run any post-process modifications to article content as necessary.
    *
@@ -165,6 +184,8 @@ Readability.prototype = {
     // Readability cannot open relative uris so we convert them to absolute uris.
     this._fixRelativeUris(articleContent);
 
+    this._simplifyNestedElements(articleContent);
+
     if (!this._keepClasses) {
       // Remove classes.
       this._cleanClasses(articleContent);
@@ -230,6 +251,21 @@ Readability.prototype = {
     Array.prototype.forEach.call(nodeList, fn, this);
   },
 
+  /**
+   * Iterate over a NodeList, and return the first node that passes
+   * the supplied test function
+   *
+   * For convenience, the current object context is applied to the provided
+   * test function.
+   *
+   * @param  NodeList nodeList The NodeList.
+   * @param  Function fn       The test function.
+   * @return void
+   */
+  _findNode: function(nodeList, fn) {
+    return Array.prototype.find.call(nodeList, fn, this);
+  },
+
   /**
    * Iterate over a NodeList, return true if any of the provided iterate
    * function calls returns true, false otherwise.
@@ -328,6 +364,7 @@ Readability.prototype = {
       if (baseURI == documentURI && uri.charAt(0) == "#") {
         return uri;
       }
+
       // Otherwise, resolve against base URI:
       try {
         return new URL(uri, baseURI).href;
@@ -362,15 +399,56 @@ Readability.prototype = {
       }
     });
 
-    var imgs = this._getAllNodesWithTag(articleContent, ["img"]);
-    this._forEachNode(imgs, function(img) {
-      var src = img.getAttribute("src");
+    var medias = this._getAllNodesWithTag(articleContent, [
+      "img", "picture", "figure", "video", "audio", "source"
+    ]);
+
+    this._forEachNode(medias, function(media) {
+      var src = media.getAttribute("src");
+      var poster = media.getAttribute("poster");
+      var srcset = media.getAttribute("srcset");
+
       if (src) {
-        img.setAttribute("src", toAbsoluteURI(src));
+        media.setAttribute("src", toAbsoluteURI(src));
+      }
+
+      if (poster) {
+        media.setAttribute("poster", toAbsoluteURI(poster));
+      }
+
+      if (srcset) {
+        var newSrcset = srcset.replace(this.REGEXPS.srcsetUrl, function(_, p1, p2, p3) {
+          return toAbsoluteURI(p1) + (p2 || "") + p3;
+        });
+
+        media.setAttribute("srcset", newSrcset);
       }
     });
   },
 
+  _simplifyNestedElements: function(articleContent) {
+    var node = articleContent;
+
+    while (node) {
+      if (node.parentNode && ["DIV", "SECTION"].includes(node.tagName) && !(node.id && node.id.startsWith("readability"))) {
+        if (this._isElementWithoutContent(node)) {
+          node = this._removeAndGetNext(node);
+          continue;
+        } else if (this._hasSingleTagInsideElement(node, "DIV") || this._hasSingleTagInsideElement(node, "SECTION")) {
+          var child = node.children[0];
+          for (var i = 0; i < node.attributes.length; i++) {
+            child.setAttribute(node.attributes[i].name, node.attributes[i].value);
+          }
+          node.parentNode.replaceChild(child, node);
+          node = child;
+          continue;
+        }
+      }
+
+      node = this._getNextNode(node);
+    }
+  },
+
   /**
    * Get the article title as an H1.
    *
@@ -841,8 +919,8 @@ Readability.prototype = {
             continue;
           }
 
-          if (node.getAttribute("role") == "complementary") {
-            this.log("Removing complementary content - " + matchString);
+          if (this.UNLIKELY_ROLES.includes(node.getAttribute("role"))) {
+            this.log("Removing content with role " + node.getAttribute("role") + " - " + matchString);
             node = this._removeAndGetNext(node);
             continue;
           }
@@ -919,7 +997,7 @@ Readability.prototype = {
           return;
 
         // Exclude nodes with no ancestor.
-        var ancestors = this._getNodeAncestors(elementToScore, 3);
+        var ancestors = this._getNodeAncestors(elementToScore, 5);
         if (ancestors.length === 0)
           return;
 
@@ -1239,12 +1317,111 @@ Readability.prototype = {
     return false;
   },
 
+  /**
+   * Converts some of the common HTML entities in string to their corresponding characters.
+   *
+   * @param str {string} - a string to unescape.
+   * @return string without HTML entity.
+   */
+  _unescapeHtmlEntities: function(str) {
+    if (!str) {
+      return str;
+    }
+
+    var htmlEscapeMap = this.HTML_ESCAPE_MAP;
+    return str.replace(/&(quot|amp|apos|lt|gt);/g, function(_, tag) {
+      return htmlEscapeMap[tag];
+    }).replace(/&#(?:x([0-9a-z]{1,4})|([0-9]{1,4}));/gi, function(_, hex, numStr) {
+      var num = parseInt(hex || numStr, hex ? 16 : 10);
+      return String.fromCharCode(num);
+    });
+  },
+
+  /**
+   * Try to extract metadata from JSON-LD object.
+   * For now, only Schema.org objects of type Article or its subtypes are supported.
+   * @return Object with any metadata that could be extracted (possibly none)
+   */
+  _getJSONLD: function (doc) {
+    var scripts = this._getAllNodesWithTag(doc, ["script"]);
+
+    var jsonLdElement = this._findNode(scripts, function(el) {
+      return el.getAttribute("type") === "application/ld+json";
+    });
+
+    if (jsonLdElement) {
+      try {
+        // Strip CDATA markers if present
+        var content = jsonLdElement.textContent.replace(/^\s*<!\[CDATA\[|\]\]>\s*$/g, "");
+        var parsed = JSON.parse(content);
+        var metadata = {};
+        if (
+          !parsed["@context"] ||
+          !parsed["@context"].match(/^https?\:\/\/schema\.org$/)
+        ) {
+          return metadata;
+        }
+
+        if (!parsed["@type"] && Array.isArray(parsed["@graph"])) {
+          parsed = parsed["@graph"].find(function(it) {
+            return (it["@type"] || "").match(
+              this.REGEXPS.jsonLdArticleTypes
+            );
+          });
+        }
+
+        if (
+          !parsed ||
+          !parsed["@type"] ||
+          !parsed["@type"].match(this.REGEXPS.jsonLdArticleTypes)
+        ) {
+          return metadata;
+        }
+        if (typeof parsed.name === "string") {
+          metadata.title = parsed.name.trim();
+        } else if (typeof parsed.headline === "string") {
+          metadata.title = parsed.headline.trim();
+        }
+        if (parsed.author) {
+          if (typeof parsed.author.name === "string") {
+            metadata.byline = parsed.author.name.trim();
+          } else if (Array.isArray(parsed.author) && parsed.author[0] && typeof parsed.author[0].name === "string") {
+            metadata.byline = parsed.author
+              .filter(function(author) {
+                return author && typeof author.name === "string";
+              })
+              .map(function(author) {
+                return author.name.trim();
+              })
+              .join(", ");
+          }
+        }
+        if (typeof parsed.description === "string") {
+          metadata.excerpt = parsed.description.trim();
+        }
+        if (
+          parsed.publisher &&
+          typeof parsed.publisher.name === "string"
+        ) {
+          metadata.siteName = parsed.publisher.name.trim();
+        }
+        return metadata;
+      } catch (err) {
+        this.log(err.message);
+      }
+    }
+    return {};
+  },
+
   /**
    * Attempts to get excerpt and byline metadata for the article.
    *
+   * @param {Object} jsonld — object containing any metadata that
+   * could be extracted from JSON-LD object.
+   *
    * @return Object with optional "excerpt" and "byline" properties
    */
-  _getArticleMetadata: function() {
+  _getArticleMetadata: function(jsonld) {
     var metadata = {};
     var values = {};
     var metaElements = this._doc.getElementsByTagName("meta");
@@ -1290,7 +1467,8 @@ Readability.prototype = {
     });
 
     // get title
-    metadata.title = values["dc:title"] ||
+    metadata.title = jsonld.title ||
+                     values["dc:title"] ||
                      values["dcterm:title"] ||
                      values["og:title"] ||
                      values["weibo:article:title"] ||
@@ -1303,12 +1481,14 @@ Readability.prototype = {
     }
 
     // get author
-    metadata.byline = values["dc:creator"] ||
+    metadata.byline = jsonld.byline ||
+                      values["dc:creator"] ||
                       values["dcterm:creator"] ||
                       values["author"];
 
     // get description
-    metadata.excerpt = values["dc:description"] ||
+    metadata.excerpt = jsonld.excerpt ||
+                       values["dc:description"] ||
                        values["dcterm:description"] ||
                        values["og:description"] ||
                        values["weibo:article:description"] ||
@@ -1317,7 +1497,15 @@ Readability.prototype = {
                        values["twitter:description"];
 
     // get site name
-    metadata.siteName = values["og:site_name"];
+    metadata.siteName = jsonld.siteName ||
+                        values["og:site_name"];
+
+    // in many sites the meta value is escaped with HTML entities,
+    // so here we need to unescape it
+    metadata.title = this._unescapeHtmlEntities(metadata.title);
+    metadata.byline = this._unescapeHtmlEntities(metadata.byline);
+    metadata.excerpt = this._unescapeHtmlEntities(metadata.excerpt);
+    metadata.siteName = this._unescapeHtmlEntities(metadata.siteName);
 
     return metadata;
   },
@@ -1745,30 +1933,67 @@ Readability.prototype = {
   /* convert images and figures that have properties like data-src into images that can be loaded without JS */
   _fixLazyImages: function (root) {
     this._forEachNode(this._getAllNodesWithTag(root, ["img", "picture", "figure"]), function (elem) {
-      // also check for "null" to work around https://github.com/jsdom/jsdom/issues/2580
-      if ((!elem.src && (!elem.srcset || elem.srcset == "null")) || elem.className.toLowerCase().indexOf("lazy") !== -1) {
+      // In some sites (e.g. Kotaku), they put 1px square image as base64 data uri in the src attribute.
+      // So, here we check if the data uri is too short, just might as well remove it.
+      if (elem.src && this.REGEXPS.b64DataUrl.test(elem.src)) {
+        // Make sure it's not SVG, because SVG can have a meaningful image in under 133 bytes.
+        var parts = this.REGEXPS.b64DataUrl.exec(elem.src);
+        if (parts[1] === "image/svg+xml") {
+          return;
+        }
+
+        // Make sure this element has other attributes which contains image.
+        // If it doesn't, then this src is important and shouldn't be removed.
+        var srcCouldBeRemoved = false;
         for (var i = 0; i < elem.attributes.length; i++) {
           var attr = elem.attributes[i];
-          if (attr.name === "src" || attr.name === "srcset") {
+          if (attr.name === "src") {
             continue;
           }
-          var copyTo = null;
-          if (/\.(jpg|jpeg|png|webp)\s+\d/.test(attr.value)) {
-            copyTo = "srcset";
-          } else if (/^\s*\S+\.(jpg|jpeg|png|webp)\S*\s*$/.test(attr.value)) {
-            copyTo = "src";
+
+          if (/\.(jpg|jpeg|png|webp)/i.test(attr.value)) {
+            srcCouldBeRemoved = true;
+            break;
           }
-          if (copyTo) {
-            //if this is an img or picture, set the attribute directly
-            if (elem.tagName === "IMG" || elem.tagName === "PICTURE") {
-              elem.setAttribute(copyTo, attr.value);
-            } else if (elem.tagName === "FIGURE" && !this._getAllNodesWithTag(elem, ["img", "picture"]).length) {
-              //if the item is a <figure> that does not contain an image or picture, create one and place it inside the figure
-              //see the nytimes-3 testcase for an example
-              var img = this._doc.createElement("img");
-              img.setAttribute(copyTo, attr.value);
-              elem.appendChild(img);
-            }
+        }
+
+        // Here we assume if image is less than 100 bytes (or 133B after encoded to base64)
+        // it will be too small, therefore it might be placeholder image.
+        if (srcCouldBeRemoved) {
+          var b64starts = elem.src.search(/base64\s*/i) + 7;
+          var b64length = elem.src.length - b64starts;
+          if (b64length < 133) {
+            elem.removeAttribute("src");
+          }
+        }
+      }
+
+      // also check for "null" to work around https://github.com/jsdom/jsdom/issues/2580
+      if ((elem.src || (elem.srcset && elem.srcset != "null")) && elem.className.toLowerCase().indexOf("lazy") === -1) {
+        return;
+      }
+
+      for (var j = 0; j < elem.attributes.length; j++) {
+        attr = elem.attributes[j];
+        if (attr.name === "src" || attr.name === "srcset") {
+          continue;
+        }
+        var copyTo = null;
+        if (/\.(jpg|jpeg|png|webp)\s+\d/.test(attr.value)) {
+          copyTo = "srcset";
+        } else if (/^\s*\S+\.(jpg|jpeg|png|webp)\S*\s*$/.test(attr.value)) {
+          copyTo = "src";
+        }
+        if (copyTo) {
+          //if this is an img or picture, set the attribute directly
+          if (elem.tagName === "IMG" || elem.tagName === "PICTURE") {
+            elem.setAttribute(copyTo, attr.value);
+          } else if (elem.tagName === "FIGURE" && !this._getAllNodesWithTag(elem, ["img", "picture"]).length) {
+            //if the item is a <figure> that does not contain an image or picture, create one and place it inside the figure
+            //see the nytimes-3 testcase for an example
+            var img = this._doc.createElement("img");
+            img.setAttribute(copyTo, attr.value);
+            elem.appendChild(img);
           }
         }
       }
@@ -1932,12 +2157,15 @@ Readability.prototype = {
     // Unwrap image from noscript
     this._unwrapNoscriptImages(this._doc);
 
+    // Extract JSON-LD metadata before removing scripts
+    var jsonLd = this._disableJSONLD ? {} : this._getJSONLD(this._doc);
+
     // Remove script tags from the document.
     this._removeScripts(this._doc);
 
     this._prepDocument();
 
-    var metadata = this._getArticleMetadata();
+    var metadata = this._getArticleMetadata(jsonLd);
     this._articleTitle = metadata.title;
 
     var articleContent = this._grabArticle();
@@ -1963,7 +2191,7 @@ Readability.prototype = {
       title: this._articleTitle,
       byline: metadata.byline || this._articleByline,
       dir: this._articleDir,
-      content: articleContent.innerHTML,
+      content: this._serializer(articleContent),
       textContent: textContent,
       length: textContent.length,
       excerpt: metadata.excerpt,

extension/lib/single-file/fetch/content/content-fetch.js

@@ -37,7 +37,7 @@ this.singlefile.extension.lib.fetch.content.resources = this.singlefile.extensio
 	async function onMessage(message) {
 		try {
 			let response = await fetch(message.url, { cache: "force-cache" });
-			if (response.status == 403) {
+			if (response.status == 401 || response.status == 403 || response.status == 404) {
 				response = hostFetch(message.url);
 			}
 			return {
@@ -56,7 +56,7 @@ this.singlefile.extension.lib.fetch.content.resources = this.singlefile.extensio
 		fetch: async url => {
 			try {
 				let response = await fetch(url, { cache: "force-cache" });
-				if (response.status == 403 || response.status == 404) {
+				if (response.status == 401 || response.status == 403 || response.status == 404) {
 					response = hostFetch(url);
 				}
 				return response;

extension/ui/bg/ui-editor.js

@@ -26,6 +26,7 @@
 singlefile.extension.ui.bg.editor = (() => {
 
 	const editorElement = document.querySelector(".editor");
+	const toolbarElement = document.querySelector(".toolbar");
 	const highlightYellowButton = document.querySelector(".highlight-yellow-button");
 	const highlightPinkButton = document.querySelector(".highlight-pink-button");
 	const highlightBlueButton = document.querySelector(".highlight-blue-button");
@@ -43,6 +44,7 @@ singlefile.extension.ui.bg.editor = (() => {
 	const cutPageButton = document.querySelector(".cut-page-button");
 	const undoCutPageButton = document.querySelector(".undo-cut-page-button");
 	const undoAllCutPageButton = document.querySelector(".undo-all-cut-page-button");
+	const redoCutPageButton = document.querySelector(".redo-cut-page-button");
 	const savePageButton = document.querySelector(".save-page-button");
 
 	let tabData, tabDataContents = [];
@@ -63,52 +65,31 @@ singlefile.extension.ui.bg.editor = (() => {
 	cutPageButton.title = browser.i18n.getMessage("editorCutPage");
 	undoCutPageButton.title = browser.i18n.getMessage("editorUndoCutPage");
 	undoAllCutPageButton.title = browser.i18n.getMessage("editorUndoAllCutPage");
+	redoCutPageButton.title = browser.i18n.getMessage("editorRedoCutPage");
 	savePageButton.title = browser.i18n.getMessage("editorSavePage");
 
 	addYellowNoteButton.onclick = () => editorElement.contentWindow.postMessage(JSON.stringify({ method: "addNote", color: "note-yellow" }), "*");
 	addPinkNoteButton.onclick = () => editorElement.contentWindow.postMessage(JSON.stringify({ method: "addNote", color: "note-pink" }), "*");
 	addBlueNoteButton.onclick = () => editorElement.contentWindow.postMessage(JSON.stringify({ method: "addNote", color: "note-blue" }), "*");
 	addGreenNoteButton.onclick = () => editorElement.contentWindow.postMessage(JSON.stringify({ method: "addNote", color: "note-green" }), "*");
-	highlightYellowButton.onclick = () => {
-		if (highlightYellowButton.classList.contains("highlight-disabled")) {
-			highlightButtons.forEach(highlightButton => highlightButton.classList.add("highlight-disabled"));
-			highlightYellowButton.classList.remove("highlight-disabled");
-			editorElement.contentWindow.postMessage(JSON.stringify({ method: "enableHighlight", color: "single-file-highlight-yellow" }), "*");
-		} else {
-			highlightYellowButton.classList.add("highlight-disabled");
-			editorElement.contentWindow.postMessage(JSON.stringify({ method: "disableHighlight" }), "*");
-		}
-	};
-	highlightPinkButton.onclick = () => {
-		if (highlightPinkButton.classList.contains("highlight-disabled")) {
-			highlightButtons.forEach(highlightButton => highlightButton.classList.add("highlight-disabled"));
-			highlightPinkButton.classList.remove("highlight-disabled");
-			editorElement.contentWindow.postMessage(JSON.stringify({ method: "enableHighlight", color: "single-file-highlight-pink" }), "*");
-		} else {
-			highlightPinkButton.classList.add("highlight-disabled");
-			editorElement.contentWindow.postMessage(JSON.stringify({ method: "disableHighlight" }), "*");
-		}
-	};
-	highlightBlueButton.onclick = () => {
-		if (highlightBlueButton.classList.contains("highlight-disabled")) {
-			highlightButtons.forEach(highlightButton => highlightButton.classList.add("highlight-disabled"));
-			highlightBlueButton.classList.remove("highlight-disabled");
-			editorElement.contentWindow.postMessage(JSON.stringify({ method: "enableHighlight", color: "single-file-highlight-blue" }), "*");
-		} else {
-			highlightBlueButton.classList.add("highlight-disabled");
-			editorElement.contentWindow.postMessage(JSON.stringify({ method: "disableHighlight" }), "*");
-		}
-	};
-	highlightGreenButton.onclick = () => {
-		if (highlightGreenButton.classList.contains("highlight-disabled")) {
-			highlightButtons.forEach(highlightButton => highlightButton.classList.add("highlight-disabled"));
-			highlightGreenButton.classList.remove("highlight-disabled");
-			editorElement.contentWindow.postMessage(JSON.stringify({ method: "enableHighlight", color: "single-file-highlight-green" }), "*");
-		} else {
-			highlightGreenButton.classList.add("highlight-disabled");
-			editorElement.contentWindow.postMessage(JSON.stringify({ method: "disableHighlight" }), "*");
-		}
-	};
+	highlightButtons.forEach(highlightButton => {
+		highlightButton.onclick = () => {
+			if (toolbarElement.classList.contains("cut-mode")) {
+				disableCutPage();
+			}
+			if (toolbarElement.classList.contains("remove-highlight-mode")) {
+				disableRemoveHighlights();
+			}
+			const disabled = highlightButton.classList.contains("highlight-disabled");
+			resetHighlightButtons();
+			if (disabled) {
+				highlightButton.classList.remove("highlight-disabled");
+				editorElement.contentWindow.postMessage(JSON.stringify({ method: "enableHighlight", color: "single-file-highlight-" + highlightButton.dataset.color }), "*");
+			} else {
+				highlightButton.classList.add("highlight-disabled");
+			}
+		};
+	});
 	toggleNotesButton.onclick = () => {
 		if (toggleNotesButton.getAttribute("src") == "/extension/ui/resources/button_note_visible.png") {
 			toggleNotesButton.src = "/extension/ui/resources/button_note_hidden.png";
@@ -123,52 +104,65 @@ singlefile.extension.ui.bg.editor = (() => {
 			toggleHighlightsButton.src = "/extension/ui/resources/button_highlighter_hidden.png";
 			editorElement.contentWindow.postMessage(JSON.stringify({ method: "hideHighlights" }), "*");
 		} else {
-			toggleHighlightsButton.src = "/extension/ui/resources/button_highlighter_visible.png";
-			editorElement.contentWindow.postMessage(JSON.stringify({ method: "displayHighlights" }), "*");
+			displayHighlights();
 		}
 	};
 	removeHighlightButton.onclick = () => {
+		if (toolbarElement.classList.contains("cut-mode")) {
+			disableCutPage();
+		}
 		if (removeHighlightButton.classList.contains("remove-highlight-disabled")) {
 			removeHighlightButton.classList.remove("remove-highlight-disabled");
+			toolbarElement.classList.add("remove-highlight-mode");
+			resetHighlightButtons();
+			displayHighlights();
 			editorElement.contentWindow.postMessage(JSON.stringify({ method: "enableRemoveHighlights" }), "*");
+			editorElement.contentWindow.postMessage(JSON.stringify({ method: "displayHighlights" }), "*");
 		} else {
-			removeHighlightButton.classList.add("remove-highlight-disabled");
-			editorElement.contentWindow.postMessage(JSON.stringify({ method: "disableRemoveHighlights" }), "*");
+			disableRemoveHighlights();
 		}
 	};
 	editPageButton.onclick = () => {
+		if (toolbarElement.classList.contains("cut-mode")) {
+			disableCutPage();
+		}
 		if (editPageButton.classList.contains("edit-disabled")) {
-			editPageButton.classList.remove("edit-disabled");
-			editorElement.contentWindow.postMessage(JSON.stringify({ method: "enableEditPage" }), "*");
+			enableEditPage();
 		} else {
-			editPageButton.classList.add("edit-disabled");
-			editorElement.contentWindow.postMessage(JSON.stringify({ method: "disableEditPage" }), "*");
+			disableEditPage();
 		}
 	};
 	formatPageButton.onclick = () => {
-		if (formatPageButton.classList.contains("format-disabled")) {
-			formatPageButton.classList.remove("format-disabled");
-			editorElement.contentWindow.postMessage(JSON.stringify({ method: tabData.options.applySystemTheme ? "formatPage" : "formatPageNoTheme" }), "*");
-		}
+		enableFormatPage();
 	};
 	cutPageButton.onclick = () => {
+		if (toolbarElement.classList.contains("edit-mode")) {
+			disableEditPage();
+		}
 		if (cutPageButton.classList.contains("cut-disabled")) {
-			cutPageButton.classList.remove("cut-disabled");
-			editorElement.contentWindow.postMessage(JSON.stringify({ method: "enableCutPage" }), "*");
+			enableCutPage();
+			editorElement.contentWindow.focus();
 		} else {
-			cutPageButton.classList.add("cut-disabled");
-			editorElement.contentWindow.postMessage(JSON.stringify({ method: "disableCutPage" }), "*");
+			disableCutPage();
 		}
 	};
 	undoCutPageButton.onclick = () => {
 		editorElement.contentWindow.postMessage(JSON.stringify({ method: "undoCutPage" }), "*");
+		editorElement.contentWindow.focus();
 	};
 	undoAllCutPageButton.onclick = () => {
 		editorElement.contentWindow.postMessage(JSON.stringify({ method: "undoAllCutPage" }), "*");
+		editorElement.contentWindow.focus();
+	};
+	redoCutPageButton.onclick = () => {
+		editorElement.contentWindow.postMessage(JSON.stringify({ method: "redoCutPage" }), "*");
+		editorElement.contentWindow.focus();
 	};
 	savePageButton.onclick = () => {
 		savePage();
 	};
+	let updatedResources = {};
+
 	window.onmessage = event => {
 		const message = JSON.parse(event.data);
 		if (message.method == "setMetadata") {
@@ -189,13 +183,58 @@ singlefile.extension.ui.bg.editor = (() => {
 			singlefile.extension.core.content.download.downloadPage(pageData, tabData.options);
 		}
 		if (message.method == "disableFormatPage") {
+			tabData.options.disableFormatPage = true;
 			formatPageButton.remove();
 		}
 		if (message.method == "onUpdate") {
 			tabData.docSaved = message.saved;
 		}
+		if (message.method == "onInit") {
+			if (tabData.options.defaultEditorMode == "edit") {
+				enableEditPage();
+			} else if (tabData.options.defaultEditorMode == "format" && !tabData.options.disableFormatPage) {
+				enableFormatPage();
+			} else if (tabData.options.defaultEditorMode == "cut") {
+				enableCutPage();
+			}
+		}
 	};
-	window.onload = browser.runtime.sendMessage({ method: "editor.getTabData" });
+
+	window.onload = () => {
+		browser.runtime.sendMessage({ method: "editor.getTabData" });
+		browser.runtime.onMessage.addListener(message => {
+			if (message.method == "devtools.resourceCommitted") {
+				updatedResources[message.url] = { content: message.content, type: message.type, encoding: message.encoding };
+				return Promise.resolve({});
+			}
+			if (message.method == "content.save") {
+				tabData.options = message.options;
+				savePage();
+				browser.runtime.sendMessage({ method: "ui.processInit" });
+				return Promise.resolve({});
+			}
+			if (message.method == "common.promptValueRequest") {
+				browser.runtime.sendMessage({ method: "tabs.promptValueResponse", value: prompt(message.promptMessage) });
+				return Promise.resolve({});
+			}
+			if (message.method == "editor.setTabData") {
+				if (message.truncated) {
+					tabDataContents.push(message.content);
+				} else {
+					tabDataContents = [message.content];
+				}
+				if (!message.truncated || message.finished) {
+					tabData = JSON.parse(tabDataContents.join(""));
+					tabData.docSaved = true;
+					tabDataContents = [];
+					editorElement.contentWindow.postMessage(JSON.stringify({ method: "init", content: tabData.content }), "*");
+					delete tabData.content;
+				}
+				return Promise.resolve({});
+			}
+		});
+	};
+
 	window.onbeforeunload = event => {
 		if (tabData.options.warnUnsavedPage && !tabData.docSaved) {
 			event.preventDefault();
@@ -203,36 +242,58 @@ singlefile.extension.ui.bg.editor = (() => {
 		}
 	};
 
-	browser.runtime.onMessage.addListener(message => {
-		if (message.method == "content.save") {
-			tabData.options = message.options;
-			savePage();
-			browser.runtime.sendMessage({ method: "ui.processInit" });
-			return Promise.resolve({});
-		}
-		if (message.method == "common.promptValueRequest") {
-			browser.runtime.sendMessage({ method: "tabs.promptValueResponse", value: prompt(message.promptMessage) });
-			return Promise.resolve({});
-		}
-		if (message.method == "editor.setTabData") {
-			if (message.truncated) {
-				tabDataContents.push(message.content);
-			} else {
-				tabDataContents = [message.content];
-			}
-			if (!message.truncated || message.finished) {
-				tabData = JSON.parse(tabDataContents.join(""));
-				tabData.docSaved = true;
-				tabDataContents = [];
-				editorElement.contentWindow.postMessage(JSON.stringify({ method: "init", content: tabData.content }), "*");
-				delete tabData.content;
-			}
-			return Promise.resolve({});
+	function disableEditPage() {
+		editPageButton.classList.add("edit-disabled");
+		toolbarElement.classList.remove("edit-mode");
+		editorElement.contentWindow.postMessage(JSON.stringify({ method: "disableEditPage" }), "*");
+	}
+
+	function disableCutPage() {
+		cutPageButton.classList.add("cut-disabled");
+		toolbarElement.classList.remove("cut-mode");
+		editorElement.contentWindow.postMessage(JSON.stringify({ method: "disableCutPage" }), "*");
+	}
+
+	function resetHighlightButtons() {
+		highlightButtons.forEach(highlightButton => highlightButton.classList.add("highlight-disabled"));
+		editorElement.contentWindow.postMessage(JSON.stringify({ method: "disableHighlight" }), "*");
+	}
+
+	function disableRemoveHighlights() {
+		toolbarElement.classList.remove("remove-highlight-mode");
+		removeHighlightButton.classList.add("remove-highlight-disabled");
+		editorElement.contentWindow.postMessage(JSON.stringify({ method: "disableRemoveHighlights" }), "*");
+	}
+
+	function displayHighlights() {
+		toggleHighlightsButton.src = "/extension/ui/resources/button_highlighter_visible.png";
+		editorElement.contentWindow.postMessage(JSON.stringify({ method: "displayHighlights" }), "*");
+	}
+
+	function enableEditPage() {
+		editPageButton.classList.remove("edit-disabled");
+		toolbarElement.classList.add("edit-mode");
+		editorElement.contentWindow.postMessage(JSON.stringify({ method: "enableEditPage" }), "*");
+	}
+
+	function enableFormatPage() {
+		if (formatPageButton.classList.contains("format-disabled")) {
+			formatPageButton.classList.remove("format-disabled");
+			updatedResources = {};
+			editorElement.contentWindow.postMessage(JSON.stringify({ method: tabData.options.applySystemTheme ? "formatPage" : "formatPageNoTheme" }), "*");
 		}
-	});
+	}
+
+	function enableCutPage() {
+		cutPageButton.classList.remove("cut-disabled");
+		toolbarElement.classList.add("cut-mode");
+		resetHighlightButtons();
+		disableRemoveHighlights();
+		editorElement.contentWindow.postMessage(JSON.stringify({ method: "enableCutPage" }), "*");
+	}
 
 	function savePage() {
-		editorElement.contentWindow.postMessage(JSON.stringify({ method: "getContent", compressHTML: tabData.options.compressHTML }), "*");
+		editorElement.contentWindow.postMessage(JSON.stringify({ method: "getContent", compressHTML: tabData.options.compressHTML, updatedResources }), "*");
 	}
 
 	return {};

extension/ui/bg/ui-options.js

@@ -93,6 +93,7 @@
 	const editorLabel = document.getElementById("editorLabel");
 	const openEditorLabel = document.getElementById("openEditorLabel");
 	const autoOpenEditorLabel = document.getElementById("autoOpenEditorLabel");
+	const defaultEditorModeLabel = document.getElementById("defaultEditorModeLabel");
 	const applySystemThemeLabel = document.getElementById("applySystemThemeLabel");
 	const warnUnsavedPageLabel = document.getElementById("warnUnsavedPageLabel");
 	const infobarTemplateLabel = document.getElementById("infobarTemplateLabel");
@@ -155,6 +156,11 @@
 	const autoCloseInput = document.getElementById("autoCloseInput");
 	const openEditorInput = document.getElementById("openEditorInput");
 	const autoOpenEditorInput = document.getElementById("autoOpenEditorInput");
+	const defaultEditorModeInput = document.getElementById("defaultEditorModeInput");
+	const defaultEditorModeNormalLabel = document.getElementById("defaultEditorModeNormalLabel");
+	const defaultEditorModeEditLabel = document.getElementById("defaultEditorModeEditLabel");
+	const defaultEditorModeFormatLabel = document.getElementById("defaultEditorModeFormatLabel");
+	const defaultEditorModeCutLabel = document.getElementById("defaultEditorModeCutLabel");
 	const applySystemThemeInput = document.getElementById("applySystemThemeInput");
 	const warnUnsavedPageInput = document.getElementById("warnUnsavedPageInput");
 	const expandAllButton = document.getElementById("expandAllButton");
@@ -494,6 +500,11 @@
 	editorLabel.textContent = browser.i18n.getMessage("optionsEditorSubTitle");
 	openEditorLabel.textContent = browser.i18n.getMessage("optionOpenEditor");
 	autoOpenEditorLabel.textContent = browser.i18n.getMessage("optionAutoOpenEditor");
+	defaultEditorModeLabel.textContent = browser.i18n.getMessage("optionDefaultEditorMode");
+	defaultEditorModeNormalLabel.textContent = browser.i18n.getMessage("optionDefaultEditorModeNormal");
+	defaultEditorModeEditLabel.textContent = browser.i18n.getMessage("optionDefaultEditorModeEdit");
+	defaultEditorModeFormatLabel.textContent = browser.i18n.getMessage("optionDefaultEditorModeFormat");
+	defaultEditorModeCutLabel.textContent = browser.i18n.getMessage("optionDefaultEditorModeCut");
 	applySystemThemeLabel.textContent = browser.i18n.getMessage("optionApplySystemTheme");
 	warnUnsavedPageLabel.textContent = browser.i18n.getMessage("optionWarnUnsavedPage");
 	resetButton.textContent = browser.i18n.getMessage("optionsResetButton");
@@ -682,6 +693,7 @@
 		autoCloseInput.checked = profileOptions.autoClose;
 		openEditorInput.checked = profileOptions.openEditor;
 		autoOpenEditorInput.checked = profileOptions.autoOpenEditor;
+		defaultEditorModeInput.value = profileOptions.defaultEditorMode;
 		applySystemThemeInput.checked = profileOptions.applySystemTheme;
 		warnUnsavedPageInput.checked = profileOptions.warnUnsavedPage;
 		removeFramesInput.disabled = saveRawPageInput.checked;
@@ -749,6 +761,7 @@
 				autoClose: autoCloseInput.checked,
 				openEditor: openEditorInput.checked,
 				autoOpenEditor: autoOpenEditorInput.checked,
+				defaultEditorMode: defaultEditorModeInput.value,
 				applySystemTheme: applySystemThemeInput.checked,
 				warnUnsavedPage: warnUnsavedPageInput.checked
 			}

extension/ui/content/content-ui-editor-web.js

@@ -46,6 +46,8 @@
 	const REMOVED_CONTENT_CLASS = "single-file-removed";
 	const HIGHLIGHT_HIDDEN_CLASS = "single-file-highlight-hidden";
 	const PAGE_MASK_ACTIVE_CLASS = "page-mask-active";
+	const CUT_HOVER_CLASS = "single-file-hover";
+	const CUT_CONTAINER_HOVER_CLASS = "single-file-container-hover";
 	const NOTE_INITIAL_POSITION_X = 20;
 	const NOTE_INITIAL_POSITION_Y = 20;
 	const NOTE_INITIAL_WIDTH = 150;
@@ -808,13 +810,14 @@ table {
 }`;
 
 	let NOTES_WEB_STYLESHEET, MASK_WEB_STYLESHEET, HIGHLIGHTS_WEB_STYLESHEET;
-	let selectedNote, anchorElement, maskNoteElement, maskPageElement, highlightSelectionMode, removeHighlightMode, resizingNoteMode, movingNoteMode, highlightColor, collapseNoteTimeout, cuttingMode;
-	let removedElements = [];
+	let selectedNote, anchorElement, maskNoteElement, maskPageElement, highlightSelectionMode, removeHighlightMode, resizingNoteMode, movingNoteMode, highlightColor, collapseNoteTimeout, cuttingMode, cuttingPath, cuttingPathIndex, cuttingElementContainer;
+	let removedElements = [], removedElementIndex = 0;
 
 	window.onmessage = async event => {
 		const message = JSON.parse(event.data);
 		if (message.method == "init") {
 			await init(message.content);
+			window.parent.postMessage(JSON.stringify({ "method": "onInit" }), "*");
 		}
 		if (message.method == "addNote") {
 			addNote(message);
@@ -840,11 +843,13 @@ table {
 		}
 		if (message.method == "enableRemoveHighlights") {
 			removeHighlightMode = true;
+			document.documentElement.classList.add("single-file-remove-highlights-mode");
 		}
 		if (message.method == "disableRemoveHighlights") {
 			removeHighlightMode = false;
+			document.documentElement.classList.remove("single-file-remove-highlights-mode");
 		}
-		if (message.method == "enableEditPage") {			
+		if (message.method == "enableEditPage") {
 			document.body.contentEditable = true;
 			onUpdate(false);
 		}
@@ -859,27 +864,29 @@ table {
 		}
 		if (message.method == "enableCutPage") {
 			cuttingMode = true;
-			document.body.addEventListener("mouseover", highlightElementToCut);
-			document.body.addEventListener("mouseout", highlightElementToCut);
 		}
 		if (message.method == "disableCutPage") {
 			cuttingMode = false;
-			document.body.removeEventListener("mouseover", highlightElementToCut);
-			document.body.removeEventListener("mouseout", highlightElementToCut);
+			if (cuttingPath) {
+				unhighlightCutElement();
+				cuttingPath = null;
+			}
 		}
 		if (message.method == "undoCutPage") {
-			if (removedElements.length) {
-				removedElements.pop().classList.remove(REMOVED_CONTENT_CLASS);
-			}
+			undoCutPage();
 		}
 		if (message.method == "undoAllCutPage") {
-			while (removedElements.length) {
-				removedElements.pop().classList.remove(REMOVED_CONTENT_CLASS);
+			while (removedElementIndex) {
+				removedElements[removedElementIndex - 1].classList.remove(REMOVED_CONTENT_CLASS);
+				removedElementIndex--;
 			}
 		}
+		if (message.method == "redoCutPage") {
+			redoCutPage();
+		}
 		if (message.method == "getContent") {
 			onUpdate(true);
-			getContent(message.compressHTML);
+			getContent(message.compressHTML, message.updatedResources);
 		}
 	};
 	window.onresize = reflowNotes;
@@ -917,6 +924,9 @@ table {
 		maskPageElement = getMaskElement(PAGE_MASK_CLASS, PAGE_MASK_CONTAINER_CLASS);
 		maskNoteElement = getMaskElement(NOTE_MASK_CLASS);
 		document.documentElement.onmouseup = document.documentElement.ontouchend = onMouseUp;
+		document.documentElement.onmouseover = onMouseOver;
+		document.documentElement.onmouseout = onMouseOut;
+		document.documentElement.onkeydown = onKeyDown;
 		window.onclick = event => event.preventDefault();
 	}
 
@@ -1181,10 +1191,122 @@ table {
 			collapseNoteTimeout = null;
 		}
 		if (cuttingMode) {
-			let element = event.target;
+			validateCutElement();
+		}
+	}
+
+	function onMouseOver(event) {
+		if (cuttingMode) {
+			const target = event.target;
+			if (target.classList) {
+				cuttingPath = getEventPath(event);
+				cuttingPathIndex = 0;
+				highlightCutElement(target);
+			}
+		}
+	}
+
+	function onMouseOut() {
+		if (cuttingMode) {
+			if (cuttingPath) {
+				unhighlightCutElement();
+				cuttingPath = null;
+			}
+		}
+	}
+
+	function onKeyDown(event) {
+		if (cuttingMode) {
+			if (event.code == "Tab") {
+				if (cuttingPath) {
+					const delta = event.shiftKey ? -1 : 1;
+					let element = cuttingPath[cuttingPathIndex];
+					let nextElement = cuttingPath[cuttingPathIndex + delta];
+					if (nextElement) {
+						let pathIndex = cuttingPathIndex + delta;
+						while (
+							nextElement &&
+							(
+								(delta == 1 &&
+									element.getBoundingClientRect().width >= nextElement.getBoundingClientRect().width &&
+									element.getBoundingClientRect().height >= nextElement.getBoundingClientRect().height) ||
+								(delta == -1 &&
+									element.getBoundingClientRect().width <= nextElement.getBoundingClientRect().width &&
+									element.getBoundingClientRect().height <= nextElement.getBoundingClientRect().height))) {
+							pathIndex += delta;
+							nextElement = cuttingPath[pathIndex];
+						}
+						if (nextElement && nextElement.classList && nextElement != document.body && nextElement != document.documentElement) {
+							unhighlightCutElement();
+							cuttingPathIndex = pathIndex;
+							highlightCutElement(cuttingPath[cuttingPathIndex]);
+						}
+					}
+				}
+				event.preventDefault();
+			}
+			if (event.code == "Space") {
+				validateCutElement();
+				event.preventDefault();
+			}
+			if (event.key.toLowerCase() == "z" && event.ctrlKey) {
+				if (event.shiftKey) {
+					redoCutPage();
+				} else {
+					undoCutPage();
+				}
+				event.preventDefault();
+			}
+		}
+	}
+
+	function highlightCutElement() {
+		const element = cuttingPath[cuttingPathIndex];
+		element.classList.add(CUT_HOVER_CLASS);
+		let parentElement = element.parentElement;
+		while (parentElement && getComputedStyle(parentElement).getPropertyValue("overflow") != "hidden") {
+			parentElement = parentElement.parentElement;
+		}
+		if (parentElement) {
+			cuttingElementContainer = parentElement;
+			cuttingElementContainer.classList.add(CUT_CONTAINER_HOVER_CLASS);
+		} else {
+			cuttingElementContainer = null;
+		}
+	}
+
+	function unhighlightCutElement() {
+		if (cuttingPath) {
+			const element = cuttingPath[cuttingPathIndex];
+			element.classList.remove(CUT_HOVER_CLASS);
+			if (cuttingElementContainer) {
+				cuttingElementContainer.classList.remove(CUT_CONTAINER_HOVER_CLASS);
+			}
+		}
+	}
+
+	function undoCutPage() {
+		if (removedElementIndex) {
+			removedElements[removedElementIndex - 1].classList.remove(REMOVED_CONTENT_CLASS);
+			removedElementIndex--;
+		}
+	}
+
+	function redoCutPage() {
+		if (removedElementIndex < removedElements.length) {
+			removedElements[removedElementIndex].classList.add(REMOVED_CONTENT_CLASS);
+			removedElementIndex++;
+		}
+	}
+
+	function validateCutElement() {
+		if (cuttingPath) {
+			const element = cuttingPath[cuttingPathIndex];
 			if (document.documentElement != element && element.tagName.toLowerCase() != NOTE_TAGNAME) {
 				element.classList.add(REMOVED_CONTENT_CLASS);
-				removedElements.push(element);
+				removedElements[removedElementIndex] = element;
+				removedElementIndex++;
+				removedElements.length = removedElementIndex;
 				onUpdate(false);
 			}
 		}
@@ -1317,13 +1439,14 @@ table {
 		}
 	}
 
-	function highlightElementToCut(event) {
-		if (event.type != "mouseover" && event.type != "mouseout") return;
-		var target = event.target;
-		if ("classList" in target) {
-			target.classList.toggle("single-file-hover");
+	function getEventPath(event) {
+		const path = [];
+		let element = event.target;
+		while (element) {
+			path.push(element);
+			element = element.parentElement;
 		}
-		event.stopPropagation();
+		return path;
 	}
 
 	function formatPage(applySystemTheme) {
@@ -1337,6 +1460,7 @@ table {
 		});
 		const article = new Readability(document, { classesToPreserve }).parse();
 		removedElements = [];
+		removedElementIndex = 0;
 		document.body.innerHTML = "";
 		const domParser = new DOMParser();
 		const doc = domParser.parseFromString(article.content, "text/html");
@@ -1380,7 +1504,8 @@ table {
 		onUpdate(false);
 	}
 
-	function getContent(compressHTML) {
+	function getContent(compressHTML, updatedResources) {
+		unhighlightCutElement();
 		serializeShadowRoots(document);
 		const doc = document.cloneNode(true);
 		deserializeShadowRoots(doc);
@@ -1406,11 +1531,17 @@ table {
 			element.style.setProperty(pointerEvents, element.style.getPropertyValue("-sf-" + pointerEvents), element.style.getPropertyPriority("-sf-" + pointerEvents));
 			element.style.removeProperty("-sf-" + pointerEvents);
 		});
-		delete doc.body.contentEditable;
+		doc.body.removeAttribute("contentEditable");
 		const scriptElement = doc.createElement("script");
 		scriptElement.setAttribute(SCRIPT_TEMPLATE_SHADOW_ROOT, "");
 		scriptElement.textContent = getEmbedScript();
 		doc.body.appendChild(scriptElement);
+		const newResources = Object.keys(updatedResources).filter(url => updatedResources[url].type == "stylesheet").map(url => updatedResources[url]);
+		newResources.forEach(resource => {
+			const element = doc.createElement("style");
+			doc.body.appendChild(element);
+			element.textContent = resource.content;
+		});
 		window.parent.postMessage(JSON.stringify({ "method": "setContent", content: singlefile.lib.modules.serializer.process(doc, compressHTML) }), "*");
 	}
 
@@ -1553,6 +1684,7 @@ table {
 			const maskPageElement = getMaskElement(${JSON.stringify(PAGE_MASK_CLASS)}, ${JSON.stringify(PAGE_MASK_CONTAINER_CLASS)});
 			let selectedNote, highlightSelectionMode, removeHighlightMode, resizingNoteMode, movingNoteMode, collapseNoteTimeout, cuttingMode;
 			window.onresize = reflowNotes;
+			window.onUpdate = () => {};
 			document.documentElement.onmouseup = document.documentElement.ontouchend = onMouseUp;
 			window.addEventListener("DOMContentLoaded", () => {
 				processNode(document);

extension/ui/content/content-ui-main.js

@@ -68,8 +68,7 @@ this.singlefile.extension.ui.content.main = this.singlefile.extension.ui.content
 					}
 					maskElement.offsetWidth;
 					maskElement.style.setProperty("background-color", "black", "important");
-					maskElement.style.setProperty("opacity", .3, "important");
-					document.body.offsetWidth;
+					maskElement.style.setProperty("opacity", .3, "important");					
 				}
 			}
 		},
@@ -446,6 +445,7 @@ this.singlefile.extension.ui.content.main = this.singlefile.extension.ui.content
 		logsWindowElement.style.setProperty("min-height", "16px", "important");
 		logsWindowElement.style.setProperty("transition", "height 100ms", "important");
 		logsWindowElement.style.setProperty("will-change", "height", "important");
+		logsWindowElement.style.setProperty("inset-block", "auto");
 	}
 
 	function updateLog(id, textContent, textStatus, options) {

extension/ui/pages/editor-frame-web.css

@@ -54,7 +54,16 @@ single-file-note {
 }
 
 .single-file-hover {
-    outline: 1px solid red !important;
+    outline: 4px dotted red !important;
+    transition: outline-width 125ms;
+}
+
+.single-file-hover, .single-file-remove-highlights-mode .single-file-highlight:hover {
+    cursor: crosshair !important;
+}
+
+.single-file-container-hover {
+    overflow: visible !important;
 }
 
 .single-file-removed {

extension/ui/pages/editor.css

@@ -50,8 +50,8 @@ img[type=button].remove-highlight-disabled {
 
 img[type=button].edit-disabled:hover,
 img[type=button].cut-disabled:hover,
-img[type=button].highlight-disabled:hover,
-img[type=button].remove-highlight-disabled:hover {
+.toolbar:not(.cut-mode):not(.remove-highlight-mode) img[type=button].highlight-disabled:hover,
+.toolbar:not(.cut-mode) img[type=button].remove-highlight-disabled:hover {
     filter: brightness(0.875);
 }
 

extension/ui/pages/editor.html

@@ -19,13 +19,13 @@
 			<div class="separator"></div>
 		</div>
 		<div class="buttons">
-			<img type="button" class="highlight-button highlight-yellow-button highlight-disabled"
+			<img type="button" class="highlight-button highlight-yellow-button highlight-disabled" data-color="yellow"
 				src="/extension/ui/resources/button_highlighter_yellow.png">
-			<img type="button" class="highlight-button highlight-pink-button highlight-disabled"
+			<img type="button" class="highlight-button highlight-pink-button highlight-disabled" data-color="pink"
 				src="/extension/ui/resources/button_highlighter_pink.png">
-			<img type="button" class="highlight-button highlight-blue-button highlight-disabled"
+			<img type="button" class="highlight-button highlight-blue-button highlight-disabled" data-color="blue"
 				src="/extension/ui/resources/button_highlighter_blue.png">
-			<img type="button" class="highlight-button highlight-green-button highlight-disabled"
+			<img type="button" class="highlight-button highlight-green-button highlight-disabled" data-color="green"
 				src="/extension/ui/resources/button_highlighter_green.png">
 			<img type="button" class="toggle-highlights-button"
 				src="/extension/ui/resources/button_highlighter_visible.png">
@@ -44,6 +44,7 @@
 			<img type="button" class="cut-page-button cut-disabled" src="/extension/ui/resources/button_cut.png">
 			<img type="button" class="undo-cut-page-button" src="/extension/ui/resources/button_undo_cut.png">
 			<img type="button" class="undo-all-cut-page-button" src="/extension/ui/resources/button_undo_all_cut.png">
+			<img type="button" class="redo-cut-page-button" src="/extension/ui/resources/button_redo_cut.png">
 			<div class="separator"></div>
 		</div>
 		<div class="buttons">
@@ -61,6 +62,8 @@
 	<script src="/extension/core/content/content-download.js"></script>
 	<script src="/extension/ui/index.js"></script>
 	<script src="/extension/ui/bg/ui-editor.js"></script>
+	<script src="/common/index.js"></script>
+	<script src="/common/ui/content/content-infobar.js"></script>
 </body>
 
 </html>

extension/ui/pages/help.css

@@ -1,5 +1,6 @@
 body {
     background-color: #eee;
+    font-size: 12pt;
 }
 
 body>div {
@@ -95,8 +96,7 @@ ol>li>ul>li>ul {
     white-space: nowrap;
 }
 
-h2,
-h4 {
+h2, h4 {
     margin-bottom: 0px;
 }
 
@@ -118,8 +118,7 @@ code {
     font-size: 1.1em;
 }
 
-kbd,
-.key {
+kbd, .key {
     display: inline;
     display: inline-block;
     min-width: 1em;
@@ -137,17 +136,11 @@ kbd,
     user-select: none;
 }
 
-kbd[title],
-.key[title] {
+kbd[title], .key[title] {
     cursor: help;
 }
 
-kbd,
-kbd.dark,
-.dark-keys kbd,
-.key,
-.key.dark,
-.dark-keys .key {
+kbd, kbd.dark, .dark-keys kbd, .key, .key.dark, .dark-keys .key {
     background: rgb(80, 80, 80);
     background: -moz-linear-gradient(top, rgb(60, 60, 60), rgb(80, 80, 80));
     background: -webkit-gradient(linear, left top, left bottom, from(rgb(60, 60, 60)), to(rgb(80, 80, 80)));
@@ -158,10 +151,7 @@ kbd.dark,
     box-shadow: inset 0 0 1px rgb(150, 150, 150), inset 0 -.05em .4em rgb(80, 80, 80), 0 .1em 0 rgb(30, 30, 30), 0 .1em .1em rgba(0, 0, 0, .3);
 }
 
-kbd.light,
-.light-keys kbd,
-.key.light,
-.light-keys .key {
+kbd.light, .light-keys kbd, .key.light, .light-keys .key {
     background: rgb(250, 250, 250);
     background: -moz-linear-gradient(top, rgb(210, 210, 210), rgb(255, 255, 255));
     background: -webkit-gradient(linear, left top, left bottom, from(rgb(210, 210, 210)), to(rgb(255, 255, 255)));
@@ -177,15 +167,12 @@ kbd.light,
         background-color: white;
         margin: 0px;
     }
-
     body>div {
         border-width: 0px;
     }
-
     ol {
         padding-left: 20px;
     }
-
     ul {
         padding-left: 10px;
     }
@@ -195,23 +182,18 @@ kbd.light,
     body {
         background-color: #373737;
     }
-
     @media (max-width:800px) {
         body {
             background-color: #202023;
         }
     }
-
     body>div {
         background-color: #202023;
         border-color: rgb(81, 81, 81);
     }
-
-    body>div,
-    a {
+    body>div, a {
         color: #fdfdfd;
     }
-
     .option {
         color: #afafaf;
     }

extension/ui/pages/help.html

@@ -14,9 +14,10 @@
 			<h2>SingleFile</h2>
 			<h4>Save a complete page into a single HTML file</h4>
 		</div> <span id="index"> <a href="#getting-started">Getting started</a> - <a href="#general-notes">Additional
-				notes</a> - <a href="#options">Options description</a> - <a href="#notes">Technical notes</a> - <a
-				href="#template-variables">Template variables</a> - <a href="#known-issues">Known issues</a> - <a
-				href="#unknown-issues">Troubleshooting unknown issues</a> - <a href="#contributors">Contributors</a>
+				notes</a> - <a href="#options">Options description</a> - <a href="#annotation-editor">Anootation
+				editor</a> - <a href="#notes">Technical notes</a> - <a href="#template-variables">Template variables</a>
+			- <a href="#known-issues">Known issues</a> - <a href="#unknown-issues">Troubleshooting unknown issues</a> -
+			<a href="#contributors">Contributors</a>
 		</span>
 		<hr>
 		<ol>
@@ -52,8 +53,9 @@
 							<li>or all the tabs.</li>
 						</ul>
 					</li>
-					<li>You can highlight text, add notes, and remove content before saving the page by selecting
-						"Annotate and save the page..." in the context menu</li>
+					<li>You can highlight text, add notes, format and remove content before saving the page with the
+						<a href="#annotation-editor">Annotation editor</a> by selecting "Annotate and save the page..."
+						in the context menu</li>
 					<li>With auto-save active, pages are automatically saved every time after being loaded (or before
 						being unloaded if not). </li>
 					<li>Right-click on the SingleFile button and select "Options" to open the options page.</li>
@@ -343,6 +345,20 @@
 				</ul>
 				<p>Annotation editor</p>
 				<ul>
+					<li data-options-label="defaultEditorModeLabel"> <span class="option">Option: default mode</span>
+						<p>Select the default mode when opening the annotation editor. The available choices are:
+						<ul>
+							<li><code>normal</code>: default value</li>
+							<li><code>edit the page</code>: enable the button <img
+									src="../resources/button_note_edit.png" class="icon"></li>
+							<li><code>format the page</code>: enable the button <img
+									src="../resources/button_note_format.png" class="icon"> if the page can be
+								formatted</li>
+							<li><code>remove elements</code>: enable the button <img src="../resources/button_cut.png"
+									class="icon"></li>
+						</ul>
+						</p>
+					</li>
 					<li data-options-label="applySystemThemeLabel"> <span class="option">Option: apply the system theme
 							when formatting a page in the annotation editor</span>
 						<p>Uncheck this option if you do not want to apply the theme of the operating system or the
@@ -488,6 +504,49 @@
 					</li>
 				</ul>
 			</li>
+			<li><a id="annotation-editor">Annotation editor</a>
+				<p>The annotation editor can be opened by selecting "Annotate and save the page" in the context menu or
+					by enabling the option "Annotation editor &gt; edit page before saving". It allows you to:
+				<ul>
+					<li>add notes by clicking one of these buttons: <img src="../resources/button_note_yellow.png"
+							class="icon">
+						<img src="../resources/button_note_pink.png" class="icon"> <img
+							src="../resources/button_note_blue.png" class="icon">
+						<img src="../resources/button_note_green.png" class="icon"></li>
+					<li>hide or show notes by clicking the button <img src="../resources/button_note_visible.png"
+							class="icon"></li>
+					<li>highlight text by clicking one of these buttons: <img
+							src="../resources/button_highlighter_yellow.png" class="icon"> <img
+							src="../resources/button_highlighter_pink.png" class="icon"> <img
+							src="../resources/button_highlighter_blue.png" class="icon"> <img
+							src="../resources/button_highlighter_green.png" class="icon"></li>
+					<li>hide or show highligted text by clicking the button <img
+							src="../resources/button_highlighter_visible.png" class="icon"></li>
+					<li>remove text highlighting by clicking the button <img
+							src="../resources/button_highlighter_delete.png" class="icon"></li>
+					<li>edit the page by clicking the button <img src="../resources/button_note_edit.png" class="icon">
+					</li>
+					<li>format the page to improve readability (when possible) by clicking the button <img
+							src="../resources/button_note_format.png" class="icon">
+					</li>
+					<li>remove contents by clicking the button <img src="../resources/button_cut.png" class="icon">,
+						undo removes by clicking the button <img src="../resources/button_undo_cut.png" class="icon"> or
+						the button <img src="../resources/button_undo_all_cut.png" class="icon">, and redo removes by
+						clicking the button <img src="../resources/button_redo_cut.png" class="icon">.
+						<br>
+						You can also use the following keyboard shortcuts when removing contents:
+						<ul>
+							<li><code>Tab</code>: expand the selection</li>
+							<li><code>Shift-Tab</code>: reduce the selection</li>
+							<li><code>Space</code>: remove the selected element</li>
+							<li><code>Ctrl-Z</code>: undo the last removal</li>
+							<li><code>Ctrl-Shift-Z</code>: redo the last removal</li>
+						</ul>
+					</li>
+					<li>save the page by clicking the button <img src="../resources/button_download.png" class="icon">.
+				</ul>
+				</p>
+			</li>
 			<li><a id="template-variables">Template variables</a>
 				<p>The template variables are used to customize the infobar content or the file name of a saved page.
 					They help to insert dynamic values like the save date or the page title.</p>
@@ -648,4 +707,4 @@
 	</div>
 </body>
 
-</html>
+</html>

extension/ui/pages/options.html

@@ -195,6 +195,15 @@
 		</details>
 		<details>
 			<summary id="editorLabel"></summary>
+			<div class="option">
+				<label for="defaultEditorModeInput" id="defaultEditorModeLabel"></label>
+				<select id="defaultEditorModeInput">
+					<option id="defaultEditorModeNormalLabel" value="normal"></option>
+					<option id="defaultEditorModeEditLabel" value="edit"></option>
+					<option id="defaultEditorModeFormatLabel" value="format"></option>
+					<option id="defaultEditorModeCutLabel" value="cut"></option>
+				</select>
+			</div>
 			<div class="option">
 				<label for="applySystemThemeInput" id="applySystemThemeLabel"></label>
 				<input type="checkbox" id="applySystemThemeInput">

lib/single-file/processors/frame-tree/content/content-frame-tree.js

@@ -73,7 +73,9 @@ this.singlefile.lib.processors.frameTree.content.frames = this.singlefile.lib.pr
 			event.stopPropagation();
 			const message = JSON.parse(event.data.substring(MESSAGE_PREFIX.length));
 			if (message.method == INIT_REQUEST_MESSAGE) {
-				sendMessage(event.source, { method: ACK_INIT_REQUEST_MESSAGE, windowId: message.windowId, sessionId: message.sessionId });
+				if (event.source) {
+					sendMessage(event.source, { method: ACK_INIT_REQUEST_MESSAGE, windowId: message.windowId, sessionId: message.sessionId });
+				}
 				if (!TOP_WINDOW) {
 					window.stop();
 					if (message.options.loadDeferredImages && singlefile.lib.processors.lazy.content.loader) {

lib/single-file/processors/hooks/content/content-hooks-frames-web.js

@@ -21,7 +21,7 @@
  *   Source.
  */
 
-/* global window */
+/* global window, Event */
 
 (() => {
 
@@ -145,21 +145,26 @@
 			const transformPriority = document.documentElement.style.getPropertyPriority("transform");
 			const transformOrigin = document.documentElement.style.getPropertyValue("transform-origin");
 			const transformOriginPriority = document.documentElement.style.getPropertyPriority("transform-origin");
+			const minHeight = document.documentElement.style.getPropertyValue("min-height");
+			const minHeightPriority = document.documentElement.style.getPropertyPriority("min-height");
 			document.documentElement.style.setProperty("transform-origin", (zoomFactorX < 1 ? "50%" : "0") + " " + (zoomFactorY < 1 ? "50%" : "0") + " 0", "important");
 			document.documentElement.style.setProperty("transform", "scale3d(" + zoomFactor + ", " + zoomFactor + ", 1)", "important");
+			document.documentElement.style.setProperty("min-height", (100 / zoomFactor) + "vh", "important");
 			dispatchEvent.call(window, new UIEvent("resize"));
-			dispatchEvent.call(window, new UIEvent("scroll"));
+			dispatchEvent.call(window, new Event("scroll"));
 			if (keepZoomLevel) {
 				document.documentElement.style.setProperty("-sf-transform", transform, transformPriority);
 				document.documentElement.style.setProperty("-sf-transform-origin", transformOrigin, transformOriginPriority);
+				document.documentElement.style.setProperty("-sf-min-height", minHeight, minHeightPriority);
 			} else {
 				document.documentElement.style.setProperty("transform", transform, transformPriority);
 				document.documentElement.style.setProperty("transform-origin", transformOrigin, transformOriginPriority);
+				document.documentElement.style.setProperty("min-height", minHeight, minHeightPriority);
 			}
 		}
 		if (!keepZoomLevel) {
 			dispatchEvent.call(window, new UIEvent("resize"));
-			dispatchEvent.call(window, new UIEvent("scroll"));
+			dispatchEvent.call(window, new Event("scroll"));
 			const docBoundingRect = scrollingElement.getBoundingClientRect();
 			[...observers].forEach(([intersectionObserver, observer]) => {
 				const rootBoundingRect = observer.options && observer.options.root && observer.options.root.getBoundingClientRect();
@@ -185,10 +190,14 @@
 		const transformPriority = document.documentElement.style.getPropertyPriority("-sf-transform");
 		const transformOrigin = document.documentElement.style.getPropertyValue("-sf-transform-origin");
 		const transformOriginPriority = document.documentElement.style.getPropertyPriority("-sf-transform-origin");
+		const minHeight = document.documentElement.style.getPropertyValue("-sf-min-height");
+		const minHeightPriority = document.documentElement.style.getPropertyPriority("-sf-min-height");
 		document.documentElement.style.setProperty("transform", transform, transformPriority);
 		document.documentElement.style.setProperty("transform-origin", transformOrigin, transformOriginPriority);
+		document.documentElement.style.setProperty("min-height", minHeight, minHeightPriority);
 		document.documentElement.style.removeProperty("-sf-transform");
 		document.documentElement.style.removeProperty("-sf-transform-origin");
+		document.documentElement.style.removeProperty("-sf-min-height");
 	});
 
 	function loadDeferredImagesEnd(keepZoomLevel) {
@@ -218,7 +227,7 @@
 		}
 		if (!keepZoomLevel) {
 			dispatchEvent.call(window, new UIEvent("resize"));
-			dispatchEvent.call(window, new UIEvent("scroll"));
+			dispatchEvent.call(window, new Event("scroll"));
 		}
 	}
 

lib/single-file/processors/lazy/content/content-lazy-loader.js

@@ -39,10 +39,12 @@ this.singlefile.lib.processors.lazy.content.loader = this.singlefile.lib.process
 
 	return {
 		process: async options => {
-			const maxScrollY = Math.max(document.documentElement.scrollHeight - (document.documentElement.clientHeight * 1.5), 0);
-			const maxScrollX = Math.max(document.documentElement.scrollWidth - (document.documentElement.clientWidth * 1.5), 0);
-			if (scrollY <= maxScrollY && scrollX <= maxScrollX) {
-				return process(options);
+			if (document.documentElement) {
+				const maxScrollY = Math.max(document.documentElement.scrollHeight - (document.documentElement.clientHeight * 1.5), 0);
+				const maxScrollX = Math.max(document.documentElement.scrollWidth - (document.documentElement.clientWidth * 1.5), 0);
+				if (scrollY <= maxScrollY && scrollX <= maxScrollX) {
+					return process(options);
+				}
 			}
 		},
 		resetZoomLevel: () => {
@@ -64,6 +66,7 @@ this.singlefile.lib.processors.lazy.content.loader = this.singlefile.lib.process
 					const updated = mutations.filter(mutation => {
 						if (mutation.attributeName == "src") {
 							mutation.target.setAttribute(singlefile.lib.helper.LAZY_SRC_ATTRIBUTE_NAME, mutation.target.src);
+							mutation.target.addEventListener("load", onResourceLoad);
 						}
 						if (mutation.attributeName == "src" || mutation.attributeName == "srcset" || mutation.target.tagName == "SOURCE") {
 							return mutation.target.className != SINGLE_FILE_UI_ELEMENT_CLASS;
@@ -92,6 +95,12 @@ this.singlefile.lib.processors.lazy.content.loader = this.singlefile.lib.process
 				frames.loadDeferredImagesStart(options);
 			}
 
+			function onResourceLoad(event) {
+				const element = event.target;
+				element.removeAttribute(singlefile.lib.helper.LAZY_SRC_ATTRIBUTE_NAME);
+				element.removeEventListener("load", onResourceLoad);
+			}
+
 			async function onImageLoadEvent(event) {
 				loadingImages = true;
 				maxTimeoutId = await deferForceLazyLoadEnd(timeoutId, idleTimeoutId, maxTimeoutId, observer, options, cleanupAndResolve);

lib/single-file/single-file-core.js

@@ -393,6 +393,7 @@ this.singlefile.lib.core = this.singlefile.lib.core || (() => {
 	const SHADOW_DELEGATE_FOCUS_ATTRIBUTE_NAME = "delegatesfocus";
 	const SCRIPT_TEMPLATE_SHADOW_ROOT = "data-template-shadow-root";
 	const UTF8_CHARSET = "utf-8";
+	const SINGLE_FILE_UI_ELEMENT_CLASS = "single-file-ui-element";
 
 	class Processor {
 		constructor(options, batchRequest) {
@@ -692,19 +693,19 @@ this.singlefile.lib.core = this.singlefile.lib.core || (() => {
 		}
 
 		removeDiscardedResources() {
-			this.doc.querySelectorAll("singlefile-infobar, singlefile-mask, singlefile-logs-window").forEach(element => element.remove());
+			this.doc.querySelectorAll("." + SINGLE_FILE_UI_ELEMENT_CLASS).forEach(element => element.remove());
 			this.doc.querySelectorAll("meta[http-equiv=refresh], meta[disabled-http-equiv], meta[http-equiv=\"content-security-policy\"]").forEach(element => element.remove());
 			const objectElements = this.doc.querySelectorAll("applet, object[data]:not([type=\"image/svg+xml\"]):not([type=\"image/svg-xml\"]):not([type=\"text/html\"]), embed[src]:not([src*=\".svg\"]):not([src*=\".pdf\"])");
 			this.stats.set("discarded", "objects", objectElements.length);
 			this.stats.set("processed", "objects", objectElements.length);
 			objectElements.forEach(element => element.remove());
-			const replacedAttributeValue = this.doc.querySelectorAll("link[rel~=preconnect], link[rel~=prerender], link[rel~=dns-prefetch], link[rel~=preload], link[rel~=prefetch]");
+			const replacedAttributeValue = this.doc.querySelectorAll("link[rel~=preconnect], link[rel~=prerender], link[rel~=dns-prefetch], link[rel~=preload], link[rel~=manifest], link[rel~=prefetch]");
 			replacedAttributeValue.forEach(element => {
 				let regExp;
 				if (this.options.removeScripts) {
-					regExp = /(preconnect|prerender|dns-prefetch|preload|prefetch)/g;
+					regExp = /(preconnect|prerender|dns-prefetch|preload|prefetch|manifest)/g;
 				} else {
-					regExp = /(preconnect|prerender|dns-prefetch|prefetch)/g;
+					regExp = /(preconnect|prerender|dns-prefetch|prefetch|manifest)/g;
 				}
 				const relValue = element.getAttribute("rel").replace(regExp, "").trim();
 				if (relValue.length) {
@@ -714,7 +715,7 @@ this.singlefile.lib.core = this.singlefile.lib.core || (() => {
 				}
 			});
 			this.doc.querySelectorAll("link[rel*=stylesheet][rel*=alternate][title],link[rel*=stylesheet]:not([href]),link[rel*=stylesheet][href=\"\"]").forEach(element => element.remove());
-			if (this.options.compressHTML) {
+			if (this.options.removeHiddenElements) {
 				this.doc.querySelectorAll("input[type=hidden]").forEach(element => element.remove());
 			}
 			if (!this.options.saveFavicon) {
@@ -836,7 +837,15 @@ this.singlefile.lib.core = this.singlefile.lib.core || (() => {
 				const styleElement = this.doc.createElement("style");
 				styleElement.textContent = ".sf-hidden{display:none!important;}";
 				this.doc.head.appendChild(styleElement);
-				hiddenElements.forEach(element => element.classList.add("sf-hidden"));
+				hiddenElements.forEach(element => {
+					if (element.style.getPropertyValue("display") != "none") {
+						if (element.style.getPropertyPriority("display") == "important") {
+							element.style.setProperty("display", "none", "important");
+						} else {
+							element.classList.add("sf-hidden");
+						}
+					}
+				});
 			}
 			removedElements.forEach(element => element.remove());
 		}
@@ -1209,14 +1218,14 @@ this.singlefile.lib.core = this.singlefile.lib.core || (() => {
 
 		async processFrames() {
 			if (this.options.frames) {
-				const frameElements = Array.from(this.doc.querySelectorAll("iframe:not([" + util.HIDDEN_FRAME_ATTRIBUTE_NAME + "]), frame, object[type=\"text/html\"][data]"));
+				const frameElements = Array.from(this.doc.querySelectorAll("iframe, frame, object[type=\"text/html\"][data]"));
 				await Promise.all(frameElements.map(async frameElement => {
 					const frameWindowId = frameElement.getAttribute(util.WIN_ID_ATTRIBUTE_NAME);
 					if (frameWindowId) {
 						const frameData = this.options.frames.find(frame => frame.windowId == frameWindowId);
 						if (frameData) {
 							this.options.frames = this.options.frames.filter(frame => frame.windowId != frameWindowId);
-							if (frameData.runner) {
+							if (frameData.runner && frameElement.getAttribute(util.HIDDEN_FRAME_ATTRIBUTE_NAME) != "") {
 								this.stats.add("processed", "frames", 1);
 								await frameData.runner.run();
 								const pageData = await frameData.runner.getPageData();
@@ -1238,6 +1247,7 @@ this.singlefile.lib.core = this.singlefile.lib.core || (() => {
 								}
 								this.stats.addAll(pageData);
 							} else {
+								frameElement.removeAttribute(util.WIN_ID_ATTRIBUTE_NAME);
 								this.stats.add("discarded", "frames", 1);
 							}
 						}
@@ -1394,7 +1404,6 @@ this.singlefile.lib.core = this.singlefile.lib.core || (() => {
 			const publisherElement = this.doc.querySelector("meta[name=publisher]");
 			const headingElement = this.doc.querySelector("h1");
 			this.options.title = titleElement ? titleElement.textContent.trim() : "";
-			this.options.infobarContent = await ProcessorHelper.evalTemplate(this.options.infobarTemplate, this.options, null, true);
 			this.options.info = {
 				description: descriptionElement && descriptionElement.content ? descriptionElement.content.trim() : "",
 				lang: this.doc.documentElement.lang,
@@ -1403,6 +1412,7 @@ this.singlefile.lib.core = this.singlefile.lib.core || (() => {
 				publisher: publisherElement && publisherElement.content ? publisherElement.content.trim() : "",
 				heading: headingElement && headingElement.textContent ? headingElement.textContent.trim() : ""
 			};
+			this.options.infobarContent = await ProcessorHelper.evalTemplate(this.options.infobarTemplate, this.options, null, true);
 		}
 	}
 
@@ -2140,4 +2150,4 @@ this.singlefile.lib.core = this.singlefile.lib.core || (() => {
 
 	return { getClass };
 
-})();
+})();

lib/single-file/single-file-helper.js

@@ -156,7 +156,7 @@ this.singlefile.lib.helper = this.singlefile.lib.helper || (() => {
 			if (!options.autoSaveExternalSave && (options.removeHiddenElements || options.removeUnusedFonts || options.compressHTML)) {
 				computedStyle = win.getComputedStyle(element);
 				if (options.removeHiddenElements) {
-					elementKept = (ascendantHidden || element.closest("html > head")) && KEPT_TAG_NAMES.includes(element.tagName);
+					elementKept = ((ascendantHidden || element.closest("html > head")) && KEPT_TAG_NAMES.includes(element.tagName)) || element.closest("details");
 					if (!elementKept) {
 						elementHidden = ascendantHidden || testHiddenElement(element, computedStyle);
 						if (elementHidden) {
@@ -198,8 +198,10 @@ this.singlefile.lib.helper = this.singlefile.lib.helper || (() => {
 			getElementsInfo(win, doc, element, options, data, elementHidden);
 			if (!options.autoSaveExternalSave && options.removeHiddenElements && ascendantHidden) {
 				if (elementKept || element.getAttribute(KEPT_CONTENT_ATTRIBUTE_NAME) == "") {
-					element.parentElement.setAttribute(KEPT_CONTENT_ATTRIBUTE_NAME, "");
-					data.markedElements.push(element);
+					if (element.parentElement) {
+						element.parentElement.setAttribute(KEPT_CONTENT_ATTRIBUTE_NAME, "");
+						data.markedElements.push(element.parentElement);
+					}
 				} else if (elementHidden) {
 					element.setAttribute(REMOVED_CONTENT_ATTRIBUTE_NAME, "");
 					data.markedElements.push(element);

manifest.json

@@ -8,7 +8,7 @@
 		"64": "extension/ui/resources/icon_64.png",
 		"128": "extension/ui/resources/icon_128.png"
 	},
-	"version": "1.18.1",
+	"version": "1.18.12",
 	"description": "__MSG_extensionDescription__",
 	"content_scripts": [
 		{

package.json

@@ -1,21 +1,20 @@
 {
 	"name": "single-file",
-	"version": "0.1.1",
+	"version": "0.1.16",
 	"description": "SingleFile",
 	"author": "Gildas Lormeau",
 	"license": "AGPL-3.0-or-later",
-	"main": "cli/singlefile-cli-api.js",
+	"main": "cli/single-file-cli-api.js",
 	"bin": {
 		"single-file": "./cli/single-file"
 	},
 	"dependencies": {
 		"file-url": "^3.0.0",
-		"iconv-lite": "^0.5.2",
-		"jsdom": "^16.3.0",
-		"puppeteer-core": "^3.0.4",
-		"request-promise-native": "1.0.8",
+		"iconv-lite": "^0.6.2",
+		"jsdom": "^16.4.0",
+		"puppeteer-core": "^5.3.0",
 		"selenium-webdriver": "4.0.0-alpha.7",
 		"strong-data-uri": "^1.0.6",
-		"yargs": "^15.4.1"
+		"yargs": "^16.0.3"
 	}
-}
+}