added singlefile_companion (experimental)

companion/lib/messaging.js

@@ -0,0 +1,130 @@
+// https://github.com/andy-portmen/native-client/blob/master/messaging.js
+
+// chrome-native-messaging module
+//
+// Defines three Transform streams:
+//
+// - Input - transform native messages to JavaScript objects
+// - Output - transform JavaScript objects to native messages
+// - Transform - transform message objects to reply objects
+// - Debug - transform JavaScript objects to lines of JSON (for debugging, obviously)
+
+const stream = require('stream');
+const util = require('util');
+
+function Input() {
+  stream.Transform.call(this);
+
+  // Transform bytes...
+  this._writableState.objectMode = false;
+  // ...into objects.
+  this._readableState.objectMode = true;
+
+  // Unparsed data.
+  this.buf = Buffer.alloc(0);
+  // Parsed length.
+  this.len = null;
+}
+
+util.inherits(Input, stream.Transform);
+
+Input.prototype._transform = function(chunk, encoding, done) {
+  // Save this chunk.
+  this.buf = Buffer.concat([this.buf, chunk]);
+
+  const self = this;
+
+  function parseBuf() {
+    // Do we have a length yet?
+    if (typeof self.len !== 'number') {
+      // Nope. Do we have enough bytes for the length?
+      if (self.buf.length >= 4) {
+        // Yep. Parse the bytes.
+        self.len = self.buf.readUInt32LE(0);
+        // Remove the length bytes from the buffer.
+        self.buf = self.buf.slice(4);
+      }
+    }
+
+    // Do we have a length yet? (We may have just parsed it.)
+    if (typeof self.len === 'number') {
+      // Yep. Do we have enough bytes for the message?
+      if (self.buf.length >= self.len) {
+        // Yep. Slice off the bytes we need.
+        const message = self.buf.slice(0, self.len);
+        // Remove the bytes for the message from the buffer.
+        self.buf = self.buf.slice(self.len);
+        // Clear the length so we know we need to parse it again.
+        self.len = null;
+        // Parse the message bytes.
+        const obj = JSON.parse(message.toString());
+        // Enqueue it for reading.
+        self.push(obj);
+        // We could have more messages in the buffer so check again.
+        parseBuf();
+      }
+    }
+  }
+
+  // Check for a parsable buffer (both length and message).
+  parseBuf();
+
+  // We're done.
+  done();
+};
+
+function Output() {
+  stream.Transform.call(this);
+
+  this._writableState.objectMode = true;
+  this._readableState.objectMode = false;
+}
+
+util.inherits(Output, stream.Transform);
+
+Output.prototype._transform = function(chunk, encoding, done) {
+  const len = Buffer.alloc(4);
+  const buf = Buffer.from(JSON.stringify(chunk), 'utf8');
+
+  len.writeUInt32LE(buf.length, 0);
+
+  this.push(len);
+  this.push(buf);
+
+  done();
+};
+
+function Transform(handler) {
+  stream.Transform.call(this);
+
+  this._writableState.objectMode = true;
+  this._readableState.objectMode = true;
+
+  this.handler = handler;
+}
+
+util.inherits(Transform, stream.Transform);
+
+Transform.prototype._transform = function(msg, encoding, done) {
+  this.handler(msg, this.push.bind(this), done);
+};
+
+function Debug() {
+  stream.Transform.call(this);
+
+  this._writableState.objectMode = true;
+  this._readableState.objectMode = false;
+}
+
+util.inherits(Debug, stream.Transform);
+
+Debug.prototype._transform = function(chunk, encoding, done) {
+  this.push(JSON.stringify(chunk) + '\n');
+
+  done();
+};
+
+exports.Input = Input;
+exports.Output = Output;
+exports.Transform = Transform;
+exports.Debug = Debug;

companion/options.json

@@ -0,0 +1,4 @@
+{
+    "browserHeadless": true,
+    "browserDebug": false
+}

companion/singlefile_companion.js

@@ -0,0 +1,80 @@
+#!/usr/local/bin/node
+
+/*
+ * Copyright 2010-2020 Gildas Lormeau
+ * contact : gildas.lormeau <at> gmail.com
+ * 
+ * This file is part of SingleFile.
+ *
+ *   The code in this file is free software: you can redistribute it and/or 
+ *   modify it under the terms of the GNU Affero General Public License 
+ *   (GNU AGPL) as published by the Free Software Foundation, either version 3
+ *   of the License, or (at your option) any later version.
+ * 
+ *   The code in this file is distributed in the hope that it will be useful, 
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
+ *   General Public License for more details.
+ *
+ *   As additional permission under GNU AGPL version 3 section 7, you may 
+ *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
+ *   AGPL normally required by section 4, provided you include this license 
+ *   notice and a URL through which recipients can access the Corresponding 
+ *   Source.
+ */
+
+/* global require, process */
+
+const fs = require("fs");
+const nativeMessage = require("./lib/messaging.js");
+const backend = require("./../cli/back-ends/puppeteer.js");
+
+process.stdin
+	.pipe(new nativeMessage.Input())
+	.pipe(new nativeMessage.Transform(function (msg, push, done) {
+		capturePage(msg);
+		push(JSON.stringify(msg));
+		done();
+	}))
+	.pipe(new nativeMessage.Output())
+	.pipe(process.stdout);
+
+async function capturePage(options) {
+	await backend.initialize(require("./options.json"));
+	try {
+		const pageData = await backend.getPageData(options);
+		if (options.output) {
+			fs.writeFileSync(getFilename(options.output), pageData.content);
+		} else if (options.filenameTemplate && pageData.filename) {
+			pageData.filename = "../" + pageData.filename;
+			fs.writeFileSync(getFilename(pageData.filename), pageData.content);
+		}
+		return pageData;
+	} catch (error) {
+		if (options.errorFile) {
+			const message = "URL: " + options.url + "\nStack: " + error.stack + "\n";
+			fs.writeFileSync(options.errorFile, message, { flag: "a" });
+		}
+	} finally {
+		await backend.closeBrowser();
+		process.exit(0);
+	}
+}
+
+function getFilename(filename, index = 1) {
+	let newFilename = filename;
+	if (index > 1) {
+		const regExpMatchExtension = /(\.[^.]+)$/;
+		const matchExtension = newFilename.match(regExpMatchExtension);
+		if (matchExtension && matchExtension[1]) {
+			newFilename = newFilename.replace(regExpMatchExtension, " - " + index + matchExtension[1]);
+		} else {
+			newFilename += " - " + index;
+		}
+	}
+	if (fs.existsSync(newFilename)) {
+		return getFilename(filename, index + 1);
+	} else {
+		return newFilename;
+	}
+}

companion/win/singlefile_companion.bat

@@ -0,0 +1,2 @@
+@echo off
+node ../singlefile_companion.js

companion/win/singlefile_companion.json

@@ -0,0 +1,9 @@
+{
+    "name": "singlefile_companion",
+    "description": "SingleFile Companion",
+    "path": "singlefile_companion.bat",
+    "type": "stdio",
+    "allowed_extensions": [
+        "{531906d3-e22f-4a6c-a102-8057b88a1a63}"
+    ]
+}

extension/core/bg/autosave.js

@@ -118,19 +118,23 @@ singlefile.extension.core.bg.autosave = (() => {
 		options.tabIndex = tab.index;
 		let pageData;
 		try {
-			pageData = await singlefile.extension.getPageData(options, null, null, { fetch });
-			if (options.includeInfobar) {
-				await singlefile.common.ui.content.infobar.includeScript(pageData);
-			}
-			const blob = new Blob([pageData.content], { type: "text/html" });
-			if (options.saveToGDrive) {
-				await singlefile.extension.core.bg.downloads.uploadPage(message.taskId, pageData.filename, blob, options, {});
+			if (options.externalSave) {
+				await singlefile.extension.core.bg.companion.save(options);
 			} else {
-				pageData.url = URL.createObjectURL(blob);
-				await singlefile.extension.core.bg.downloads.downloadPage(pageData, options);
-			}
-			if (pageData.hash) {
-				await woleet.anchor(pageData.hash);
+				pageData = await singlefile.extension.getPageData(options, null, null, { fetch });
+				if (options.includeInfobar) {
+					await singlefile.common.ui.content.infobar.includeScript(pageData);
+				}
+				const blob = new Blob([pageData.content], { type: "text/html" });
+				if (options.saveToGDrive) {
+					await singlefile.extension.core.bg.downloads.uploadPage(message.taskId, pageData.filename, blob, options, {});
+				} else {
+					pageData.url = URL.createObjectURL(blob);
+					await singlefile.extension.core.bg.downloads.downloadPage(pageData, options);
+				}
+				if (pageData.hash) {
+					await woleet.anchor(pageData.hash);
+				}
 			}
 		} finally {
 			singlefile.extension.core.bg.business.onSaveEnd(message.taskId);

extension/core/bg/companion.js

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2010-2020 Gildas Lormeau
+ * contact : gildas.lormeau <at> gmail.com
+ * 
+ * This file is part of SingleFile.
+ *
+ *   The code in this file is free software: you can redistribute it and/or 
+ *   modify it under the terms of the GNU Affero General Public License 
+ *   (GNU AGPL) as published by the Free Software Foundation, either version 3
+ *   of the License, or (at your option) any later version.
+ * 
+ *   The code in this file is distributed in the hope that it will be useful, 
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
+ *   General Public License for more details.
+ *
+ *   As additional permission under GNU AGPL version 3 section 7, you may 
+ *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
+ *   AGPL normally required by section 4, provided you include this license 
+ *   notice and a URL through which recipients can access the Corresponding 
+ *   Source.
+ */
+
+/* global singlefile, browser */
+
+singlefile.extension.core.bg.companion = {
+
+	save: async options => {
+		try {
+			options.externalSave = false;
+			const port = browser.runtime.connectNative("singlefile_companion");
+			port.postMessage(options);
+			await new Promise((resolve, reject) => {
+				port.onDisconnect.addListener(() => {
+					if (port.error) {
+						reject(port.error.message);
+					} else {
+						resolve();
+					}
+				});
+			});
+			singlefile.extension.ui.bg.main.onEnd(options.tabId, options.autoSave);
+		} catch (error) {
+			console.error(error); // eslint-disable-line no-console			
+			singlefile.extension.ui.bg.main.onError(options.tabId);
+		}
+	}
+
+};

extension/lib/single-file/browser-polyfill/chrome-browser-polyfill.js

@@ -404,6 +404,7 @@
 					addListener: listener => nativeAPI.tabs.onRemoved.addListener(listener),
 					removeListener: listener => nativeAPI.tabs.onRemoved.removeListener(listener)
 				},
+				connectNative: application => nativeAPI.runtime.connectNative(application),
 				executeScript: (tabId, details) => new Promise((resolve, reject) => {
 					nativeAPI.tabs.executeScript(tabId, details, () => {
 						if (nativeAPI.runtime.lastError) {

lib/single-file/single-file-helper.js

@@ -152,7 +152,7 @@ this.singlefile.lib.helper = this.singlefile.lib.helper || (() => {
 		const elements = Array.from(element.childNodes).filter(node => node instanceof win.HTMLElement);
 		elements.forEach(element => {
 			let elementHidden, computedStyle;
-			if (options.removeHiddenElements || options.removeUnusedFonts || options.compressHTML) {
+			if (!options.externalSave && (options.removeHiddenElements || options.removeUnusedFonts || options.compressHTML)) {
 				computedStyle = win.getComputedStyle(element);
 				if (options.removeHiddenElements) {
 					if (ascendantHidden) {

manifest.json

@@ -95,6 +95,7 @@
 			"extension/core/bg/devtools.js",
 			"extension/core/bg/editor.js",
 			"extension/core/bg/bookmarks.js",
+			"extension/core/bg/companion.js",
 			"extension/ui/bg/ui-main.js",
 			"extension/ui/bg/ui-menus.js",
 			"extension/ui/bg/ui-commands.js",