SingleFile/lib/single-file/single-file-util.js

/*
 * Copyright 2010-2020 Gildas Lormeau
 * contact : gildas.lormeau <at> gmail.com
 * 
 * This file is part of SingleFile.
 *
 *   The code in this file is free software: you can redistribute it and/or 
 *   modify it under the terms of the GNU Affero General Public License 
 *   (GNU AGPL) as published by the Free Software Foundation, either version 3
 *   of the License, or (at your option) any later version.
 * 
 *   The code in this file is distributed in the hope that it will be useful, 
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
 *   General Public License for more details.
 *
 *   As additional permission under GNU AGPL version 3 section 7, you may 
 *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
 *   AGPL normally required by section 4, provided you include this license 
 *   notice and a URL through which recipients can access the Corresponding 
 *   Source.
 */

/* global globalThis */

import * as vendor from "./vendor/index.js";
import * as modules from "./modules/index.js";
import * as helper from "./single-file-helper.js";

const DEBUG = false;
const ONE_MB = 1024 * 1024;
const PREFIX_CONTENT_TYPE_TEXT = "text/";
const DEFAULT_REPLACED_CHARACTERS = ["~", "+", "\\\\", "?", "%", "*", ":", "|", "\"", "<", ">", "\x00-\x1f", "\x7F"];
const DEFAULT_REPLACEMENT_CHARACTER = "_";

const URL = globalThis.URL;
const DOMParser = globalThis.DOMParser;
const Blob = globalThis.Blob;
const FileReader = globalThis.FileReader;
const fetch = url => globalThis.fetch(url);
const crypto = globalThis.crypto;
const TextDecoder = globalThis.TextDecoder;
const TextEncoder = globalThis.TextEncoder;

export {
	getInstance
};

function getInstance(utilOptions) {
	utilOptions = utilOptions || {};
	utilOptions.fetch = utilOptions.fetch || fetch;
	utilOptions.frameFetch = utilOptions.frameFetch || utilOptions.fetch || fetch;
	return {
		getContent,
		parseURL(resourceURL, baseURI) {
			if (baseURI === undefined) {
				return new URL(resourceURL);
			} else {
				return new URL(resourceURL, baseURI);
			}
		},
		resolveURL(resourceURL, baseURI) {
			return this.parseURL(resourceURL, baseURI).href;
		},
		getValidFilename(filename, replacedCharacters = DEFAULT_REPLACED_CHARACTERS, replacementCharacter = DEFAULT_REPLACEMENT_CHARACTER) {
			replacedCharacters.forEach(replacedCharacter => filename = filename.replace(new RegExp("[" + replacedCharacter + "]+", "g"), replacementCharacter));
			filename = filename
				.replace(/\.\.\//g, "")
				.replace(/^\/+/, "")
				.replace(/\/+/g, "/")
				.replace(/\/$/, "")
				.replace(/\.$/, "")
				.replace(/\.\//g, "." + replacementCharacter)
				.replace(/\/\./g, "/" + replacementCharacter);
			return filename;
		},
		parseDocContent(content, baseURI) {
			const doc = (new DOMParser()).parseFromString(content, "text/html");
			if (!doc.head) {
				doc.documentElement.insertBefore(doc.createElement("HEAD"), doc.body);
			}
			let baseElement = doc.querySelector("base");
			if (!baseElement || !baseElement.getAttribute("href")) {
				if (baseElement) {
					baseElement.remove();
				}
				baseElement = doc.createElement("base");
				baseElement.setAttribute("href", baseURI);
				doc.head.insertBefore(baseElement, doc.head.firstChild);
			}
			return doc;
		},
		parseXMLContent(content) {
			return (new DOMParser()).parseFromString(content, "text/xml");
		},
		parseSVGContent(content) {
			return (new DOMParser()).parseFromString(content, "image/svg+xml");
		},
		async digest(algo, text) {
			try {
				const hash = await crypto.subtle.digest(algo, new TextEncoder("utf-8").encode(text));
				return hex(hash);
			} catch (error) {
				return "";
			}
		},
		getContentSize(content) {
			return new Blob([content]).size;
		},
		truncateText(content, maxSize) {
			const blob = new Blob([content]);
			const reader = new FileReader();
			reader.readAsText(blob.slice(0, maxSize));
			return new Promise((resolve, reject) => {
				reader.addEventListener("load", () => {
					if (content.startsWith(reader.result)) {
						resolve(reader.result);
					} else {
						this.truncateText(content, maxSize - 1).then(resolve).catch(reject);
					}
				}, false);
				reader.addEventListener("error", reject, false);
			});
		},
		minifyHTML(doc, options) {
			return modules.htmlMinifier.process(doc, options);
		},
		minifyCSSRules(stylesheets, styles, mediaAllInfo) {
			return modules.cssRulesMinifier.process(stylesheets, styles, mediaAllInfo);
		},
		removeUnusedFonts(doc, stylesheets, styles, options) {
			return modules.fontsMinifier.process(doc, stylesheets, styles, options);
		},
		removeAlternativeFonts(doc, stylesheets, fontURLs, fontTests) {
			return modules.fontsAltMinifier.process(doc, stylesheets, fontURLs, fontTests);
		},
		getMediaAllInfo(doc, stylesheets, styles) {
			return modules.matchedRules.getMediaAllInfo(doc, stylesheets, styles);
		},
		compressCSS(content, options) {
			return vendor.cssMinifier.processString(content, options);
		},
		minifyMedias(stylesheets) {
			return modules.mediasAltMinifier.process(stylesheets);
		},
		removeAlternativeImages(doc) {
			return modules.imagesAltMinifier.process(doc);
		},
		parseSrcset(srcset) {
			return vendor.srcsetParser.process(srcset);
		},
		preProcessDoc(doc, win, options) {
			return helper.preProcessDoc(doc, win, options);
		},
		postProcessDoc(doc, markedElements) {
			helper.postProcessDoc(doc, markedElements);
		},
		serialize(doc, compressHTML) {
			return modules.serializer.process(doc, compressHTML);
		},
		removeQuotes(string) {
			return helper.removeQuotes(string);
		},
		waitForUserScript: helper.waitForUserScript,
		ON_BEFORE_CAPTURE_EVENT_NAME: helper.ON_BEFORE_CAPTURE_EVENT_NAME,
		ON_AFTER_CAPTURE_EVENT_NAME: helper.ON_AFTER_CAPTURE_EVENT_NAME,
		WIN_ID_ATTRIBUTE_NAME: helper.WIN_ID_ATTRIBUTE_NAME,
		REMOVED_CONTENT_ATTRIBUTE_NAME: helper.REMOVED_CONTENT_ATTRIBUTE_NAME,
		HIDDEN_CONTENT_ATTRIBUTE_NAME: helper.HIDDEN_CONTENT_ATTRIBUTE_NAME,
		HIDDEN_FRAME_ATTRIBUTE_NAME: helper.HIDDEN_FRAME_ATTRIBUTE_NAME,
		IMAGE_ATTRIBUTE_NAME: helper.IMAGE_ATTRIBUTE_NAME,
		POSTER_ATTRIBUTE_NAME: helper.POSTER_ATTRIBUTE_NAME,
		CANVAS_ATTRIBUTE_NAME: helper.CANVAS_ATTRIBUTE_NAME,
		HTML_IMPORT_ATTRIBUTE_NAME: helper.HTML_IMPORT_ATTRIBUTE_NAME,
		INPUT_VALUE_ATTRIBUTE_NAME: helper.INPUT_VALUE_ATTRIBUTE_NAME,
		SHADOW_ROOT_ATTRIBUTE_NAME: helper.SHADOW_ROOT_ATTRIBUTE_NAME,
		PRESERVED_SPACE_ELEMENT_ATTRIBUTE_NAME: helper.PRESERVED_SPACE_ELEMENT_ATTRIBUTE_NAME,
		STYLESHEET_ATTRIBUTE_NAME: helper.STYLESHEET_ATTRIBUTE_NAME,
		SELECTED_CONTENT_ATTRIBUTE_NAME: helper.SELECTED_CONTENT_ATTRIBUTE_NAME,
		COMMENT_HEADER: helper.COMMENT_HEADER,
		COMMENT_HEADER_LEGACY: helper.COMMENT_HEADER_LEGACY,
		SINGLE_FILE_UI_ELEMENT_CLASS: helper.SINGLE_FILE_UI_ELEMENT_CLASS
	};

	async function getContent(resourceURL, options) {
		let response, startTime;
		const fetchResource = utilOptions.fetch;
		const fetchFrameResource = utilOptions.frameFetch;
		if (DEBUG) {
			startTime = Date.now();
			log("  // STARTED download url =", resourceURL, "asBinary =", options.asBinary);
		}
		try {
			if (options.frameId) {
				try {
					response = await fetchFrameResource(resourceURL, { frameId: options.frameId, referrer: options.resourceReferrer });
				} catch (error) {
					response = await fetchResource(resourceURL);
				}
			} else {
				response = await fetchResource(resourceURL, { referrer: options.resourceReferrer });
			}
		} catch (error) {
			return { data: options.asBinary ? "data:null;base64," : "", resourceURL };
		}
		let buffer;
		try {
			buffer = await response.arrayBuffer();
		} catch (error) {
			return { data: options.asBinary ? "data:null;base64," : "", resourceURL };
		}
		resourceURL = response.url || resourceURL;
		let contentType = "", charset;
		try {
			const mimeType = new vendor.MIMEType(response.headers.get("content-type"));
			contentType = mimeType.type + "/" + mimeType.subtype;
			charset = mimeType.parameters.get("charset");
		} catch (error) {
			// ignored
		}
		if (!contentType) {
			contentType = guessMIMEType(options.expectedType, buffer);
		}
		if (!charset && options.charset) {
			charset = options.charset;
		}
		if (options.asBinary) {
			try {
				if (DEBUG) {
					log("  // ENDED   download url =", resourceURL, "delay =", Date.now() - startTime);
				}
				if (options.maxResourceSizeEnabled && buffer.byteLength > options.maxResourceSize * ONE_MB) {
					return { data: "data:null;base64,", resourceURL };
				} else {
					const reader = new FileReader();
					reader.readAsDataURL(new Blob([buffer], { type: contentType + (options.charset ? ";charset=" + options.charset : "") }));
					const dataUri = await new Promise((resolve, reject) => {
						reader.addEventListener("load", () => resolve(reader.result), false);
						reader.addEventListener("error", reject, false);
					});
					return { data: dataUri, resourceURL };
				}
			} catch (error) {
				return { data: "data:null;base64,", resourceURL };
			}
		} else {
			if (response.status >= 400 || (options.validateTextContentType && contentType && !contentType.startsWith(PREFIX_CONTENT_TYPE_TEXT))) {
				return { data: "", resourceURL };
			}
			if (!charset) {
				charset = "utf-8";
			}
			if (DEBUG) {
				log("  // ENDED   download url =", resourceURL, "delay =", Date.now() - startTime);
			}
			if (options.maxResourceSizeEnabled && buffer.byteLength > options.maxResourceSize * ONE_MB) {
				return { data: "", resourceURL, charset };
			} else {
				try {
					return { data: new TextDecoder(charset).decode(buffer), resourceURL, charset };
				} catch (error) {
					try {
						charset = "utf-8";
						return { data: new TextDecoder(charset).decode(buffer), resourceURL, charset };
					} catch (error) {
						return { data: "", resourceURL, charset };
					}
				}
			}
		}
	}
}

function guessMIMEType(expectedType, buffer) {
	if (expectedType == "image") {
		if (compareBytes([255, 255, 255, 255], [0, 0, 1, 0])) {
			return "image/x-icon";
		}
		if (compareBytes([255, 255, 255, 255], [0, 0, 2, 0])) {
			return "image/x-icon";
		}
		if (compareBytes([255, 255], [78, 77])) {
			return "image/bmp";
		}
		if (compareBytes([255, 255, 255, 255, 255, 255], [71, 73, 70, 56, 57, 97])) {
			return "image/gif";
		}
		if (compareBytes([255, 255, 255, 255, 255, 255], [71, 73, 70, 56, 59, 97])) {
			return "image/gif";
		}
		if (compareBytes([255, 255, 255, 255, 0, 0, 0, 0, 255, 255, 255, 255, 255, 255], [82, 73, 70, 70, 0, 0, 0, 0, 87, 69, 66, 80, 86, 80])) {
			return "image/webp";
		}
		if (compareBytes([255, 255, 255, 255, 255, 255, 255, 255], [137, 80, 78, 71, 13, 10, 26, 10])) {
			return "image/png";
		}
		if (compareBytes([255, 255, 255], [255, 216, 255])) {
			return "image/jpeg";
		}
	}
	if (expectedType == "font") {
		if (compareBytes([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255],
			[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 80])) {
			return "application/vnd.ms-fontobject";
		}
		if (compareBytes([255, 255, 255, 255], [0, 1, 0, 0])) {
			return "font/ttf";
		}
		if (compareBytes([255, 255, 255, 255], [79, 84, 84, 79])) {
			return "font/otf";
		}
		if (compareBytes([255, 255, 255, 255], [116, 116, 99, 102])) {
			return "font/collection";
		}
		if (compareBytes([255, 255, 255, 255], [119, 79, 70, 70])) {
			return "font/woff";
		}
		if (compareBytes([255, 255, 255, 255], [119, 79, 70, 50])) {
			return "font/woff2";
		}
	}

	function compareBytes(mask, pattern) {
		let patternMatch = true, index = 0;
		if (buffer.byteLength >= pattern.length) {
			const value = new Uint8Array(buffer, 0, mask.length);
			for (index = 0; index < mask.length && patternMatch; index++) {
				patternMatch = patternMatch && ((value[index] & mask[index]) == pattern[index]);
			}
			return patternMatch;
		}
	}
}

// https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest
function hex(buffer) {
	const hexCodes = [];
	const view = new DataView(buffer);
	for (let i = 0; i < view.byteLength; i += 4) {
		const value = view.getUint32(i);
		const stringValue = value.toString(16);
		const padding = "00000000";
		const paddedValue = (padding + stringValue).slice(-padding.length);
		hexCodes.push(paddedValue);
	}
	return hexCodes.join("");
}

function log(...args) {
	console.log("S-File <browser>", ...args); // eslint-disable-line no-console
}