rollback

lib/single-file/single-file-core.js

@@ -230,18 +230,16 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 	class BatchRequest {
 		constructor() {
 			this.requests = new Map();
-			this.hashes = [];
-			this.pendingDuplicateCandidates = new Map();
 		}
 
-		async addURL(resourceURL, callback, asDataURI = true) {
+		async addURL(resourceURL, asDataURI = true) {
 			return new Promise((resolve, reject) => {
 				const requestKey = JSON.stringify([resourceURL, asDataURI]);
 				const resourceRequests = this.requests.get(requestKey);
 				if (resourceRequests) {
-					resourceRequests.push({ resolve, reject, callback });
+					resourceRequests.push({ resolve, reject });
 				} else {
-					this.requests.set(requestKey, [{ resolve, reject, callback }]);
+					this.requests.set(requestKey, [{ resolve, reject }]);
 				}
 			});
 		}
@@ -257,26 +255,10 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 				const [resourceURL, asDataURI] = JSON.parse(requestKey);
 				const resourceRequests = this.requests.get(requestKey);
 				try {
-					const result = await Download.getContent(resourceURL, { asDataURI, maxResourceSize: options.maxResourceSize, maxResourceSizeEnabled: options.maxResourceSizeEnabled });
-					let duplicate = Boolean(resourceRequests.length > 1);
-					if (!result.empty) {
-						indexResource = this.hashes.indexOf(result.hash);
-						if (indexResource == -1) {
-							indexResource = this.hashes.length;
-							this.hashes.push(result.hash);
-							this.pendingDuplicateCandidates.set(result.hash, resourceRequests);
-						} else {
-							duplicate = true;
-							const duplicateCandidate = this.pendingDuplicateCandidates.get(result.hash);
-							if (duplicateCandidate) {
-								duplicateCandidate.forEach(resourceRequest => resourceRequest.callback({ content: result.content, empty: result.empty, indexResource, duplicate }));
-								this.pendingDuplicateCandidates.delete(result.hash);
-							}
-						}
-					}
+					const resourceContent = await Download.getContent(resourceURL, { asDataURI, maxResourceSize: options.maxResourceSize, maxResourceSizeEnabled: options.maxResourceSizeEnabled });
+					indexResource = indexResource + 1;
 					onloadListener({ index: indexResource, url: resourceURL });
-					resourceRequests.forEach(resourceRequest => resourceRequest.callback({ content: result.content, empty: result.empty, indexResource, duplicate }));
-					resourceRequests.forEach(resourceRequest => resourceRequest.resolve());
+					resourceRequests.forEach(resourceRequest => resourceRequest.resolve({ content: resourceContent, indexResource, duplicate: Boolean(resourceRequests.length > 1) }));
 				} catch (error) {
 					indexResource = indexResource + 1;
 					onloadListener({ index: indexResource, url: resourceURL });
@@ -290,6 +272,7 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 	// ------------
 	// DOMProcessor
 	// ------------
+	const EMPTY_DATA_URI = "data:base64,";
 	const EMPTY_IMAGE = "data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==";
 
 	class DOMProcessor {
@@ -308,12 +291,11 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 			this.stats.set("processed", "resources", this.maxResources);
 		}
 
-		async loadPage(content) {
-			if (!content || this.options.saveRawPage) {
-				const result = await Download.getContent(this.baseURI, { asDataURI: false, maxResourceSize: this.options.maxResourceSize, maxResourceSizeEnabled: this.options.maxResourceSizeEnabled });
-				content = result.content;
+		async loadPage(pageContent) {
+			if (!pageContent || this.options.saveRawPage) {
+				pageContent = await Download.getContent(this.baseURI, { asDataURI: false, maxResourceSize: this.options.maxResourceSize, maxResourceSizeEnabled: this.options.maxResourceSizeEnabled });
 			}
-			this.doc = DOM.createDoc(content, this.baseURI);
+			this.doc = DOM.createDoc(pageContent, this.baseURI);
 			this.onEventAttributeNames = DOM.getOnEventAttributeNames(this.doc);
 		}
 
@@ -714,7 +696,7 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 			if (this.options.removeAlternativeImages) {
 				const shortcutIcons = Array.from(this.doc.querySelectorAll("link[href][rel=\"icon\"], link[href][rel=\"shortcut icon\"]"));
 				shortcutIcons.sort((linkElement1, linkElement2) => (parseInt(linkElement2.sizes, 10) || 16) - (parseInt(linkElement1.sizes, 10) || 16));
-				const shortcutIcon = shortcutIcons[0];
+				const shortcutIcon = shortcutIcons.find(linkElement => linkElement.href && linkElement.href != EMPTY_DATA_URI);
 				if (shortcutIcon) {
 					this.doc.querySelectorAll("link[href][rel*=\"icon\"]").forEach(linkElement => {
 						if (linkElement != shortcutIcon) {
@@ -740,8 +722,8 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 			await Promise.all(Array.from(this.doc.querySelectorAll("script[src]")).map(async scriptElement => {
 				if (scriptElement.src) {
 					this.stats.add("processed", "scripts", 1);
-					const result = await Download.getContent(scriptElement.src, { asDataURI: false, maxResourceSize: this.options.maxResourceSize, maxResourceSizeEnabled: this.options.maxResourceSizeEnabled });
-					scriptElement.textContent = result.content.replace(/<\/script>/gi, "<\\/script>");
+					const scriptContent = await Download.getContent(scriptElement.src, { asDataURI: false, maxResourceSize: this.options.maxResourceSize, maxResourceSizeEnabled: this.options.maxResourceSizeEnabled });
+					scriptElement.textContent = scriptContent.replace(/<\/script>/gi, "<\\/script>");
 				}
 				scriptElement.removeAttribute("src");
 			}));
@@ -753,7 +735,6 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 				await Promise.all(frameElements.map(async frameElement => {
 					DomProcessorHelper.setFrameEmptySrc(frameElement);
 					frameElement.setAttribute("sandbox", "allow-scripts allow-same-origin");
-					frameElement.removeAttribute("src");
 					const frameWindowId = frameElement.getAttribute(DOM.windowIdAttributeName(this.options.sessionId));
 					if (frameWindowId) {
 						const frameData = this.options.framesData.find(frame => frame.windowId == frameWindowId);
@@ -816,7 +797,7 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 			}
 			await Promise.all(linkElements.map(async linkElement => {
 				const resourceURL = linkElement.href;
-				linkElement.removeAttribute("href");
+				linkElement.setAttribute("href", EMPTY_DATA_URI);
 				const options = Object.create(this.options);
 				options.insertSingleFileComment = false;
 				options.insertFaviconLink = false;
@@ -1003,8 +984,8 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 					if (!DomUtil.testIgnoredPath(resourceURL) && DomUtil.testValidPath(resourceURL)) {
 						resourceURL = new URL(match.resourceURL, baseURI).href;
 						if (DomUtil.testValidURL(resourceURL, baseURI)) {
-							const result = await Download.getContent(resourceURL, { asDataURI: false, maxResourceSize: options.maxResourceSize, maxResourceSizeEnabled: options.maxResourceSizeEnabled });
-							let importedStylesheetContent = DomUtil.wrapMediaQuery(result.content, match.media);
+							let importedStylesheetContent = await Download.getContent(resourceURL, { asDataURI: false, maxResourceSize: options.maxResourceSize, maxResourceSizeEnabled: options.maxResourceSizeEnabled });
+							importedStylesheetContent = DomUtil.wrapMediaQuery(importedStylesheetContent, match.media);
 							if (stylesheetContent.includes(cssImport)) {
 								importedStylesheetContent = await DomProcessorHelper.resolveImportURLs(importedStylesheetContent, resourceURL, options);
 								stylesheetContent = stylesheetContent.replace(DomUtil.getRegExp(cssImport), importedStylesheetContent);
@@ -1043,8 +1024,8 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 		static async resolveLinkStylesheetURLs(resourceURL, baseURI, media, options) {
 			resourceURL = DomUtil.normalizeURL(resourceURL);
 			if (resourceURL && resourceURL != baseURI && resourceURL != ABOUT_BLANK_URI) {
-				const result = await Download.getContent(resourceURL, { asDataURI: false, maxResourceSize: options.maxResourceSize, maxResourceSizeEnabled: options.maxResourceSizeEnabled, charSet: options.charSet });
-				let stylesheetContent = await DomProcessorHelper.resolveImportURLs(result.content, resourceURL, options);
+				let stylesheetContent = await Download.getContent(resourceURL, { asDataURI: false, maxResourceSize: options.maxResourceSize, maxResourceSizeEnabled: options.maxResourceSizeEnabled, charSet: options.charSet });
+				stylesheetContent = await DomProcessorHelper.resolveImportURLs(stylesheetContent, resourceURL, options);
 				stylesheetContent = DomUtil.wrapMediaQuery(stylesheetContent, media);
 				return stylesheetContent;
 			}
@@ -1059,15 +1040,14 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 				const resourceURL = DomUtil.normalizeURL(originalResourceURL);
 				if (!DomUtil.testIgnoredPath(resourceURL)) {
 					if (DomUtil.testValidURL(resourceURL, baseURI) && stylesheetContent.includes(urlFunction)) {
-						await batchRequest.addURL(resourceURL, ({ content, indexResource, duplicate }) => {
-							urlFunction = "url(" + JSON.stringify(resourceURL) + ")";
-							const regExpUrlFunction = DomUtil.getRegExp(urlFunction);
-							if (duplicate && options.groupDuplicateImages) {
-								resourceInfos.set(resourceURL, { regExpUrlFunction, indexResource, dataURI: content, variableName: "var(" + SINGLE_FILE_VARIABLE_NAME_PREFIX + indexResource + ")" });
-							} else {
-								resourceInfos.set(resourceURL, { regExpUrlFunction, indexResource, dataURI: content });
-							}
-						});
+						const { content, indexResource, duplicate } = await batchRequest.addURL(resourceURL);
+						urlFunction = "url(" + JSON.stringify(resourceURL) + ")";
+						const regExpUrlFunction = DomUtil.getRegExp(urlFunction);
+						if (duplicate && options.groupDuplicateImages) {
+							resourceInfos.set(resourceURL, { regExpUrlFunction, indexResource, dataURI: content, variableName: "var(" + SINGLE_FILE_VARIABLE_NAME_PREFIX + indexResource + ")" });
+						} else {
+							resourceInfos.set(resourceURL, { regExpUrlFunction, indexResource, dataURI: content });
+						}
 					}
 				}
 			}));
@@ -1124,19 +1104,18 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 						resourceURL = new URL(resourceURL, baseURI).href;
 						if (DomUtil.testValidURL(resourceURL, baseURI)) {
 							try {
-								await batchRequest.addURL(resourceURL, ({ content, indexResource, duplicate, empty }) => {
-									if (removeElementIfMissing && empty) {
-										resourceElement.remove();
-									} else {
-										if (content.startsWith(prefixDataURI) || content.startsWith(PREFIX_DATA_URI_NO_MIMETYPE) || content.startsWith(PREFIX_DATA_URI_OCTET_STREAM)) {
-											if (processDuplicates && duplicate && options.groupDuplicateImages && !content.startsWith(PREFIX_DATA_URI_IMAGE_SVG) && DomUtil.replaceImageSource(resourceElement, SINGLE_FILE_VARIABLE_NAME_PREFIX + indexResource, options)) {
-												DomUtil.insertVariable(doc, indexResource, content, options);
-											} else {
-												resourceElement.setAttribute(attributeName, content);
-											}
+								const { content, indexResource, duplicate } = await batchRequest.addURL(resourceURL);
+								if (removeElementIfMissing && content == EMPTY_DATA_URI) {
+									resourceElement.remove();
+								} else {
+									if (content.startsWith(prefixDataURI) || content.startsWith(PREFIX_DATA_URI_NO_MIMETYPE) || content.startsWith(PREFIX_DATA_URI_OCTET_STREAM)) {
+										if (processDuplicates && duplicate && options.groupDuplicateImages && !content.startsWith(PREFIX_DATA_URI_IMAGE_SVG) && DomUtil.replaceImageSource(resourceElement, SINGLE_FILE_VARIABLE_NAME_PREFIX + indexResource, options)) {
+											DomUtil.insertVariable(doc, indexResource, content, options);
+										} else {
+											resourceElement.setAttribute(attributeName, content);
 										}
 									}
-								});
+								}
 							} catch (error) {
 								/* ignored */
 							}
@@ -1156,24 +1135,23 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 					resourceURL = new URL(resourceURL, baseURI).href;
 					if (DomUtil.testValidURL(resourceURL, baseURI)) {
 						try {
-							await batchRequest.addURL(resourceURL, ({ content }) => {
-								const DOMParser = DOM.getParser();
-								if (DOMParser) {
-									const svgDoc = new DOMParser().parseFromString(content, "image/svg+xml");
-									const hashMatch = originalResourceURL.match(REGEXP_URL_HASH);
-									if (hashMatch && hashMatch[0]) {
-										const symbolElement = svgDoc.querySelector(hashMatch[0]);
-										if (symbolElement) {
-											resourceElement.setAttribute(attributeName, hashMatch[0]);
-											resourceElement.parentElement.insertBefore(symbolElement, resourceElement.parentElement.firstChild);
-										}
-									} else {
-										resourceElement.setAttribute(attributeName, "data:image/svg+xml," + content);
+							const { content } = await batchRequest.addURL(resourceURL, false);
+							const DOMParser = DOM.getParser();
+							if (DOMParser) {
+								const svgDoc = new DOMParser().parseFromString(content, "image/svg+xml");
+								const hashMatch = originalResourceURL.match(REGEXP_URL_HASH);
+								if (hashMatch && hashMatch[0]) {
+									const symbolElement = svgDoc.querySelector(hashMatch[0]);
+									if (symbolElement) {
+										resourceElement.setAttribute(attributeName, hashMatch[0]);
+										resourceElement.parentElement.insertBefore(symbolElement, resourceElement.parentElement.firstChild);
 									}
 								} else {
 									resourceElement.setAttribute(attributeName, "data:image/svg+xml," + content);
 								}
-							}, false);
+							} else {
+								resourceElement.setAttribute(attributeName, "data:image/svg+xml," + content);
+							}
 						} catch (error) {
 							/* ignored */
 						}
@@ -1191,14 +1169,15 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 						if (DomUtil.testValidPath(resourceURL)) {
 							resourceURL = new URL(resourceURL, baseURI).href;
 							if (DomUtil.testValidURL(resourceURL, baseURI)) {
-								await batchRequest.addURL(resourceURL, ({ content }) => {
-									const attributeValue = resourceElement.getAttribute(attributeName);
-									if (content.startsWith(prefixDataURI) && (content.startsWith(PREFIX_DATA_URI_NO_MIMETYPE) || !content.startsWith(PREFIX_DATA_URI_OCTET_STREAM))) {
-										attributeValue.replace(DomUtil.getRegExp(srcsetValue.url), content);
-									} else {
-										attributeValue.replace(DomUtil.getRegExp(srcsetValue.url), EMPTY_IMAGE);
+								try {
+									const { content } = await batchRequest.addURL(resourceURL);
+									if (!content.startsWith(prefixDataURI) && !content.startsWith(PREFIX_DATA_URI_NO_MIMETYPE) && !content.startsWith(PREFIX_DATA_URI_OCTET_STREAM)) {
+										resourceElement.setAttribute(attributeName, EMPTY_IMAGE);
 									}
-								});
+									return content + (srcsetValue.w ? " " + srcsetValue.w + "w" : srcsetValue.d ? " " + srcsetValue.d + "x" : "");
+								} catch (error) {
+									resourceElement.setAttribute(attributeName, EMPTY_IMAGE);
+								}
 							} else {
 								resourceElement.setAttribute(attributeName, EMPTY_IMAGE);
 							}
@@ -1219,7 +1198,6 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 	const DATA_URI_PREFIX = "data:";
 	const BLOB_URI_PREFIX = "blob:";
 	const HTTP_URI_PREFIX = /^https?:\/\//;
-	const FILE_URI_PREFIX = /^file:\//;
 	const EMPTY_URL = /^https?:\/\/+\s*$/;
 	const ABOUT_BLANK_URI = "about:blank";
 	const NOT_EMPTY_URL = /^https?:\/\/.+/;
@@ -1302,7 +1280,7 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 		}
 
 		static testValidURL(resourceURL, baseURI) {
-			return DomUtil.testValidPath(resourceURL, baseURI) && (resourceURL.match(HTTP_URI_PREFIX) || resourceURL.match(FILE_URI_PREFIX)) && resourceURL.match(NOT_EMPTY_URL);
+			return DomUtil.testValidPath(resourceURL, baseURI) && resourceURL.match(HTTP_URI_PREFIX) && resourceURL.match(NOT_EMPTY_URL);
 		}
 
 		static matchImport(stylesheetContent) {