downloads: fixed grouping by content hash

lib/single-file/single-file-browser.js

@@ -104,7 +104,7 @@ this.SingleFile = this.SingleFile || (() => {
 					if (options.maxResourceSizeEnabled && buffer.byteLength > options.maxResourceSize * ONE_MB) {
 						return { empty: true };
 					} else {
-						const hash = await crypto.subtle.digest("SHA-256", uInt8Array);
+						const hash = hex(await crypto.subtle.digest("SHA-256", uInt8Array));
 						return { content: dataURI, hash };
 					}
 				} catch (error) {
@@ -129,7 +129,7 @@ this.SingleFile = this.SingleFile || (() => {
 					if (options.maxResourceSizeEnabled && textContent.length > options.maxResourceSize * ONE_MB) {
 						return { content: "" };
 					} else {
-						const hash = await crypto.subtle.digest("SHA-256", arrayBuffer);
+						const hash = hex(await crypto.subtle.digest("SHA-256", arrayBuffer));
 						return { content: textContent, hash };
 					}
 				} catch (error) {

lib/single-file/single-file-core.js

@@ -231,16 +231,17 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 		constructor() {
 			this.requests = new Map();
 			this.hashes = [];
+			this.pendingDuplicateCandidates = new Map();
 		}
 
-		async addURL(resourceURL, asDataURI = true) {
+		async addURL(resourceURL, callback, asDataURI = true) {
 			return new Promise((resolve, reject) => {
 				const requestKey = JSON.stringify([resourceURL, asDataURI]);
 				const resourceRequests = this.requests.get(requestKey);
 				if (resourceRequests) {
-					resourceRequests.push({ resolve, reject });
+					resourceRequests.push({ resolve, reject, callback });
 				} else {
-					this.requests.set(requestKey, [{ resolve, reject }]);
+					this.requests.set(requestKey, [{ resolve, reject, callback }]);
 				}
 			});
 		}
@@ -257,16 +258,25 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 				const resourceRequests = this.requests.get(requestKey);
 				try {
 					const result = await Download.getContent(resourceURL, { asDataURI, maxResourceSize: options.maxResourceSize, maxResourceSizeEnabled: options.maxResourceSizeEnabled });
-					indexResource = this.hashes.indexOf(result.hash);
 					let duplicate = Boolean(resourceRequests.length > 1);
-					if (indexResource == -1) {
-						indexResource = this.hashes.length;
-						this.hashes.push(result.hash);
-					} else {
-						duplicate = true;
+					if (!result.empty) {
+						indexResource = this.hashes.indexOf(result.hash);
+						if (indexResource == -1) {
+							indexResource = this.hashes.length;
+							this.hashes.push(result.hash);
+							this.pendingDuplicateCandidates.set(result.hash, resourceRequests);
+						} else {
+							duplicate = true;
+							const duplicateCandidate = this.pendingDuplicateCandidates.get(result.hash);
+							if (duplicateCandidate) {
+								duplicateCandidate.forEach(resourceRequest => resourceRequest.callback({ content: result.content, empty: result.empty, indexResource, duplicate }));
+								this.pendingDuplicateCandidates.delete(result.hash);
+							}
+						}
 					}
 					onloadListener({ index: indexResource, url: resourceURL });
-					resourceRequests.forEach(resourceRequest => resourceRequest.resolve({ content: result.content, empty: result.empty, indexResource, duplicate }));
+					resourceRequests.forEach(resourceRequest => resourceRequest.callback({ content: result.content, empty: result.empty, indexResource, duplicate }));
+					resourceRequests.forEach(resourceRequest => resourceRequest.resolve());
 				} catch (error) {
 					indexResource = indexResource + 1;
 					onloadListener({ index: indexResource, url: resourceURL });
@@ -1049,14 +1059,15 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 				const resourceURL = DomUtil.normalizeURL(originalResourceURL);
 				if (!DomUtil.testIgnoredPath(resourceURL)) {
 					if (DomUtil.testValidURL(resourceURL, baseURI) && stylesheetContent.includes(urlFunction)) {
-						const { content, indexResource, duplicate } = await batchRequest.addURL(resourceURL);
-						urlFunction = "url(" + JSON.stringify(resourceURL) + ")";
-						const regExpUrlFunction = DomUtil.getRegExp(urlFunction);
-						if (duplicate && options.groupDuplicateImages) {
-							resourceInfos.set(resourceURL, { regExpUrlFunction, indexResource, dataURI: content, variableName: "var(" + SINGLE_FILE_VARIABLE_NAME_PREFIX + indexResource + ")" });
-						} else {
-							resourceInfos.set(resourceURL, { regExpUrlFunction, indexResource, dataURI: content });
-						}
+						await batchRequest.addURL(resourceURL, ({ content, indexResource, duplicate }) => {
+							urlFunction = "url(" + JSON.stringify(resourceURL) + ")";
+							const regExpUrlFunction = DomUtil.getRegExp(urlFunction);
+							if (duplicate && options.groupDuplicateImages) {
+								resourceInfos.set(resourceURL, { regExpUrlFunction, indexResource, dataURI: content, variableName: "var(" + SINGLE_FILE_VARIABLE_NAME_PREFIX + indexResource + ")" });
+							} else {
+								resourceInfos.set(resourceURL, { regExpUrlFunction, indexResource, dataURI: content });
+							}
+						});
 					}
 				}
 			}));
@@ -1113,18 +1124,19 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 						resourceURL = new URL(resourceURL, baseURI).href;
 						if (DomUtil.testValidURL(resourceURL, baseURI)) {
 							try {
-								const { content, indexResource, duplicate, empty } = await batchRequest.addURL(resourceURL);
-								if (removeElementIfMissing && empty) {
-									resourceElement.remove();
-								} else {
-									if (content.startsWith(prefixDataURI) || content.startsWith(PREFIX_DATA_URI_NO_MIMETYPE) || content.startsWith(PREFIX_DATA_URI_OCTET_STREAM)) {
-										if (processDuplicates && duplicate && options.groupDuplicateImages && !content.startsWith(PREFIX_DATA_URI_IMAGE_SVG) && DomUtil.replaceImageSource(resourceElement, SINGLE_FILE_VARIABLE_NAME_PREFIX + indexResource, options)) {
-											DomUtil.insertVariable(doc, indexResource, content, options);
-										} else {
-											resourceElement.setAttribute(attributeName, content);
+								await batchRequest.addURL(resourceURL, ({ content, indexResource, duplicate, empty }) => {
+									if (removeElementIfMissing && empty) {
+										resourceElement.remove();
+									} else {
+										if (content.startsWith(prefixDataURI) || content.startsWith(PREFIX_DATA_URI_NO_MIMETYPE) || content.startsWith(PREFIX_DATA_URI_OCTET_STREAM)) {
+											if (processDuplicates && duplicate && options.groupDuplicateImages && !content.startsWith(PREFIX_DATA_URI_IMAGE_SVG) && DomUtil.replaceImageSource(resourceElement, SINGLE_FILE_VARIABLE_NAME_PREFIX + indexResource, options)) {
+												DomUtil.insertVariable(doc, indexResource, content, options);
+											} else {
+												resourceElement.setAttribute(attributeName, content);
+											}
 										}
 									}
-								}
+								});
 							} catch (error) {
 								/* ignored */
 							}
@@ -1144,23 +1156,24 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 					resourceURL = new URL(resourceURL, baseURI).href;
 					if (DomUtil.testValidURL(resourceURL, baseURI)) {
 						try {
-							const { content } = await batchRequest.addURL(resourceURL, false);
-							const DOMParser = DOM.getParser();
-							if (DOMParser) {
-								const svgDoc = new DOMParser().parseFromString(content, "image/svg+xml");
-								const hashMatch = originalResourceURL.match(REGEXP_URL_HASH);
-								if (hashMatch && hashMatch[0]) {
-									const symbolElement = svgDoc.querySelector(hashMatch[0]);
-									if (symbolElement) {
-										resourceElement.setAttribute(attributeName, hashMatch[0]);
-										resourceElement.parentElement.insertBefore(symbolElement, resourceElement.parentElement.firstChild);
+							await batchRequest.addURL(resourceURL, ({ content }) => {
+								const DOMParser = DOM.getParser();
+								if (DOMParser) {
+									const svgDoc = new DOMParser().parseFromString(content, "image/svg+xml");
+									const hashMatch = originalResourceURL.match(REGEXP_URL_HASH);
+									if (hashMatch && hashMatch[0]) {
+										const symbolElement = svgDoc.querySelector(hashMatch[0]);
+										if (symbolElement) {
+											resourceElement.setAttribute(attributeName, hashMatch[0]);
+											resourceElement.parentElement.insertBefore(symbolElement, resourceElement.parentElement.firstChild);
+										}
+									} else {
+										resourceElement.setAttribute(attributeName, "data:image/svg+xml," + content);
 									}
 								} else {
 									resourceElement.setAttribute(attributeName, "data:image/svg+xml," + content);
 								}
-							} else {
-								resourceElement.setAttribute(attributeName, "data:image/svg+xml," + content);
-							}
+							}, false);
 						} catch (error) {
 							/* ignored */
 						}
@@ -1178,15 +1191,14 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 						if (DomUtil.testValidPath(resourceURL)) {
 							resourceURL = new URL(resourceURL, baseURI).href;
 							if (DomUtil.testValidURL(resourceURL, baseURI)) {
-								try {
-									const { content } = await batchRequest.addURL(resourceURL);
-									if (!content.startsWith(prefixDataURI) && !content.startsWith(PREFIX_DATA_URI_NO_MIMETYPE) && !content.startsWith(PREFIX_DATA_URI_OCTET_STREAM)) {
-										resourceElement.setAttribute(attributeName, EMPTY_IMAGE);
+								await batchRequest.addURL(resourceURL, ({ content }) => {
+									const attributeValue = resourceElement.getAttribute(attributeName);
+									if (content.startsWith(prefixDataURI) && (content.startsWith(PREFIX_DATA_URI_NO_MIMETYPE) || !content.startsWith(PREFIX_DATA_URI_OCTET_STREAM))) {
+										attributeValue.replace(DomUtil.getRegExp(srcsetValue.url), content);
+									} else {
+										attributeValue.replace(DomUtil.getRegExp(srcsetValue.url), EMPTY_IMAGE);
 									}
-									return content + (srcsetValue.w ? " " + srcsetValue.w + "w" : srcsetValue.d ? " " + srcsetValue.d + "x" : "");
-								} catch (error) {
-									resourceElement.setAttribute(attributeName, EMPTY_IMAGE);
-								}
+								});
 							} else {
 								resourceElement.setAttribute(attributeName, EMPTY_IMAGE);
 							}
@@ -1207,6 +1219,7 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 	const DATA_URI_PREFIX = "data:";
 	const BLOB_URI_PREFIX = "blob:";
 	const HTTP_URI_PREFIX = /^https?:\/\//;
+	const FILE_URI_PREFIX = /^file:\//;
 	const EMPTY_URL = /^https?:\/\/+\s*$/;
 	const ABOUT_BLANK_URI = "about:blank";
 	const NOT_EMPTY_URL = /^https?:\/\/.+/;
@@ -1289,7 +1302,7 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 		}
 
 		static testValidURL(resourceURL, baseURI) {
-			return DomUtil.testValidPath(resourceURL, baseURI) && resourceURL.match(HTTP_URI_PREFIX) && resourceURL.match(NOT_EMPTY_URL);
+			return DomUtil.testValidPath(resourceURL, baseURI) && (resourceURL.match(HTTP_URI_PREFIX) || resourceURL.match(FILE_URI_PREFIX)) && !resourceURL.match(NOT_EMPTY_URL);
 		}
 
 		static matchImport(stylesheetContent) {