moved BatchRequest instance into PageProcessor

extension/core/bg/core.js

@@ -144,9 +144,14 @@ singlefile.core = (() => {
 		options.backgroundTab = true;
 		options.autoSave = true;
 		options.incognito = tab.incognito;
+		let index = 0, maxIndex = 0;
 		options.onprogress = async event => {
+			if (event.type == event.RESOURCES_INITIALIZED) {
+				maxIndex = event.details.max;
+			}
 			if (event.type == event.RESOURCES_INITIALIZED || event.type == event.RESOURCE_LOADED) {
-				singlefile.ui.button.onProgress(tab.id, event.details.index, event.details.max, { autoSave: true });
+				index++;
+				singlefile.ui.button.onProgress(tab.id, index, maxIndex, { autoSave: true });
 			} else if (event.type == event.PAGE_ENDED) {
 				singlefile.ui.button.onEnd(tab.id, { autoSave: true });
 			}

extension/core/content/content.js

@@ -66,11 +66,16 @@ this.singlefile.top = this.singlefile.top || (() => {
 		}
 		options.doc = document;
 		options.win = window;
+		let index = 0, maxIndex = 0;
 		options.onprogress = event => {
+			if (event.type == event.RESOURCES_INITIALIZED) {
+				maxIndex = event.details.max;
+			}
 			if (event.type == event.RESOURCES_INITIALIZED || event.type == event.RESOURCE_LOADED) {
-				browser.runtime.sendMessage({ processProgress: true, index: event.details.index, maxIndex: event.details.max, options: { autoSave: false } });
+				index++;
+				browser.runtime.sendMessage({ processProgress: true, index, maxIndex, options: { autoSave: false } });
 				if (options.shadowEnabled) {
-					singlefile.ui.onprogress(event);
+					singlefile.ui.onprogress(index, maxIndex);
 				}
 			} else if (event.type == event.PAGE_ENDED) {
 				browser.runtime.sendMessage({ processEnd: true, options: { autoSave: false } });

extension/ui/content/content-ui.js

@@ -56,12 +56,12 @@ this.singlefile.ui = this.singlefile.ui || (() => {
 				document.body.offsetWidth;
 			}
 		},
-		onprogress(event) {
+		onprogress(index, maxIndex) {
 			const progressBarElement = document.querySelector(PROGRESS_BAR_TAGNAME);
-			if (progressBarElement && event.details.max) {
-				const width = Math.floor((event.details.index / event.details.max) * 100) + "%";
+			if (progressBarElement && maxIndex) {
+				const width = Math.floor((index / maxIndex) * 100) + "%";
 				if (progressBarElement.style.width != width) {
-					progressBarElement.style.setProperty("width", Math.floor((event.details.index / event.details.max) * 100) + "%", "important");
+					progressBarElement.style.setProperty("width", Math.floor((index / maxIndex) * 100) + "%", "important");
 					progressBarElement.offsetWidth;
 				}
 			}

lib/single-file/single-file-core.js

@@ -126,7 +126,8 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 		constructor(options) {
 			this.options = options;
 			this.options.url = this.options.url || this.options.doc.location.href;
-			this.processor = new DOMProcessor(options);
+			this.batchRequest = new BatchRequest();
+			this.processor = new DOMProcessor(options, this.batchRequest);
 			if (this.options.doc) {
 				const docData = DOM.preProcessDoc(this.options.doc, this.options.win, this.options);
 				this.options.canvasData = docData.canvasData;
@@ -152,24 +153,26 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 				this.options.doc = null;
 				this.options.win = null;
 			}
-			this.onprogress(new ProgressEvent(RESOURCES_INITIALIZED, { pageURL: this.options.url, index: 0, max: batchRequest.getMaxResources() }));
 		}
 
 		async preparePageData() {
 			if (!this.options.windowId) {
-				await this.processor.retrieveResources(
-					details => {
-						details.pageURL = this.options.url;
-						this.onprogress(new ProgressEvent(RESOURCE_LOADED, details));
-					});
+				this.processor.initialize(this.batchRequest);
+				this.onprogress(new ProgressEvent(RESOURCES_INITIALIZED, { pageURL: this.options.url, index: 0, max: this.processor.maxResources }));
 			}
+			await this.batchRequest.run(details => {
+				details.pageURL = this.options.url;
+				this.onprogress(new ProgressEvent(RESOURCE_LOADED, details));
+			}, this.options);
 			await this.pendingPromises;
 			await this.executeStage(2);
 			await this.executeStage(3);
 		}
 
 		getPageData() {
-			this.onprogress(new ProgressEvent(PAGE_ENDED, { pageURL: this.options.url }));
+			if (!this.options.windowId) {
+				this.onprogress(new ProgressEvent(PAGE_ENDED, { pageURL: this.options.url }));
+			}
 			return this.processor.getPageData();
 		}
 
@@ -220,11 +223,11 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 				try {
 					const dataURI = await Download.getContent(resourceURL, { asDataURI, maxResourceSize: options.maxResourceSize, maxResourceSizeEnabled: options.maxResourceSizeEnabled });
 					indexResource = indexResource + 1;
-					onloadListener({ index: indexResource, max: resourceURLs.length, url: resourceURL });
+					onloadListener({ index: indexResource, url: resourceURL });
 					resourceRequests.forEach(resourceRequest => resourceRequest.resolve(dataURI));
 				} catch (error) {
 					indexResource = indexResource + 1;
-					onloadListener({ index: indexResource, max: resourceURLs.length, url: resourceURL });
+					onloadListener({ index: indexResource, url: resourceURL });
 					resourceRequests.forEach(resourceRequest => resourceRequest.reject(error));
 				}
 				this.requests.delete(requestKey);
@@ -238,13 +241,20 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 	const ESCAPED_FRAGMENT = "_escaped_fragment_=";
 	const EMPTY_DATA_URI = "data:base64,";
 
-	const batchRequest = new BatchRequest();
-
 	class DOMProcessor {
-		constructor(options) {
+		constructor(options, batchRequest) {
 			this.options = options;
 			this.stats = new Stats(options);
 			this.baseURI = DomUtil.normalizeURL(options.url);
+			this.batchRequest = batchRequest;
+		}
+
+		initialize() {
+			this.maxResources = this.batchRequest.getMaxResources();
+			if (!this.options.removeFrames) {
+				this.options.framesData.forEach(frameData => this.maxResources += frameData.maxResources || 0);
+			}
+			this.stats.set("processed", "resources", this.maxResources);
 		}
 
 		async loadPage(pageContent) {
@@ -268,11 +278,6 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 			await this.loadPage();
 		}
 
-		async retrieveResources(onloadListener) {
-			this.stats.set("processed", "resources", batchRequest.getMaxResources());
-			await batchRequest.run(onloadListener, this.options);
-		}
-
 		getPageData() {
 			DOM.postProcessDoc(this.doc, this.options);
 			if (this.options.selected) {
@@ -584,14 +589,14 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 
 		async pageResources() {
 			const resourcePromises = [
-				DomProcessorHelper.processAttribute(this.doc.querySelectorAll("link[href][rel*=\"icon\"]"), "href", this.baseURI),
-				DomProcessorHelper.processAttribute(this.doc.querySelectorAll("object[type=\"image/svg+xml\"], object[type=\"image/svg-xml\"]"), "data", this.baseURI),
-				DomProcessorHelper.processAttribute(this.doc.querySelectorAll("img[src], input[src][type=image], embed[src*=\".svg\"]"), "src", this.baseURI),
-				DomProcessorHelper.processAttribute(this.doc.querySelectorAll("video[poster]"), "poster", this.baseURI),
-				DomProcessorHelper.processAttribute(this.doc.querySelectorAll("*[background]"), "background", this.baseURI),
-				DomProcessorHelper.processAttribute(this.doc.querySelectorAll("image"), "xlink:href", this.baseURI),
-				DomProcessorHelper.processXLinks(this.doc.querySelectorAll("use"), this.baseURI),
-				DomProcessorHelper.processSrcset(this.doc.querySelectorAll("[srcset]"), "srcset", this.baseURI)
+				DomProcessorHelper.processAttribute(this.doc.querySelectorAll("link[href][rel*=\"icon\"]"), "href", this.baseURI, this.batchRequest),
+				DomProcessorHelper.processAttribute(this.doc.querySelectorAll("object[type=\"image/svg+xml\"], object[type=\"image/svg-xml\"]"), "data", this.baseURI, this.batchRequest),
+				DomProcessorHelper.processAttribute(this.doc.querySelectorAll("img[src], input[src][type=image], embed[src*=\".svg\"]"), "src", this.baseURI, this.batchRequest),
+				DomProcessorHelper.processAttribute(this.doc.querySelectorAll("video[poster]"), "poster", this.baseURI, this.batchRequest),
+				DomProcessorHelper.processAttribute(this.doc.querySelectorAll("*[background]"), "background", this.baseURI, this.batchRequest),
+				DomProcessorHelper.processAttribute(this.doc.querySelectorAll("image"), "xlink:href", this.baseURI, this.batchRequest),
+				DomProcessorHelper.processXLinks(this.doc.querySelectorAll("use"), this.baseURI, this.batchRequest),
+				DomProcessorHelper.processSrcset(this.doc.querySelectorAll("[srcset]"), "srcset", this.baseURI, this.batchRequest)
 			];
 			if (!this.options.removeAudioSrc) {
 				resourcePromises.push(DomProcessorHelper.processAttribute(this.doc.querySelectorAll("audio[src], audio > source[src]"), "src", this.baseURI));
@@ -616,7 +621,7 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 				if (initialization) {
 					stylesheetContent = await DomProcessorHelper.resolveImportURLs(styleElement.textContent, this.baseURI, { maxResourceSize: this.options.maxResourceSize, maxResourceSizeEnabled: this.options.maxResourceSizeEnabled });
 				} else {
-					stylesheetContent = await DomProcessorHelper.processStylesheet(styleElement.textContent, this.baseURI);
+					stylesheetContent = await DomProcessorHelper.processStylesheet(styleElement.textContent, this.baseURI, this.batchRequest);
 				}
 				styleElement.textContent = stylesheetContent;
 			}));
@@ -659,7 +664,8 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 									frameData.processor = new PageProcessor(options);
 									frameData.frameElement = frameElement;
 									await frameData.processor.loadPage();
-									return frameData.processor.initialize();
+									await frameData.processor.initialize();
+									frameData.maxResources = this.batchRequest.getMaxResources();
 								}
 							} else {
 								if (frameData.processor) {
@@ -723,7 +729,7 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 				if (initialization) {
 					stylesheetContent = DomProcessorHelper.resolveStylesheetURLs(stylesheetContent, this.baseURI);
 				} else {
-					stylesheetContent = await DomProcessorHelper.processStylesheet(element.getAttribute("style"), this.baseURI);
+					stylesheetContent = await DomProcessorHelper.processStylesheet(element.getAttribute("style"), this.baseURI, this.batchRequest);
 				}
 				element.setAttribute("style", stylesheetContent);
 			}));
@@ -843,7 +849,7 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 			}
 		}
 
-		static async processStylesheet(stylesheetContent, baseURI) {
+		static async processStylesheet(stylesheetContent, baseURI, batchRequest) {
 			stylesheetContent = DomProcessorHelper.resolveStylesheetURLs(stylesheetContent, baseURI);
 			const urlFunctions = DomUtil.getUrlFunctions(stylesheetContent);
 			await Promise.all(urlFunctions.map(async urlFunction => {
@@ -857,7 +863,7 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 			return stylesheetContent;
 		}
 
-		static async processAttribute(resourceElements, attributeName, baseURI) {
+		static async processAttribute(resourceElements, attributeName, baseURI, batchRequest) {
 			await Promise.all(Array.from(resourceElements).map(async resourceElement => {
 				let resourceURL = resourceElement.getAttribute(attributeName);
 				if (resourceURL) {
@@ -874,7 +880,7 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 			}));
 		}
 
-		static async processXLinks(resourceElements, baseURI) {
+		static async processXLinks(resourceElements, baseURI, batchRequest) {
 			await Promise.all(Array.from(resourceElements).map(async resourceElement => {
 				const originalResourceURL = resourceElement.getAttribute("xlink:href");
 				if (originalResourceURL) {
@@ -906,7 +912,7 @@ this.SingleFileCore = this.SingleFileCore || (() => {
 			}));
 		}
 
-		static async processSrcset(resourceElements, attributeName, baseURI) {
+		static async processSrcset(resourceElements, attributeName, baseURI, batchRequest) {
 			await Promise.all(Array.from(resourceElements).map(async resourceElement => {
 				const srcset = DOM.parseSrcset(resourceElement.getAttribute(attributeName));
 				const srcsetValues = await Promise.all(srcset.map(async srcsetValue => {