split large responses (fix #873)

extension/lib/single-file/fetch/bg/fetch.js

@@ -25,6 +25,8 @@
 
 const referrers = new Map();
 const REQUEST_ID_HEADER_NAME = "x-single-file-request-id";
+const MAX_CONTENT_SIZE = 8 * (1024 * 1024);
+
 export {
 	REQUEST_ID_HEADER_NAME,
 	referrers
@@ -40,14 +42,40 @@ browser.runtime.onMessage.addListener((message, sender) => {
 	}
 });
 
-function onRequest(message, sender) {
+async function onRequest(message, sender) {
 	if (message.method == "singlefile.fetch") {
-		return fetchResource(message.url, { referrer: message.referrer, headers: message.headers });
+		try {
+			const response = await fetchResource(message.url, { referrer: message.referrer, headers: message.headers });
+			return sendResponse(sender.tab.id, message.requestId, response);
+		} catch (error) {
+			return sendResponse(sender.tab.id, message.requestId, { error: error.message, arrray: [] });
+		}
 	} else if (message.method == "singlefile.fetchFrame") {
 		return browser.tabs.sendMessage(sender.tab.id, message);
 	}
 }
 
+async function sendResponse(tabId, requestId, response) {
+	for (let blockIndex = 0; blockIndex * MAX_CONTENT_SIZE < response.array.length; blockIndex++) {
+		const message = {
+			method: "singlefile.fetchResponse",
+			requestId,
+			headers: response.headers,
+			status: response.status,
+			error: response.error
+		};
+		message.truncated = response.array.length > MAX_CONTENT_SIZE;
+		if (message.truncated) {
+			message.finished = (blockIndex + 1) * MAX_CONTENT_SIZE > response.array.length;
+			message.array = response.array.slice(blockIndex * MAX_CONTENT_SIZE, (blockIndex + 1) * MAX_CONTENT_SIZE);
+		} else {
+			message.array = response.array;
+		}
+		await browser.tabs.sendMessage(tabId, message);
+	}
+	return {};
+}
+
 function fetchResource(url, options = {}, includeRequestId) {
 	return new Promise((resolve, reject) => {
 		const xhrRequest = new XMLHttpRequest();

extension/lib/single-file/fetch/content/content-fetch.js

@@ -31,13 +31,18 @@ const dispatchEvent = event => window.dispatchEvent(event);
 const removeEventListener = (type, listener, options) => window.removeEventListener(type, listener, options);
 const fetch = (url, options) => window.fetch(url, options);
 
+let requestId = 0, pendingResponses = new Map();
+
 browser.runtime.onMessage.addListener(message => {
 	if (message.method == "singlefile.fetchFrame" && window.frameId && window.frameId == message.frameId) {
-		return onMessage(message);
+		return onFetchFrame(message);
+	}
+	if (message.method == "singlefile.fetchResponse") {
+		return onFetchResponse(message);
 	}
 });
 
-async function onMessage(message) {
+async function onFetchFrame(message) {
 	try {
 		let response = await fetch(message.url, { cache: "force-cache", headers: message.headers });
 		if (response.status == 401 || response.status == 403 || response.status == 404) {
@@ -59,6 +64,37 @@ async function onMessage(message) {
 	}
 }
 
+async function onFetchResponse(message) {
+	const pendingResponse = pendingResponses.get(message.requestId);
+	if (pendingResponse) {
+		if (message.error) {
+			pendingResponse.reject(new Error(message.error));
+			pendingResponses.delete(message.requestId);
+		} else {
+			if (message.truncated) {
+				if (pendingResponse.array) {
+					pendingResponse.array = pendingResponse.array.concat(message.array);
+				} else {
+					pendingResponse.array = message.array;
+					pendingResponses.set(message.requestId, pendingResponse);
+				}
+				if (message.finished) {
+					message.array = pendingResponse.array;
+				}
+			}
+			if (!message.truncated || message.finished) {
+				pendingResponse.resolve({
+					status: message.status,
+					headers: { get: headerName => message.headers && message.headers[headerName] },
+					arrayBuffer: async () => new Uint8Array(message.array).buffer
+				});
+				pendingResponses.delete(message.requestId);
+			}
+		}
+	}
+	return {};
+}
+
 export {
 	fetchResource as fetch,
 	frameFetch
@@ -73,12 +109,10 @@ async function fetchResource(url, options = {}) {
 		return response;
 	}
 	catch (error) {
-		const response = await sendMessage({ method: "singlefile.fetch", url, referrer: options.referrer, headers: options.headers });
-		return {
-			status: response.status,
-			headers: { get: headerName => response.headers && response.headers[headerName] },
-			arrayBuffer: async () => new Uint8Array(response.array).buffer
-		};
+		requestId++;
+		const promise = new Promise((resolve, reject) => pendingResponses.set(requestId, { resolve, reject }));
+		await sendMessage({ method: "singlefile.fetch", url, requestId, referrer: options.referrer, headers: options.headers });
+		return promise;
 	}
 }