add option --browser-cookie (fix #559)

cli/args.js

@@ -43,6 +43,7 @@ const args = require("yargs")
 		"browser-scripts": [],
 		"browser-args": "",
 		"browser-start-minimized": false,
+		"browser-cookie": [],
 		"compress-CSS": false,
 		"compress-HTML": true,
 		"dump-content": false,
@@ -108,6 +109,8 @@ const args = require("yargs")
 	.string("browser-args")
 	.options("browser-start-minimized", { description: "Minimize the browser (puppeteer)" })
 	.boolean("browser-start-minimized")
+	.options("browser-cookie", { description: "Ordered list of cookie parameters separated by a comma: name,value,domain,path,expires,httpOnly,secure,sameSite,url (puppeteer, webdriver-gecko, webdriver-chromium, jsdom)" })
+	.array("browser-cookie")
 	.options("compress-CSS", { description: "Compress CSS stylesheets" })
 	.boolean("compress-CSS")
 	.options("compress-HTML", { description: "Compress HTML content" })
@@ -215,6 +218,22 @@ headers.forEach(header => {
 		args.httpHeaders[matchedHeader[1].trim()] = matchedHeader[2].trimLeft();
 	}
 });
+const cookies = args.browserCookie;
+delete args.browserCookie;
+args.browserCookies = cookies.map(cookieValue => {
+	const value = cookieValue.split(/(?<!\\),/);
+	return {
+		name: value[0],
+		value: value[1],
+		domain: value[2] || undefined,
+		path: value[3] || undefined,
+		expires: value[4] && Number(value[4]) || undefined,
+		httpOnly: value[5] && value[5] == "true" || undefined,
+		secure: value[6] && value[5] == "true" || undefined,
+		sameSite: value[7] || undefined,
+		url: value[8] || undefined
+	};
+});
 Object.keys(args).filter(optionName => optionName.includes("-"))
 	.forEach(optionName => delete args[optionName]);
 delete args["$0"];

cli/back-ends/jsdom.js

@@ -34,7 +34,7 @@ exports.initialize = async () => { };
 exports.getPageData = async options => {
 	let win;
 	try {
-		const dom = await JSDOM.fromURL(options.url, getBrowserOptions(options));
+		const dom = await JSDOM.fromURL(options.url, await getBrowserOptions(options));
 		win = dom.window;
 		return await getPageData(win, options);
 	} finally {
@@ -104,7 +104,7 @@ async function getPageData(win, options) {
 	}
 }
 
-function getBrowserOptions(options) {
+async function getBrowserOptions(options) {
 	class ResourceLoader extends jsdom.ResourceLoader {
 		_getRequestOptions(fetchOptions) {
 			const requestOptions = super._getRequestOptions(fetchOptions);
@@ -130,6 +130,31 @@ function getBrowserOptions(options) {
 			window.outerHeight = window.innerHeight = options.browserHeight;
 		};
 	}
+	if (options.browserCookies && options.browserCookies.length) {
+		jsdomOptions.cookieJar = new jsdom.CookieJar();
+		await Promise.all(options.browserCookies.map(cookie => {
+			let cookieString = cookie.name + "=" + cookie.value;
+			if (cookie.path) {
+				cookieString += ";path=" + cookie.path;
+			}
+			if (cookie.domain) {
+				cookieString += ";domain=" + cookie.domain;
+			}
+			if (cookie.expires) {
+				cookieString += ";max-age=" + cookie.expires;
+			}
+			if (cookie.secure) {
+				cookieString += ";secure";
+			}
+			if (cookie.sameSite) {
+				cookieString += ";samesite=" + options.sameSite;
+			}
+			const cookieOptions = {
+				http: Boolean(cookie.httpOnly)
+			};
+			return new Promise((resolve, reject) => jsdomOptions.cookieJar.setCookie(cookieString, options.url, cookieOptions, error => error ? reject(error) : resolve()));
+		}));
+	}
 	return jsdomOptions;
 }
 

cli/back-ends/playwright-chromium.js

@@ -35,11 +35,13 @@ exports.initialize = async options => {
 };
 
 exports.getPageData = async options => {
-	let page;
+	let page, context;
 	try {
-		page = await browser.newPage({
+		context = await browser.newContext({
 			bypassCSP: options.browserBypassCSP === undefined || options.browserBypassCSP
 		});
+		await setContextOptions(context, options);
+		page = await context.newPage();
 		await setPageOptions(page, options);
 		return await getPageData(page, options);
 	} finally {
@@ -67,6 +69,12 @@ function getBrowserOptions(options) {
 	return browserOptions;
 }
 
+async function setContextOptions(context, options) {
+	if (options.browserCookies && options.browserCookies.length) {
+		await context.addCookies(options.browserCookies);
+	}
+}
+
 async function setPageOptions(page, options) {
 	if (options.browserWidth && options.browserHeight) {
 		await page.setViewportSize({

cli/back-ends/playwright-firefox.js

@@ -35,11 +35,13 @@ exports.initialize = async options => {
 };
 
 exports.getPageData = async options => {
-	let page;
+	let page, context;
 	try {
-		page = await browser.newPage({
+		context = await browser.newContext({
 			bypassCSP: options.browserBypassCSP === undefined || options.browserBypassCSP
 		});
+		await setContextOptions(context, options);
+		page = await context.newPage();
 		await setPageOptions(page, options);
 		return await getPageData(page, options);
 	} finally {
@@ -67,6 +69,12 @@ function getBrowserOptions(options) {
 	return browserOptions;
 }
 
+async function setContextOptions(context, options) {
+	if (options.browserCookies && options.browserCookies.length) {
+		await context.addCookies(options.browserCookies);
+	}
+}
+
 async function setPageOptions(page, options) {
 	if (options.browserWidth && options.browserHeight) {
 		await page.setViewportSize({

cli/back-ends/puppeteer-firefox.js

@@ -88,6 +88,9 @@ async function setPageOptions(page, options) {
 			// ignored
 		}
 	}
+	if (options.browserCookies && options.browserCookies.length) {
+		await page.setCookie(...options.browserCookies);
+	}
 }
 
 async function getPageData(browser, page, options) {

cli/back-ends/puppeteer.js

@@ -96,6 +96,9 @@ async function setPageOptions(page, options) {
 		const { windowId } = await session.send("Browser.getWindowForTarget");
 		await session.send("Browser.setWindowBounds", { windowId, bounds: { windowState: "minimized" } });
 	}
+	if (options.browserCookies && options.browserCookies.length) {
+		await page.setCookie(...options.browserCookies);
+	}
 }
 
 async function getPageData(browser, page, options) {

cli/back-ends/webdriver-chromium.js

@@ -109,6 +109,16 @@ async function getPageData(driver, options) {
 		// await driver.sleep(3000);
 	}
 	await driver.get(options.url);
+	if (options.browserCookies) {
+		await Promise.all(options.browserCookies.map(cookie => {
+			if (cookie.expires) {
+				cookie.expiry = cookie.expires;
+				delete cookie.expires;
+			}
+			return driver.manage().addCookie(cookie);
+		}));
+		await driver.get(options.url);
+	}
 	await driver.executeScript(scripts);
 	if (options.browserWaitUntil != "domcontentloaded") {
 		let scriptPromise;

cli/back-ends/webdriver-gecko.js

@@ -98,10 +98,23 @@ async function getPageData(driver, options) {
 		await driver.sleep(3000);
 	}
 	await driver.get(options.url);
-	scripts = scripts.replace(/\n(this)\.([^ ]+) = (this)\.([^ ]+) \|\|/g, "\nwindow.$2 = window.$4 ||");
 	while (await driver.getCurrentUrl() == "about:blank") {
 		// do nothing
 	}
+	if (options.browserCookies) {
+		await Promise.all(options.browserCookies.map(cookie => {
+			if (cookie.expires) {
+				cookie.expiry = cookie.expires;
+			}
+			return driver.manage().addCookie(cookie);
+		}));
+		await driver.get("about:blank");
+		await driver.get(options.url);
+		while (await driver.getCurrentUrl() == "about:blank") {
+			// do nothing
+		}
+	}
+	scripts = scripts.replace(/\n(this)\.([^ ]+) = (this)\.([^ ]+) \|\|/g, "\nwindow.$2 = window.$4 ||");
 	await driver.executeScript(scripts);
 	if (options.browserWaitUntil != "domcontentloaded") {
 		let scriptPromise;