/* * Copyright 2010-2020 Gildas Lormeau * contact : gildas.lormeau gmail.com * * This file is part of SingleFile. * * The code in this file is free software: you can redistribute it and/or * modify it under the terms of the GNU Affero General Public License * (GNU AGPL) as published by the Free Software Foundation, either version 3 * of the License, or (at your option) any later version. * * The code in this file is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero * General Public License for more details. * * As additional permission under GNU AGPL version 3 section 7, you may * distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU * AGPL normally required by section 4, provided you include this license * notice and a URL through which recipients can access the Corresponding * Source. */ /* global singlefile, common, require, exports */ const puppeteer = require("puppeteer-core"); const scripts = require("./common/scripts.js"); const EXECUTION_CONTEXT_DESTROYED_ERROR = "Execution context was destroyed"; const NETWORK_IDLE_STATE = "networkidle0"; const NETWORK_STATES = ["networkidle0", "networkidle2", "load", "domcontentloaded"]; let browser; exports.initialize = async options => { browser = await puppeteer.launch(getBrowserOptions(options)); return browser; }; exports.getPageData = async (options, page) => { const privatePage = !page; try { if (privatePage) { page = await browser.newPage(); } await setPageOptions(page, options); return await getPageData(browser, page, options); } finally { if (privatePage) { await page.close(); } } }; exports.closeBrowser = () => { if (browser) { return browser.close(); } }; function getBrowserOptions(options = {}) { const browserOptions = {}; if (options.browserHeadless !== undefined) { browserOptions.headless = options.browserHeadless && !options.browserDebug; } browserOptions.args = options.browserArgs ? JSON.parse(options.browserArgs) : []; if (options.browserDisableWebSecurity === undefined || options.browserDisableWebSecurity) { browserOptions.args.push("--disable-web-security"); } browserOptions.args.push("--no-pings"); if (!options.browserStartMinimized && options.browserDebug) { browserOptions.args.push("--auto-open-devtools-for-tabs"); } if (options.browserWidth && options.browserHeight) { browserOptions.args.push("--window-size=" + options.browserWidth + "," + options.browserHeight); } browserOptions.executablePath = options.browserExecutablePath || "chrome"; if (options.userAgent) { browserOptions.args.push("--user-agent=" + options.userAgent); } return browserOptions; } async function setPageOptions(page, options) { if (options.browserWidth && options.browserHeight) { await page.setViewport({ width: options.browserWidth, height: options.browserHeight }); } if (options.browserBypassCSP === undefined || options.browserBypassCSP) { await page.setBypassCSP(true); } if (options.httpHeaders) { page.setExtraHTTPHeaders(options.httpHeaders); } if (options.browserStartMinimized) { const session = await page.target().createCDPSession(); const { windowId } = await session.send("Browser.getWindowForTarget"); await session.send("Browser.setWindowBounds", { windowId, bounds: { windowState: "minimized" } }); } if (options.browserCookies && options.browserCookies.length) { await page.setCookie(...options.browserCookies); } } async function getPageData(browser, page, options) { const injectedScript = await scripts.get(options); await page.evaluateOnNewDocument(injectedScript); if (options.browserDebug) { await page.waitForTimeout(3000); } try { await pageGoto(page, options); } catch (error) { if (options.browserWaitUntilFallback && error.name == "TimeoutError") { const browserWaitUntil = NETWORK_STATES[(NETWORK_STATES.indexOf(options.browserWaitUntil) + 1)]; if (browserWaitUntil) { options.browserWaitUntil = browserWaitUntil; return getPageData(browser, page, options); } else { throw error; } } else if (error.name != "TimeoutError") { throw error; } } try { if (options.browserWaitDelay) { await page.waitForTimeout(options.browserWaitDelay); } return await page.evaluate(async options => { const pageData = await singlefile.getPageData(options); if (options.includeInfobar) { await common.ui.content.infobar.includeScript(pageData); } return pageData; }, options); } catch (error) { if (error.message && error.message.includes(EXECUTION_CONTEXT_DESTROYED_ERROR)) { const pageData = await handleJSRedirect(browser, options); if (pageData) { return pageData; } else { throw error; } } else { throw error; } } } async function handleJSRedirect(browser, options) { const pages = await browser.pages(); const page = pages[1] || pages[0]; try { await pageGoto(page, options); } catch (error) { if (error.name != "TimeoutError") { throw error; } } const url = page.url(); if (url != options.url) { options.url = url; await browser.close(); return exports.getPageData(options); } } async function pageGoto(page, options) { const loadOptions = { timeout: options.browserLoadMaxTime || 0, waitUntil: options.browserWaitUntil || NETWORK_IDLE_STATE }; if (options.content) { await page.goto(options.url, { waitUntil: "domcontentloaded" }); await page.setContent(options.content, loadOptions); } else { await page.goto(options.url, loadOptions); } }