single-file-core.js 71 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683
  1. /*
  2. * Copyright 2018 Gildas Lormeau
  3. * contact : gildas.lormeau <at> gmail.com
  4. *
  5. * This file is part of SingleFile.
  6. *
  7. * The code in this file is free software: you can redistribute it and/or
  8. * modify it under the terms of the GNU Affero General Public License
  9. * (GNU AGPL) as published by the Free Software Foundation, either version 3
  10. * of the License, or (at your option) any later version.
  11. *
  12. * The code in this file is distributed in the hope that it will be useful,
  13. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
  15. * General Public License for more details.
  16. *
  17. * As additional permission under GNU AGPL version 3 section 7, you may
  18. * distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU
  19. * AGPL normally required by section 4, provided you include this license
  20. * notice and a URL through which recipients can access the Corresponding
  21. * Source.
  22. */
  23. this.SingleFileCore = this.SingleFileCore || (() => {
  24. const SELECTED_CONTENT_ATTRIBUTE_NAME = "data-single-file-selected-content";
  25. const SELECTED_CONTENT_ROOT_ATTRIBUTE_NAME = "data-single-file-selected-content-root";
  26. const DEBUG = false;
  27. let docUtil, cssTree, sessionId = 0;
  28. function getClass(...args) {
  29. [docUtil, cssTree] = args;
  30. return SingleFileClass;
  31. }
  32. class SingleFileClass {
  33. constructor(options) {
  34. this.options = options;
  35. if (options.sessionId === undefined) {
  36. options.sessionId = sessionId;
  37. sessionId++;
  38. }
  39. }
  40. async initialize() {
  41. this.processor = new PageProcessor(this.options, true);
  42. await this.processor.loadPage();
  43. await this.processor.initialize();
  44. }
  45. async run() {
  46. await this.processor.run();
  47. }
  48. async getPageData() {
  49. return this.processor.getPageData();
  50. }
  51. }
  52. SingleFileClass.SELECTED_CONTENT_ATTRIBUTE_NAME = SELECTED_CONTENT_ATTRIBUTE_NAME;
  53. SingleFileClass.SELECTED_CONTENT_ROOT_ATTRIBUTE_NAME = SELECTED_CONTENT_ROOT_ATTRIBUTE_NAME;
  54. // -------------
  55. // ProgressEvent
  56. // -------------
  57. const PAGE_LOADING = "page-loading";
  58. const PAGE_LOADED = "page-loaded";
  59. const RESOURCES_INITIALIZING = "resource-initializing";
  60. const RESOURCES_INITIALIZED = "resources-initialized";
  61. const RESOURCE_LOADED = "resource-loaded";
  62. const PAGE_ENDED = "page-ended";
  63. const STAGE_STARTED = "stage-started";
  64. const STAGE_ENDED = "stage-ended";
  65. const STAGE_TASK_STARTED = "stage-task-started";
  66. const STAGE_TASK_ENDED = "stage-task-ended";
  67. class ProgressEvent {
  68. constructor(type, detail) {
  69. return { type, detail, PAGE_LOADING, PAGE_LOADED, RESOURCES_INITIALIZING, RESOURCES_INITIALIZED, RESOURCE_LOADED, PAGE_ENDED, STAGE_STARTED, STAGE_ENDED, STAGE_TASK_STARTED, STAGE_TASK_ENDED };
  70. }
  71. }
  72. // -------------
  73. // PageProcessor
  74. // -------------
  75. const RESOLVE_URLS_STAGE = 0;
  76. const REPLACE_DATA_STAGE = 1;
  77. const REPLACE_DOCS_STAGE = 2;
  78. const POST_PROCESS_STAGE = 3;
  79. const STAGES = [{
  80. sequential: [
  81. { action: "preProcessPage" },
  82. { action: "insertShadowRootContents" },
  83. { action: "replaceStyleContents" },
  84. { option: "selected", action: "removeUnselectedElements" },
  85. { option: "removeVideoSrc", action: "insertVideoPosters" },
  86. { option: "removeFrames", action: "removeFrames" },
  87. { option: "removeImports", action: "removeImports" },
  88. { option: "removeScripts", action: "removeScripts" },
  89. { action: "removeDiscardedResources" },
  90. { action: "resetCharsetMeta" },
  91. { action: "setInputValues" },
  92. { option: "insertFaviconLink", action: "insertFaviconLink" },
  93. { action: "replaceCanvasElements" },
  94. { action: "insertFonts" },
  95. { option: "removeHiddenElements", action: "removeHiddenElements" },
  96. { action: "resolveHrefs" },
  97. { action: "resolveStyleAttributeURLs" }
  98. ],
  99. parallel: [
  100. { action: "resolveStylesheetURLs" },
  101. { option: "!removeFrames", action: "resolveFrameURLs" },
  102. { option: "!removeImports", action: "resolveHtmlImportURLs" }
  103. ]
  104. }, {
  105. sequential: [
  106. { option: "removeUnusedStyles", action: "removeUnusedStyles" },
  107. { option: "removeAlternativeMedias", action: "removeAlternativeMedias" },
  108. { option: "removeUnusedStyles", action: "removeUnusedFonts" }
  109. ],
  110. parallel: [
  111. { action: "processStylesheets" },
  112. { action: "processStyleAttributes" },
  113. { action: "processPageResources" },
  114. { option: "!removeScripts", action: "processScripts" }
  115. ]
  116. }, {
  117. sequential: [
  118. { option: "removeAlternativeImages", action: "removeAlternativeImages" },
  119. { option: "removeAlternativeFonts", action: "removeAlternativeFonts" }
  120. ],
  121. parallel: [
  122. { option: "!removeFrames", action: "processFrames" },
  123. { option: "!removeImports", action: "processHtmlImports" },
  124. ]
  125. }, {
  126. sequential: [
  127. { action: "replaceStylesheets" },
  128. { action: "replaceStyleAttributes" },
  129. { action: "insertVariables" },
  130. { option: "compressHTML", action: "compressHTML" }
  131. ]
  132. }];
  133. class PageProcessor {
  134. constructor(options, root) {
  135. this.root = root;
  136. this.options = options;
  137. this.options.url = this.options.url || this.options.doc.location.href;
  138. this.options.baseURI = this.options.doc && this.options.doc.baseURI;
  139. this.batchRequest = new BatchRequest();
  140. this.processor = new DOMProcessor(options, this.batchRequest);
  141. if (this.options.doc) {
  142. const docData = docUtil.preProcessDoc(this.options.doc, this.options.win, this.options);
  143. this.options.canvasData = docData.canvasData;
  144. this.options.fontsData = docData.fontsData;
  145. this.options.stylesheetContents = docData.stylesheetContents;
  146. this.options.imageData = docData.imageData;
  147. this.options.postersData = docData.postersData;
  148. this.options.usedFonts = docData.usedFonts;
  149. this.options.shadowRootContents = docData.shadowRootContents;
  150. }
  151. this.options.content = this.options.content || (this.options.doc ? docUtil.serialize(this.options.doc, false) : null);
  152. this.onprogress = options.onprogress || (() => { });
  153. }
  154. async loadPage() {
  155. this.onprogress(new ProgressEvent(PAGE_LOADING, { pageURL: this.options.url, frame: !this.root }));
  156. await this.processor.loadPage(this.options.content);
  157. this.onprogress(new ProgressEvent(PAGE_LOADED, { pageURL: this.options.url, frame: !this.root }));
  158. }
  159. async initialize() {
  160. this.onprogress(new ProgressEvent(RESOURCES_INITIALIZING, { pageURL: this.options.url }));
  161. await this.executeStage(RESOLVE_URLS_STAGE);
  162. this.pendingPromises = this.executeStage(REPLACE_DATA_STAGE);
  163. if (this.options.doc) {
  164. docUtil.postProcessDoc(this.options.doc, this.options);
  165. this.options.doc = null;
  166. this.options.win = null;
  167. }
  168. }
  169. async run() {
  170. if (this.root) {
  171. this.processor.initialize(this.batchRequest);
  172. this.onprogress(new ProgressEvent(RESOURCES_INITIALIZED, { pageURL: this.options.url, index: 0, max: this.processor.maxResources }));
  173. }
  174. await this.batchRequest.run(detail => {
  175. detail.pageURL = this.options.url;
  176. this.onprogress(new ProgressEvent(RESOURCE_LOADED, detail));
  177. }, this.options);
  178. await this.pendingPromises;
  179. await this.executeStage(REPLACE_DOCS_STAGE);
  180. await this.executeStage(POST_PROCESS_STAGE);
  181. await this.processor.end();
  182. }
  183. async getPageData() {
  184. if (this.root) {
  185. this.onprogress(new ProgressEvent(PAGE_ENDED, { pageURL: this.options.url }));
  186. }
  187. return this.processor.getPageData();
  188. }
  189. async executeStage(step) {
  190. if (DEBUG) {
  191. log("**** STARTED STAGE", step, "****");
  192. }
  193. const frame = !this.root;
  194. this.onprogress(new ProgressEvent(STAGE_STARTED, { pageURL: this.options.url, step, frame }));
  195. STAGES[step].sequential.forEach(task => {
  196. let startTime;
  197. if (DEBUG) {
  198. startTime = Date.now();
  199. log(" -- STARTED task =", task.action);
  200. }
  201. this.onprogress(new ProgressEvent(STAGE_TASK_STARTED, { pageURL: this.options.url, step, task: task.action, frame }));
  202. this.executeTask(task);
  203. this.onprogress(new ProgressEvent(STAGE_TASK_ENDED, { pageURL: this.options.url, step, task: task.action, frame }));
  204. if (DEBUG) {
  205. log(" -- ENDED task =", task.action, "delay =", Date.now() - startTime);
  206. }
  207. });
  208. let parallelTasksPromise;
  209. if (STAGES[step].parallel) {
  210. parallelTasksPromise = await Promise.all(STAGES[step].parallel.map(async task => {
  211. let startTime;
  212. if (DEBUG) {
  213. startTime = Date.now();
  214. log(" // STARTED task =", task.action);
  215. }
  216. this.onprogress(new ProgressEvent(STAGE_TASK_STARTED, { pageURL: this.options.url, step, task: task.action, frame }));
  217. await this.executeTask(task);
  218. this.onprogress(new ProgressEvent(STAGE_TASK_ENDED, { pageURL: this.options.url, step, task: task.action, frame }));
  219. if (DEBUG) {
  220. log(" // ENDED task =", task.action, "delay =", Date.now() - startTime);
  221. }
  222. }));
  223. } else {
  224. parallelTasksPromise = Promise.resolve();
  225. }
  226. this.onprogress(new ProgressEvent(STAGE_ENDED, { pageURL: this.options.url, step, frame }));
  227. if (DEBUG) {
  228. log("**** ENDED STAGE", step, "****");
  229. }
  230. return parallelTasksPromise;
  231. }
  232. executeTask(task) {
  233. if (!task.option || ((task.option.startsWith("!") && !this.options[task.option]) || this.options[task.option])) {
  234. return this.processor[task.action]();
  235. }
  236. }
  237. }
  238. // --------
  239. // BatchRequest
  240. // --------
  241. class BatchRequest {
  242. constructor() {
  243. this.requests = new Map();
  244. }
  245. async addURL(resourceURL, asDataURI = true) {
  246. return new Promise((resolve, reject) => {
  247. const requestKey = JSON.stringify([resourceURL, asDataURI]);
  248. const resourceRequests = this.requests.get(requestKey);
  249. if (resourceRequests) {
  250. resourceRequests.push({ resolve, reject });
  251. } else {
  252. this.requests.set(requestKey, [{ resolve, reject }]);
  253. }
  254. });
  255. }
  256. getMaxResources() {
  257. return Array.from(this.requests.keys()).length;
  258. }
  259. async run(onloadListener, options) {
  260. const resourceURLs = Array.from(this.requests.keys());
  261. let indexResource = 0;
  262. return Promise.all(resourceURLs.map(async requestKey => {
  263. const [resourceURL, asDataURI] = JSON.parse(requestKey);
  264. const resourceRequests = this.requests.get(requestKey);
  265. try {
  266. const content = await docUtil.getContent(resourceURL, { asDataURI, maxResourceSize: options.maxResourceSize, maxResourceSizeEnabled: options.maxResourceSizeEnabled });
  267. indexResource = indexResource + 1;
  268. onloadListener({ index: indexResource, url: resourceURL });
  269. resourceRequests.forEach(resourceRequest => resourceRequest.resolve({ content: content.data, indexResource, duplicate: Boolean(resourceRequests.length > 1) }));
  270. } catch (error) {
  271. indexResource = indexResource + 1;
  272. onloadListener({ index: indexResource, url: resourceURL });
  273. resourceRequests.forEach(resourceRequest => resourceRequest.reject(error));
  274. }
  275. this.requests.delete(requestKey);
  276. }));
  277. }
  278. }
  279. // ------------
  280. // DOMProcessor
  281. // ------------
  282. const EMPTY_DATA_URI = "data:base64,";
  283. const EMPTY_IMAGE = "data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==";
  284. const SCRIPT_TAG_FOUND = /<script/gi;
  285. const NOSCRIPT_TAG_FOUND = /<noscript/gi;
  286. class DOMProcessor {
  287. constructor(options, batchRequest) {
  288. this.options = options;
  289. this.stats = new Stats(options);
  290. this.baseURI = DomUtil.normalizeURL(options.baseURI || options.url);
  291. this.batchRequest = batchRequest;
  292. this.stylesheets = new Map();
  293. this.styles = new Map();
  294. this.cssVariables = new Map();
  295. }
  296. initialize() {
  297. this.maxResources = this.batchRequest.getMaxResources();
  298. if (!this.options.removeFrames) {
  299. this.options.framesData.forEach(frameData => this.maxResources += frameData.maxResources || 0);
  300. }
  301. this.stats.set("processed", "resources", this.maxResources);
  302. }
  303. async loadPage(pageContent) {
  304. let resourceURL;
  305. if (!pageContent || this.options.saveRawPage) {
  306. const content = await docUtil.getContent(this.baseURI, { asDataURI: false, maxResourceSize: this.options.maxResourceSize, maxResourceSizeEnabled: this.options.maxResourceSizeEnabled });
  307. pageContent = content.data;
  308. this.baseURI = this.options.url = resourceURL;
  309. }
  310. this.doc = docUtil.createDoc(pageContent, this.baseURI);
  311. this.onEventAttributeNames = docUtil.getOnEventAttributeNames(this.doc);
  312. }
  313. async getPageData() {
  314. docUtil.postProcessDoc(this.doc, this.options);
  315. const url = docUtil.parseURL(this.baseURI);
  316. if (this.options.insertSingleFileComment) {
  317. const infobarContent = (this.options.infobarContent || "").replace(/\\n/g, "\n").replace(/\\t/g, "\t");
  318. const commentNode = this.doc.createComment("\n Page saved with SingleFile" +
  319. " \n url: " + this.options.url +
  320. " \n saved date: " + new Date() +
  321. (infobarContent ? " \n info: " + infobarContent : "") + "\n");
  322. this.doc.documentElement.insertBefore(commentNode, this.doc.documentElement.firstChild);
  323. }
  324. let size;
  325. if (this.options.displayStats) {
  326. size = docUtil.getContentSize(this.doc.documentElement.outerHTML);
  327. }
  328. const content = docUtil.serialize(this.doc, this.options.compressHTML);
  329. if (this.options.displayStats) {
  330. const contentSize = docUtil.getContentSize(content);
  331. this.stats.set("processed", "HTML bytes", contentSize);
  332. this.stats.add("discarded", "HTML bytes", size - contentSize);
  333. }
  334. let filename = await DomProcessorHelper.evalTemplate(this.options.filenameTemplate, this.options, content);
  335. filename = filename.replace(/[~\\?%*:|"<>\x00-\x1f\x7F]+/g, "_"); // eslint-disable-line no-control-regex
  336. filename = filename.replace(/\.\.\//g, "").replace(/^\/+/, "").replace(/\/+/g, "/").replace(/\/$/, "");
  337. if (!this.options.backgroundSave) {
  338. filename = filename.replace(/\//g, "_");
  339. }
  340. if (filename.length > 192) {
  341. const extensionMatch = filename.match(/(\.[^.]{3,4})$/);
  342. const extension = extensionMatch && extensionMatch[0] && extensionMatch[0].length > 1 ? extensionMatch[0] : "";
  343. filename = filename.substring(0, 192 - extension.length) + "…" + extension;
  344. }
  345. if (!filename) {
  346. filename = "Unnamed page";
  347. }
  348. const matchTitle = this.baseURI.match(/([^/]*)\/?(\.html?.*)$/) || this.baseURI.match(/\/\/([^/]*)\/?$/);
  349. return {
  350. stats: this.stats.data,
  351. title: this.options.title || (this.baseURI && matchTitle ? matchTitle[1] : (url.hostname ? url.hostname : "")),
  352. filename,
  353. content
  354. };
  355. }
  356. removeUnselectedElements() {
  357. const rootElement = this.doc.querySelector("[" + SELECTED_CONTENT_ROOT_ATTRIBUTE_NAME + "]");
  358. if (rootElement) {
  359. DomProcessorHelper.isolateElements(rootElement);
  360. rootElement.removeAttribute(SELECTED_CONTENT_ROOT_ATTRIBUTE_NAME);
  361. rootElement.removeAttribute(SELECTED_CONTENT_ATTRIBUTE_NAME);
  362. }
  363. }
  364. setInputValues() {
  365. this.doc.querySelectorAll("input").forEach(input => {
  366. const value = input.getAttribute(docUtil.inputValueAttributeName(this.options.sessionId));
  367. input.setAttribute("value", value || "");
  368. });
  369. this.doc.querySelectorAll("input[type=radio], input[type=checkbox]").forEach(input => {
  370. const value = input.getAttribute(docUtil.inputValueAttributeName(this.options.sessionId));
  371. if (value == "true") {
  372. input.setAttribute("checked", "");
  373. }
  374. });
  375. this.doc.querySelectorAll("textarea").forEach(textarea => {
  376. const value = textarea.getAttribute(docUtil.inputValueAttributeName(this.options.sessionId));
  377. textarea.textContent = value || "";
  378. });
  379. this.doc.querySelectorAll("select").forEach(select => {
  380. select.querySelectorAll("option").forEach(option => {
  381. const selected = option.getAttribute(docUtil.inputValueAttributeName(this.options.sessionId)) != null;
  382. if (selected) {
  383. option.setAttribute("selected", "");
  384. }
  385. });
  386. });
  387. }
  388. removeDiscardedResources() {
  389. const objectElements = this.doc.querySelectorAll("applet, meta[http-equiv=refresh], object[data]:not([type=\"image/svg+xml\"]):not([type=\"image/svg-xml\"]):not([type=\"text/html\"]), embed[src]:not([src*=\".svg\"])");
  390. this.stats.set("discarded", "objects", objectElements.length);
  391. this.stats.set("processed", "objects", objectElements.length);
  392. objectElements.forEach(element => element.remove());
  393. const replacedAttributeValue = this.doc.querySelectorAll("link[rel~=preconnect], link[rel~=prerender], link[rel~=dns-prefetch], link[rel~=preload], link[rel~=prefetch]");
  394. replacedAttributeValue.forEach(element => {
  395. const relValue = element.getAttribute("rel").replace(/(preconnect|prerender|dns-prefetch|preload|prefetch)/g, "").trim();
  396. if (relValue.length) {
  397. element.setAttribute("rel", relValue);
  398. } else {
  399. element.remove();
  400. }
  401. });
  402. this.doc.querySelectorAll("meta[http-equiv=\"content-security-policy\"]").forEach(element => element.remove());
  403. if (this.options.compressHTML) {
  404. this.doc.querySelectorAll("input[type=hidden]").forEach(element => element.remove());
  405. }
  406. this.doc.querySelectorAll("a[ping]").forEach(element => element.removeAttribute("ping"));
  407. if (this.options.removeScripts) {
  408. this.onEventAttributeNames.forEach(attributeName => this.doc.querySelectorAll("[" + attributeName + "]").forEach(element => element.removeAttribute(attributeName)));
  409. this.doc.querySelectorAll("[href]").forEach(element => {
  410. if (element.href && element.href.match && element.href.match(/^\s*javascript:/)) {
  411. element.setAttribute("href", "");
  412. }
  413. });
  414. this.doc.querySelectorAll("[src]").forEach(element => {
  415. if (element.src && element.src.match(/^\s*javascript:/)) {
  416. element.removeAttribute("src");
  417. }
  418. });
  419. }
  420. const audioSourceElements = this.doc.querySelectorAll("audio[src], audio > source[src]");
  421. this.stats.set("processed", "audio sources", audioSourceElements.length);
  422. if (this.options.removeAudioSrc) {
  423. this.stats.set("discarded", "audio sources", audioSourceElements.length);
  424. audioSourceElements.forEach(element => {
  425. if (element.tagName == "SOURCE") {
  426. element.remove();
  427. } else {
  428. audioSourceElements.forEach(element => element.removeAttribute("src"));
  429. }
  430. });
  431. }
  432. const videoSourceElements = this.doc.querySelectorAll("video[src], video > source[src]");
  433. this.stats.set("processed", "video sources", videoSourceElements.length);
  434. if (this.options.removeVideoSrc) {
  435. this.stats.set("discarded", "video sources", videoSourceElements.length);
  436. videoSourceElements.forEach(element => {
  437. if (element.tagName == "SOURCE") {
  438. element.remove();
  439. } else {
  440. videoSourceElements.forEach(element => element.removeAttribute("src"));
  441. }
  442. });
  443. }
  444. }
  445. async end() {
  446. const metaCharset = this.doc.head.querySelector("meta[charset]");
  447. if (metaCharset) {
  448. this.doc.head.insertBefore(metaCharset, this.doc.head.firstChild);
  449. }
  450. this.doc.querySelectorAll("base").forEach(element => element.remove());
  451. if (this.doc.head.querySelectorAll("*").length == 1 && metaCharset && this.doc.body.childNodes.length == 0) {
  452. this.doc.head.querySelector("meta[charset]").remove();
  453. }
  454. const titleElement = this.doc.querySelector("title");
  455. const descriptionElement = this.doc.querySelector("meta[name=description]");
  456. const authorElement = this.doc.querySelector("meta[name=author]");
  457. const creatorElement = this.doc.querySelector("meta[name=creator]");
  458. const publisherElement = this.doc.querySelector("meta[name=publisher]");
  459. this.options.title = titleElement ? titleElement.textContent.trim() : "";
  460. this.options.infobarContent = await DomProcessorHelper.evalTemplate(this.options.infobarTemplate, this.options, null, true);
  461. this.options.info = {
  462. description: descriptionElement ? descriptionElement.content.trim() : "",
  463. lang: this.doc.documentElement.lang,
  464. author: authorElement ? authorElement.content.trim() : "",
  465. creator: creatorElement ? creatorElement.content.trim() : "",
  466. publisher: publisherElement ? publisherElement.content.trim() : ""
  467. };
  468. }
  469. preProcessPage() {
  470. this.doc.querySelectorAll("singlefile-infobar, singlefile-mask, singlefile-logs-window").forEach(element => element.remove());
  471. if (this.options.win) {
  472. this.doc.body.querySelectorAll(":not(svg) title, meta, link[href][rel*=\"icon\"]").forEach(element => element instanceof this.options.win.HTMLElement && this.doc.head.appendChild(element));
  473. }
  474. if (this.options.imageData) {
  475. const dataAttributeName = docUtil.imagesAttributeName(this.options.sessionId);
  476. this.doc.querySelectorAll("img").forEach(imgElement => {
  477. const imgData = this.options.imageData[Number(imgElement.getAttribute(dataAttributeName))];
  478. if (this.options.removeHiddenElements && imgData.size && !imgData.size.pxWidth && !imgData.size.pxHeight) {
  479. imgElement.setAttribute("src", EMPTY_IMAGE);
  480. } else if (imgData.currentSrc) {
  481. imgElement.setAttribute("src", imgData.currentSrc);
  482. }
  483. });
  484. if (this.options.lazyLoadImages) {
  485. this.doc.querySelectorAll("img[data-src]").forEach(imgElement => {
  486. const imgData = this.options.imageData[Number(imgElement.getAttribute(dataAttributeName))];
  487. if ((!imgElement.getAttribute("src") || imgElement.getAttribute("src") == EMPTY_IMAGE) && imgElement.getAttribute("data-src")) {
  488. imgData.src = imgElement.dataset.src;
  489. imgElement.setAttribute("src", imgElement.dataset.src);
  490. imgElement.removeAttribute("data-src");
  491. }
  492. });
  493. this.doc.querySelectorAll("img[data-srcset]").forEach(imgElement => {
  494. if (!imgElement.getAttribute("srcset") && imgElement.getAttribute("data-srcset")) {
  495. imgElement.setAttribute("srcset", imgElement.dataset.srcset);
  496. imgElement.removeAttribute("data-srcset");
  497. }
  498. });
  499. }
  500. }
  501. }
  502. removeScripts() {
  503. const scriptElements = this.doc.querySelectorAll("script:not([type=\"application/ld+json\"])");
  504. this.stats.set("discarded", "scripts", scriptElements.length);
  505. this.stats.set("processed", "scripts", scriptElements.length);
  506. scriptElements.forEach(element => element.remove());
  507. }
  508. removeFrames() {
  509. const frameElements = this.doc.querySelectorAll("iframe, frame, object[type=\"text/html\"][data]");
  510. this.stats.set("discarded", "frames", frameElements.length);
  511. this.stats.set("processed", "frames", frameElements.length);
  512. this.doc.querySelectorAll("iframe, frame, object[type=\"text/html\"][data]").forEach(element => element.remove());
  513. }
  514. removeImports() {
  515. const importElements = this.doc.querySelectorAll("link[rel=import]");
  516. this.stats.set("discarded", "HTML imports", importElements.length);
  517. this.stats.set("processed", "HTML imports", importElements.length);
  518. importElements.forEach(element => element.remove());
  519. }
  520. resetCharsetMeta() {
  521. let charset;
  522. this.doc.querySelectorAll("meta[charset], meta[http-equiv=\"content-type\"]").forEach(element => {
  523. const charsetDeclaration = element.content.split(";")[1];
  524. if (charsetDeclaration && !charset) {
  525. charset = charsetDeclaration.split("=")[1];
  526. if (charset) {
  527. this.charset = charset.trim().toLowerCase();
  528. }
  529. }
  530. element.remove();
  531. });
  532. const metaElement = this.doc.createElement("meta");
  533. metaElement.setAttribute("charset", "utf-8");
  534. if (this.doc.head.firstChild) {
  535. this.doc.head.insertBefore(metaElement, this.doc.head.firstChild);
  536. } else {
  537. this.doc.head.appendChild(metaElement);
  538. }
  539. }
  540. insertFaviconLink() {
  541. let faviconElement = this.doc.querySelector("link[href][rel=\"icon\"]");
  542. if (!faviconElement) {
  543. faviconElement = this.doc.querySelector("link[href][rel=\"shortcut icon\"]");
  544. }
  545. if (!faviconElement) {
  546. faviconElement = this.doc.createElement("link");
  547. faviconElement.setAttribute("type", "image/x-icon");
  548. faviconElement.setAttribute("rel", "shortcut icon");
  549. faviconElement.setAttribute("href", "/favicon.ico");
  550. }
  551. this.doc.head.appendChild(faviconElement);
  552. }
  553. resolveHrefs() {
  554. this.doc.querySelectorAll("[href]").forEach(element => {
  555. const href = element.getAttribute("href");
  556. if (!DomUtil.testIgnoredPath(href) && DomUtil.testValidPath(href, this.baseURI, this.options.url)) {
  557. const normalizedHref = DomUtil.normalizeURL(href);
  558. if (element.tagName == "LINK") {
  559. try {
  560. element.setAttribute("href", docUtil.resolveURL(href, this.options.baseURI || this.options.url));
  561. } catch (error) {
  562. if (element.tagName == "LINK") {
  563. element.setAttribute("href", EMPTY_URL);
  564. }
  565. }
  566. } else if (normalizedHref == href) {
  567. try {
  568. element.setAttribute("href", docUtil.resolveURL(normalizedHref, this.options.baseURI || this.options.url));
  569. } catch (error) {
  570. // ignored
  571. }
  572. }
  573. }
  574. });
  575. }
  576. removeUnusedStyles() {
  577. if (!this.mediaAllInfo) {
  578. this.mediaAllInfo = docUtil.getMediaAllInfo(this.doc, this.stylesheets, this.styles);
  579. }
  580. const stats = docUtil.minifyCSSRules(this.stylesheets, this.styles, this.mediaAllInfo);
  581. this.stats.set("processed", "CSS rules", stats.processed);
  582. this.stats.set("discarded", "CSS rules", stats.discarded);
  583. }
  584. removeUnusedFonts() {
  585. docUtil.removeUnusedFonts(this.doc, this.stylesheets, this.styles, this.options);
  586. }
  587. removeAlternativeFonts() {
  588. docUtil.removeAlternativeFonts(this.doc, this.stylesheets);
  589. }
  590. removeAlternativeImages() {
  591. docUtil.removeAlternativeImages(this.doc);
  592. }
  593. removeHiddenElements() {
  594. const hiddenElements = this.doc.querySelectorAll("[" + docUtil.removedContentAttributeName(this.options.sessionId) + "]");
  595. this.stats.set("discarded", "hidden elements", hiddenElements.length);
  596. this.stats.set("processed", "hidden elements", hiddenElements.length);
  597. hiddenElements.forEach(element => element.remove());
  598. }
  599. compressHTML() {
  600. let size;
  601. if (this.options.displayStats) {
  602. size = docUtil.getContentSize(this.doc.documentElement.outerHTML);
  603. }
  604. docUtil.minifyHTML(this.doc, { preservedSpaceAttributeName: docUtil.preservedSpaceAttributeName(this.options.sessionId) });
  605. if (this.options.displayStats) {
  606. this.stats.add("discarded", "HTML bytes", size - docUtil.getContentSize(this.doc.documentElement.outerHTML));
  607. }
  608. }
  609. removeAlternativeMedias() {
  610. const stats = docUtil.minifyMedias(this.stylesheets);
  611. this.stats.set("processed", "medias", stats.processed);
  612. this.stats.set("discarded", "medias", stats.discarded);
  613. }
  614. replaceCanvasElements() {
  615. if (this.options.canvasData) {
  616. this.doc.querySelectorAll("canvas").forEach((canvasElement, indexCanvasElement) => {
  617. const canvasData = this.options.canvasData[indexCanvasElement];
  618. if (canvasData) {
  619. DomProcessorHelper.setBackgroundImage(canvasElement, "url(" + canvasData.dataURI + ")");
  620. this.stats.add("processed", "canvas", 1);
  621. }
  622. });
  623. }
  624. }
  625. insertFonts() {
  626. if (this.options.fontsData && this.options.fontsData.length) {
  627. let stylesheetContent = "";
  628. this.options.fontsData.forEach(fontStyles => {
  629. if (fontStyles["font-family"] && fontStyles.src) {
  630. stylesheetContent += "@font-face{";
  631. let stylesContent = "";
  632. Object.keys(fontStyles).forEach(fontStyle => {
  633. if (stylesContent) {
  634. stylesContent += ";";
  635. }
  636. stylesContent += fontStyle + ":" + fontStyles[fontStyle];
  637. });
  638. stylesheetContent += stylesContent + "}";
  639. }
  640. });
  641. if (stylesheetContent) {
  642. const styleElement = this.doc.createElement("style");
  643. styleElement.textContent = stylesheetContent;
  644. const existingStyleElement = this.doc.querySelector("style");
  645. if (existingStyleElement) {
  646. existingStyleElement.parentElement.insertBefore(styleElement, existingStyleElement);
  647. } else {
  648. this.doc.head.insertBefore(styleElement, this.doc.head.firstChild);
  649. }
  650. }
  651. }
  652. }
  653. replaceStyleContents() {
  654. if (this.options.stylesheetContents) {
  655. this.doc.querySelectorAll("style").forEach((styleElement, styleIndex) => {
  656. if (this.options.stylesheetContents[styleIndex]) {
  657. styleElement.textContent = this.options.stylesheetContents[styleIndex];
  658. }
  659. });
  660. }
  661. }
  662. insertShadowRootContents() {
  663. if (this.options.shadowRootContents) {
  664. this.doc.querySelectorAll("[" + docUtil.shadowRootAttributeName(this.options.sessionId) + "]").forEach((element, elementIndex) => {
  665. const DOMParser = docUtil.getParser();
  666. const elementInfo = this.options.shadowRootContents[elementIndex];
  667. if (DOMParser && elementInfo) {
  668. const frameElement = this.doc.createElement("iframe");
  669. frameElement.setAttribute("style", "all:initial!important;border:0!important;width:100%!important;height:" + elementInfo.height + "px!important");
  670. const windowId = "shadow-" + this.options.framesData.length;
  671. frameElement.setAttribute(docUtil.windowIdAttributeName(this.options.sessionId), windowId);
  672. this.options.framesData.push({ windowId, content: elementInfo.content, baseURI: this.baseURI });
  673. element.appendChild(frameElement);
  674. }
  675. });
  676. }
  677. }
  678. insertVideoPosters() {
  679. if (this.options.postersData) {
  680. this.doc.querySelectorAll("video[src], video > source[src]").forEach((element, videoIndex) => {
  681. let videoElement;
  682. if (element.tagName == "VIDEO") {
  683. videoElement = element;
  684. } else {
  685. videoElement = element.parentElement;
  686. }
  687. if (!videoElement.poster && this.options.postersData[videoIndex]) {
  688. videoElement.setAttribute("poster", this.options.postersData[videoIndex]);
  689. }
  690. });
  691. }
  692. }
  693. async processPageResources() {
  694. const processAttributeArgs = [
  695. ["link[href][rel*=\"icon\"]", "href", "data:", false, true],
  696. ["object[type=\"image/svg+xml\"], object[type=\"image/svg-xml\"]", "data", PREFIX_DATA_URI_IMAGE_SVG],
  697. ["img[src], input[src][type=image]", "src", PREFIX_DATA_URI_IMAGE, true],
  698. ["embed[src*=\".svg\"]", "src", PREFIX_DATA_URI_IMAGE_SVG],
  699. ["video[poster]", "poster", PREFIX_DATA_URI_IMAGE],
  700. ["*[background]", "background", PREFIX_DATA_URI_IMAGE],
  701. ["image", "xlink:href", PREFIX_DATA_URI_IMAGE]
  702. ];
  703. let resourcePromises = processAttributeArgs.map(([selector, attributeName, prefixDataURI, processDuplicates, removeElementIfMissing]) =>
  704. DomProcessorHelper.processAttribute(this.doc, this.doc.querySelectorAll(selector), attributeName, prefixDataURI, this.baseURI, this.options, this.cssVariables, this.styles, this.batchRequest, processDuplicates, removeElementIfMissing)
  705. );
  706. resourcePromises = resourcePromises.concat([
  707. DomProcessorHelper.processXLinks(this.doc.querySelectorAll("use"), this.baseURI, this.options, this.batchRequest),
  708. DomProcessorHelper.processSrcset(this.doc.querySelectorAll("img[srcset], source[srcset]"), "srcset", PREFIX_DATA_URI_IMAGE, this.baseURI, this.options, this.batchRequest)
  709. ]);
  710. if (!this.options.removeAudioSrc) {
  711. resourcePromises.push(DomProcessorHelper.processAttribute(this.doc, this.doc.querySelectorAll("audio[src], audio > source[src]"), "src", PREFIX_DATA_URI_AUDIO, this.baseURI, this.options, this.cssVariables, this.styles, this.batchRequest));
  712. }
  713. if (!this.options.removeVideoSrc) {
  714. resourcePromises.push(DomProcessorHelper.processAttribute(this.doc, this.doc.querySelectorAll("video[src], video > source[src]"), "src", PREFIX_DATA_URI_VIDEO, this.baseURI, this.options, this.cssVariables, this.styles, this.batchRequest));
  715. }
  716. await Promise.all(resourcePromises);
  717. DomProcessorHelper.processShortcutIcons(this.doc);
  718. }
  719. async resolveStylesheetURLs() {
  720. await Promise.all(Array.from(this.doc.querySelectorAll("style, link[rel*=stylesheet]"))
  721. .map(async element => {
  722. let mediaText;
  723. if (element.media) {
  724. mediaText = element.media;
  725. }
  726. this.stylesheets.set(element, { mediaText });
  727. const options = { maxResourceSize: this.options.maxResourceSize, maxResourceSizeEnabled: this.options.maxResourceSizeEnabled, url: this.options.url, charset: this.charset, compressCSS: this.options.compressCSS };
  728. const isLinkTag = element.tagName.toLowerCase() == "link";
  729. if (isLinkTag && element.rel.includes("alternate") && element.title) {
  730. element.remove();
  731. } else {
  732. let stylesheetContent;
  733. if (isLinkTag) {
  734. if (element.charset) {
  735. options.charset = element.charset;
  736. }
  737. stylesheetContent = await DomProcessorHelper.resolveLinkStylesheetURLs(element.href, this.baseURI, options);
  738. } else {
  739. stylesheetContent = await DomProcessorHelper.resolveImportURLs(element.textContent, this.baseURI, options);
  740. }
  741. let stylesheet;
  742. try {
  743. stylesheet = cssTree.parse(stylesheetContent);
  744. } catch (error) {
  745. // ignored
  746. }
  747. if (stylesheet && stylesheet.children) {
  748. if (this.options.compressCSS) {
  749. const removedRules = [];
  750. for (let cssRule = stylesheet.children.head; cssRule; cssRule = cssRule.next) {
  751. if (cssRule.data.type == "Raw" && cssRule.data.value && cssRule.data.value.trim().startsWith("//")) {
  752. removedRules.push(cssRule);
  753. }
  754. }
  755. removedRules.forEach(cssRule => stylesheet.children.remove(cssRule));
  756. }
  757. this.stylesheets.get(element).stylesheet = stylesheet;
  758. }
  759. }
  760. }));
  761. }
  762. async processStylesheets() {
  763. await Promise.all(Array.from(this.stylesheets).map(([, stylesheetInfo]) =>
  764. DomProcessorHelper.processStylesheet(stylesheetInfo.stylesheet.children, this.baseURI, this.options, this.cssVariables, this.batchRequest)
  765. ));
  766. }
  767. replaceStylesheets() {
  768. this.doc.querySelectorAll("style").forEach(styleElement => {
  769. const stylesheetInfo = this.stylesheets.get(styleElement);
  770. if (stylesheetInfo) {
  771. let stylesheetContent = cssTree.generate(stylesheetInfo.stylesheet);
  772. styleElement.textContent = stylesheetContent;
  773. if (stylesheetInfo.mediaText) {
  774. styleElement.media = stylesheetInfo.mediaText;
  775. }
  776. } else {
  777. styleElement.remove();
  778. }
  779. });
  780. this.doc.querySelectorAll("link[rel*=stylesheet]").forEach(linkElement => {
  781. const stylesheetInfo = this.stylesheets.get(linkElement);
  782. if (stylesheetInfo) {
  783. const styleElement = this.doc.createElement("style");
  784. if (stylesheetInfo.mediaText) {
  785. styleElement.media = stylesheetInfo.mediaText;
  786. }
  787. let stylesheetContent = cssTree.generate(stylesheetInfo.stylesheet);
  788. styleElement.textContent = stylesheetContent;
  789. linkElement.parentElement.replaceChild(styleElement, linkElement);
  790. } else {
  791. linkElement.remove();
  792. }
  793. });
  794. }
  795. insertVariables() {
  796. if (this.cssVariables.size) {
  797. const styleElement = this.doc.createElement("style");
  798. const firstStyleElement = this.doc.head.querySelector("style");
  799. if (firstStyleElement) {
  800. this.doc.head.insertBefore(styleElement, firstStyleElement);
  801. } else {
  802. this.doc.head.appendChild(styleElement);
  803. }
  804. let stylesheetContent = "";
  805. this.cssVariables.forEach((content, indexResource) => {
  806. if (stylesheetContent) {
  807. stylesheetContent += ";";
  808. }
  809. stylesheetContent += `${SINGLE_FILE_VARIABLE_NAME_PREFIX + indexResource}:url("${content}")`;
  810. });
  811. styleElement.textContent = ":root{" + stylesheetContent + "}";
  812. }
  813. }
  814. async processScripts() {
  815. await Promise.all(Array.from(this.doc.querySelectorAll("script[src]")).map(async scriptElement => {
  816. if (scriptElement.src) {
  817. this.stats.add("processed", "scripts", 1);
  818. const content = await docUtil.getContent(scriptElement.src, { asDataURI: false, maxResourceSize: this.options.maxResourceSize, maxResourceSizeEnabled: this.options.maxResourceSizeEnabled });
  819. scriptElement.textContent = content.data.replace(/<\//gi, "<\\/").replace(/\/>/gi, "\\/>");
  820. }
  821. scriptElement.removeAttribute("src");
  822. }));
  823. }
  824. async resolveFrameURLs() {
  825. if (this.options.framesData) {
  826. const frameElements = Array.from(this.doc.querySelectorAll("iframe, frame, object[type=\"text/html\"][data]"));
  827. await Promise.all(frameElements.map(async frameElement => {
  828. DomProcessorHelper.setFrameEmptySrc(frameElement);
  829. const frameWindowId = frameElement.getAttribute(docUtil.windowIdAttributeName(this.options.sessionId));
  830. if (frameWindowId) {
  831. const frameData = this.options.framesData.find(frame => frame.windowId == frameWindowId);
  832. if (frameData) {
  833. await initializeProcessor(frameData, frameElement, frameWindowId, this.batchRequest, Object.create(this.options));
  834. }
  835. }
  836. }));
  837. }
  838. async function initializeProcessor(frameData, frameElement, frameWindowId, batchRequest, options) {
  839. options.insertSingleFileComment = false;
  840. options.insertFaviconLink = false;
  841. options.doc = null;
  842. options.win = null;
  843. options.url = frameData.baseURI;
  844. options.windowId = frameWindowId;
  845. if (frameData.content) {
  846. options.content = frameData.content;
  847. options.canvasData = frameData.canvasData;
  848. options.stylesheetContents = frameData.stylesheetContents;
  849. options.fontsData = frameData.fontsData;
  850. options.imageData = frameData.imageData;
  851. options.usedFonts = frameData.usedFonts;
  852. frameData.processor = new PageProcessor(options);
  853. frameData.frameElement = frameElement;
  854. await frameData.processor.loadPage();
  855. await frameData.processor.initialize();
  856. frameData.maxResources = batchRequest.getMaxResources();
  857. }
  858. }
  859. }
  860. async processFrames() {
  861. if (this.options.framesData) {
  862. const frameElements = Array.from(this.doc.querySelectorAll("iframe, frame, object[type=\"text/html\"][data]"));
  863. await Promise.all(frameElements.map(async frameElement => {
  864. const frameWindowId = frameElement.getAttribute(docUtil.windowIdAttributeName(this.options.sessionId));
  865. if (frameWindowId) {
  866. const frameData = this.options.framesData.find(frame => frame.windowId == frameWindowId);
  867. if (frameData) {
  868. if (frameData.processor) {
  869. this.stats.add("processed", "frames", 1);
  870. await frameData.processor.run();
  871. const pageData = await frameData.processor.getPageData();
  872. frameElement.removeAttribute(docUtil.windowIdAttributeName(this.options.sessionId));
  873. if (pageData.content.match(NOSCRIPT_TAG_FOUND) || pageData.content.match(SCRIPT_TAG_FOUND)) {
  874. frameElement.setAttribute("sandbox", "allow-scripts allow-same-origin");
  875. } else {
  876. frameElement.setAttribute("sandbox", "");
  877. }
  878. DomProcessorHelper.setFrameContent(frameElement, pageData.content);
  879. this.stats.addAll(pageData);
  880. } else {
  881. this.stats.add("discarded", "frames", 1);
  882. }
  883. }
  884. }
  885. }));
  886. }
  887. }
  888. async resolveHtmlImportURLs() {
  889. const linkElements = Array.from(this.doc.querySelectorAll("link[rel=import][href]"));
  890. if (!this.relImportProcessors) {
  891. this.relImportProcessors = new Map();
  892. }
  893. await Promise.all(linkElements.map(async linkElement => {
  894. const resourceURL = linkElement.href;
  895. linkElement.removeAttribute("href");
  896. const options = Object.create(this.options);
  897. options.insertSingleFileComment = false;
  898. options.insertFaviconLink = false;
  899. options.doc = null;
  900. options.win = null;
  901. options.url = resourceURL;
  902. if (!DomUtil.testIgnoredPath(resourceURL) && DomUtil.testValidPath(resourceURL, this.baseURI, this.options.url)) {
  903. const processor = new PageProcessor(options);
  904. this.relImportProcessors.set(linkElement, processor);
  905. await processor.loadPage();
  906. return processor.initialize();
  907. }
  908. }));
  909. }
  910. async processHtmlImports() {
  911. const linkElements = Array.from(this.doc.querySelectorAll("link[rel=import][href]"));
  912. await Promise.all(linkElements.map(async linkElement => {
  913. const processor = this.relImportProcessors.get(linkElement);
  914. if (processor) {
  915. this.stats.add("processed", "HTML imports", 1);
  916. this.relImportProcessors.delete(linkElement);
  917. const pageData = await processor.getPageData();
  918. linkElement.setAttribute("href", "data:text/html," + pageData.content);
  919. this.stats.addAll(pageData);
  920. } else {
  921. this.stats.add("discarded", "HTML imports", 1);
  922. }
  923. }));
  924. }
  925. resolveStyleAttributeURLs() {
  926. this.doc.querySelectorAll("[style]").forEach(element => {
  927. let styleContent = element.getAttribute("style");
  928. if (this.options.compressCSS) {
  929. styleContent = docUtil.compressCSS(styleContent);
  930. }
  931. styleContent = DomProcessorHelper.resolveStylesheetURLs(styleContent, this.baseURI, this.options);
  932. const declarationList = cssTree.parse(styleContent, { context: "declarationList" });
  933. this.styles.set(element, declarationList);
  934. });
  935. }
  936. async processStyleAttributes() {
  937. return Promise.all(Array.from(this.styles).map(([, declarationList]) =>
  938. DomProcessorHelper.processStyle(declarationList.children.toArray(), this.baseURI, this.options, this.cssVariables, this.batchRequest)
  939. ));
  940. }
  941. replaceStyleAttributes() {
  942. this.doc.querySelectorAll("[style]").forEach(async element => {
  943. const declarations = this.styles.get(element);
  944. if (declarations) {
  945. let styleContent = cssTree.generate(declarations);
  946. element.setAttribute("style", styleContent);
  947. } else {
  948. element.setAttribute("style", "");
  949. }
  950. });
  951. }
  952. }
  953. // ---------
  954. // DomHelper
  955. // ---------
  956. const REGEXP_AMP = /&/g;
  957. const REGEXP_NBSP = /\u00a0/g;
  958. const REGEXP_START_TAG = /</g;
  959. const REGEXP_END_TAG = />/g;
  960. const REGEXP_URL_HASH = /(#.+?)$/;
  961. const PREFIX_DATA_URI_IMAGE = "data:image/";
  962. const PREFIX_DATA_URI_AUDIO = "data:audio/";
  963. const PREFIX_DATA_URI_VIDEO = "data:video/";
  964. const PREFIX_DATA_URI_IMAGE_SVG = "data:image/svg+xml";
  965. const PREFIX_DATA_URI_NO_MIMETYPE = "data:;";
  966. const PREFIX_DATA_URI_OCTET_STREAM = /^data:(application|binary)\/octet-stream/;
  967. const PREFIX_DATA_URI_NULL_STREAM = /^data:null;/;
  968. const PREFIX_DATA_URI_VND = "data:application/vnd.";
  969. const SINGLE_FILE_VARIABLE_NAME_PREFIX = "--sf-img-";
  970. class DomProcessorHelper {
  971. static async evalTemplate(template, options, content, dontReplaceSlash) {
  972. const date = new Date();
  973. const url = docUtil.parseURL(options.url);
  974. template = await DomUtil.evalTemplateVariable(template, "page-title", () => options.title || "No title", dontReplaceSlash);
  975. template = await DomUtil.evalTemplateVariable(template, "page-language", () => options.info.lang || "No language", dontReplaceSlash);
  976. template = await DomUtil.evalTemplateVariable(template, "page-description", () => options.info.description || "No description", dontReplaceSlash);
  977. template = await DomUtil.evalTemplateVariable(template, "page-author", () => options.info.author || "No author", dontReplaceSlash);
  978. template = await DomUtil.evalTemplateVariable(template, "page-creator", () => options.info.creator || "No creator", dontReplaceSlash);
  979. template = await DomUtil.evalTemplateVariable(template, "page-publisher", () => options.info.publisher || "No publisher", dontReplaceSlash);
  980. template = await DomUtil.evalTemplateVariable(template, "datetime-iso", () => date.toISOString(), dontReplaceSlash);
  981. template = await DomUtil.evalTemplateVariable(template, "date-iso", () => date.toISOString().split("T")[0], dontReplaceSlash);
  982. template = await DomUtil.evalTemplateVariable(template, "time-iso", () => date.toISOString().split("T")[1].split("Z")[0], dontReplaceSlash);
  983. template = await DomUtil.evalTemplateVariable(template, "date-locale", () => date.toLocaleDateString(), dontReplaceSlash);
  984. template = await DomUtil.evalTemplateVariable(template, "time-locale", () => date.toLocaleTimeString(), dontReplaceSlash);
  985. template = await DomUtil.evalTemplateVariable(template, "day-locale", () => String(date.getDate()).padStart(2, "0"), dontReplaceSlash);
  986. template = await DomUtil.evalTemplateVariable(template, "month-locale", () => String(date.getMonth() + 1).padStart(2, "0"), dontReplaceSlash);
  987. template = await DomUtil.evalTemplateVariable(template, "year-locale", () => String(date.getFullYear()), dontReplaceSlash);
  988. template = await DomUtil.evalTemplateVariable(template, "datetime-locale", () => date.toLocaleString(), dontReplaceSlash);
  989. template = await DomUtil.evalTemplateVariable(template, "datetime-utc", () => date.toUTCString(), dontReplaceSlash);
  990. template = await DomUtil.evalTemplateVariable(template, "day-utc", () => String(date.getUTCDate()).padStart(2, "0"), dontReplaceSlash);
  991. template = await DomUtil.evalTemplateVariable(template, "month-utc", () => String(date.getUTCMonth() + 1).padStart(2, "0"), dontReplaceSlash);
  992. template = await DomUtil.evalTemplateVariable(template, "year-utc", () => String(date.getUTCFullYear()), dontReplaceSlash);
  993. template = await DomUtil.evalTemplateVariable(template, "hours-locale", () => String(date.getHours()).padStart(2, "0"), dontReplaceSlash);
  994. template = await DomUtil.evalTemplateVariable(template, "minutes-locale", () => String(date.getMinutes()).padStart(2, "0"), dontReplaceSlash);
  995. template = await DomUtil.evalTemplateVariable(template, "seconds-locale", () => String(date.getSeconds()).padStart(2, "0"), dontReplaceSlash);
  996. template = await DomUtil.evalTemplateVariable(template, "hours-utc", () => String(date.getUTCHours()).padStart(2, "0"), dontReplaceSlash);
  997. template = await DomUtil.evalTemplateVariable(template, "minutes-utc", () => String(date.getUTCMinutes()).padStart(2, "0"), dontReplaceSlash);
  998. template = await DomUtil.evalTemplateVariable(template, "seconds-utc", () => String(date.getUTCSeconds()).padStart(2, "0"), dontReplaceSlash);
  999. template = await DomUtil.evalTemplateVariable(template, "url-hash", () => url.hash.substring(1), dontReplaceSlash);
  1000. template = await DomUtil.evalTemplateVariable(template, "url-host", () => url.host.replace(/\/$/, ""), dontReplaceSlash);
  1001. template = await DomUtil.evalTemplateVariable(template, "url-hostname", () => url.hostname.replace(/\/$/, ""), dontReplaceSlash);
  1002. template = await DomUtil.evalTemplateVariable(template, "url-href", () => url.href, dontReplaceSlash);
  1003. template = await DomUtil.evalTemplateVariable(template, "url-password", () => url.password, dontReplaceSlash);
  1004. template = await DomUtil.evalTemplateVariable(template, "url-pathname", () => url.pathname.replace(/^\//, "").replace(/\/$/, ""), dontReplaceSlash === undefined ? true : dontReplaceSlash);
  1005. template = await DomUtil.evalTemplateVariable(template, "url-port", () => url.port, dontReplaceSlash);
  1006. template = await DomUtil.evalTemplateVariable(template, "url-protocol", () => url.protocol, dontReplaceSlash);
  1007. template = await DomUtil.evalTemplateVariable(template, "url-search", () => url.search.substring(1), dontReplaceSlash);
  1008. template = await DomUtil.evalTemplateVariable(template, "url-username", () => url.username, dontReplaceSlash);
  1009. template = await DomUtil.evalTemplateVariable(template, "tab-id", () => String(options.tabId || "No tab id"), dontReplaceSlash);
  1010. template = await DomUtil.evalTemplateVariable(template, "url-last-segment", () => DomUtil.getLastSegment(url), dontReplaceSlash);
  1011. if (content) {
  1012. template = await DomUtil.evalTemplateVariable(template, "digest-sha-256", async () => docUtil.digest("SHA-256", content), dontReplaceSlash);
  1013. template = await DomUtil.evalTemplateVariable(template, "digest-sha-384", async () => docUtil.digest("SHA-384", content), dontReplaceSlash);
  1014. template = await DomUtil.evalTemplateVariable(template, "digest-sha-512", async () => docUtil.digest("SHA-512", content), dontReplaceSlash);
  1015. }
  1016. return template;
  1017. }
  1018. static setBackgroundImage(element, url, style) {
  1019. element.style.setProperty("background-blend-mode", "normal", "important");
  1020. element.style.setProperty("background-clip", "content-box", "important");
  1021. element.style.setProperty("background-position", style && style["background-position"] ? style["background-position"] : "center", "important");
  1022. element.style.setProperty("background-color", style && style["background-color"] ? style["background-color"] : "transparent", "important");
  1023. element.style.setProperty("background-image", url, "important");
  1024. element.style.setProperty("background-size", style && style["background-size"] ? style["background-size"] : "100% 100%", "important");
  1025. element.style.setProperty("background-origin", "content-box", "important");
  1026. element.style.setProperty("background-repeat", "no-repeat", "important");
  1027. }
  1028. static processShortcutIcons(doc) {
  1029. let shortcutIcon = DomUtil.findShortcutIcon(Array.from(doc.querySelectorAll("link[href][rel=\"icon\"], link[href][rel=\"shortcut icon\"]")));
  1030. if (!shortcutIcon) {
  1031. shortcutIcon = DomUtil.findShortcutIcon(Array.from(doc.querySelectorAll("link[href][rel*=\"icon\"]")));
  1032. if (shortcutIcon) {
  1033. shortcutIcon.rel = "icon";
  1034. }
  1035. }
  1036. if (shortcutIcon) {
  1037. doc.querySelectorAll("link[href][rel*=\"icon\"]").forEach(linkElement => {
  1038. if (linkElement != shortcutIcon) {
  1039. linkElement.remove();
  1040. }
  1041. });
  1042. }
  1043. }
  1044. static setFrameEmptySrc(frameElement) {
  1045. if (frameElement.tagName == "OBJECT") {
  1046. frameElement.setAttribute("data", "data:text/html,");
  1047. } else {
  1048. frameElement.removeAttribute("src");
  1049. frameElement.removeAttribute("srcdoc");
  1050. }
  1051. }
  1052. static setFrameContent(frameElement, content) {
  1053. if (frameElement.tagName == "OBJECT") {
  1054. frameElement.setAttribute("data", "data:text/html," + content);
  1055. } else {
  1056. frameElement.setAttribute("srcdoc", content);
  1057. frameElement.removeAttribute("src");
  1058. }
  1059. }
  1060. static isolateElements(rootElement) {
  1061. rootElement.querySelectorAll("*:not(style)").forEach(element => {
  1062. if (element.getAttribute(SELECTED_CONTENT_ATTRIBUTE_NAME) == "") {
  1063. element.removeAttribute(SELECTED_CONTENT_ATTRIBUTE_NAME);
  1064. } else if (!element.querySelector("[" + SELECTED_CONTENT_ATTRIBUTE_NAME + "]")) {
  1065. element.remove();
  1066. }
  1067. });
  1068. isolateParentElements(rootElement.parentElement, rootElement);
  1069. function isolateParentElements(parentElement, element) {
  1070. let elementFound = false;
  1071. Array.from(parentElement.childNodes).forEach(node => {
  1072. elementFound = elementFound || (node == element);
  1073. const tagName = node.tagName && node.tagName.toLowerCase();
  1074. if (node != element && tagName != "svg" && tagName != "style" && tagName != "link") {
  1075. if (elementFound) {
  1076. node.remove();
  1077. } else {
  1078. node.hidden = true;
  1079. if (node.childNodes && node.childNodes.length) {
  1080. Array.from(node.childNodes).forEach(node => node.remove());
  1081. }
  1082. }
  1083. }
  1084. });
  1085. element = element.parentElement;
  1086. if (element && element.parentElement && element.parentElement.tagName != "HTML") {
  1087. isolateParentElements(element.parentElement, element);
  1088. }
  1089. }
  1090. }
  1091. static async resolveImportURLs(stylesheetContent, baseURI, options) {
  1092. stylesheetContent = DomProcessorHelper.resolveStylesheetURLs(stylesheetContent, baseURI, options);
  1093. const imports = DomUtil.getImportFunctions(stylesheetContent);
  1094. await Promise.all(imports.map(async cssImport => {
  1095. const match = DomUtil.matchImport(cssImport);
  1096. if (match) {
  1097. let resourceURL = DomUtil.normalizeURL(match.resourceURL);
  1098. if (!DomUtil.testIgnoredPath(resourceURL) && DomUtil.testValidPath(resourceURL, baseURI, options.url)) {
  1099. try {
  1100. resourceURL = docUtil.resolveURL(match.resourceURL, baseURI);
  1101. } catch (error) {
  1102. // ignored
  1103. }
  1104. if (DomUtil.testValidURL(resourceURL, baseURI, options.url)) {
  1105. const downloadOptions = { asDataURI: false, maxResourceSize: options.maxResourceSize, maxResourceSizeEnabled: options.maxResourceSizeEnabled, validateTextContentType: true };
  1106. const content = await docUtil.getContent(resourceURL, downloadOptions);
  1107. resourceURL = content.resourceURL;
  1108. let importedStylesheetContent = DomUtil.removeCssComments(content.data);
  1109. if (options.compressCSS) {
  1110. importedStylesheetContent = docUtil.compressCSS(importedStylesheetContent);
  1111. }
  1112. importedStylesheetContent = DomUtil.wrapMediaQuery(importedStylesheetContent, match.media);
  1113. if (stylesheetContent.includes(cssImport)) {
  1114. importedStylesheetContent = await DomProcessorHelper.resolveImportURLs(importedStylesheetContent, resourceURL, options);
  1115. stylesheetContent = stylesheetContent.replace(DomUtil.getRegExp(cssImport), importedStylesheetContent);
  1116. }
  1117. }
  1118. }
  1119. }
  1120. }));
  1121. return stylesheetContent;
  1122. }
  1123. static resolveStylesheetURLs(stylesheetContent, baseURI, options) {
  1124. const urlFunctions = DomUtil.getUrlFunctions(stylesheetContent);
  1125. urlFunctions.map(urlFunction => {
  1126. const originalResourceURL = DomUtil.matchURL(urlFunction);
  1127. const resourceURL = DomUtil.normalizeURL(originalResourceURL);
  1128. if (!DomUtil.testIgnoredPath(resourceURL)) {
  1129. if (DomUtil.testValidPath(resourceURL, baseURI, options.url)) {
  1130. let resolvedURL;
  1131. try {
  1132. resolvedURL = docUtil.resolveURL(resourceURL, baseURI);
  1133. } catch (error) {
  1134. // ignored
  1135. }
  1136. if (DomUtil.testValidURL(resolvedURL, baseURI, options.url) && resourceURL != resolvedURL && stylesheetContent.includes(urlFunction)) {
  1137. stylesheetContent = stylesheetContent.replace(DomUtil.getRegExp(urlFunction), urlFunction.replace(originalResourceURL, resolvedURL));
  1138. }
  1139. } else {
  1140. let newUrlFunction;
  1141. if (originalResourceURL) {
  1142. newUrlFunction = urlFunction.replace(originalResourceURL, EMPTY_DATA_URI);
  1143. } else {
  1144. newUrlFunction = "url(" + EMPTY_DATA_URI + ")";
  1145. }
  1146. stylesheetContent = stylesheetContent.replace(DomUtil.getRegExp(urlFunction), newUrlFunction);
  1147. }
  1148. } else {
  1149. if (resourceURL.startsWith(DATA_URI_PREFIX)) {
  1150. const escapedResourceURL = resourceURL.replace(REGEXP_AMP, "&amp;").replace(REGEXP_NBSP, "&nbsp;").replace(REGEXP_START_TAG, "&lt;").replace(REGEXP_END_TAG, "&gt;");
  1151. if (escapedResourceURL != resourceURL && stylesheetContent.includes(urlFunction)) {
  1152. stylesheetContent = stylesheetContent.replace(DomUtil.getRegExp(urlFunction), urlFunction.replace(originalResourceURL, escapedResourceURL));
  1153. }
  1154. }
  1155. }
  1156. });
  1157. return stylesheetContent;
  1158. }
  1159. static async resolveLinkStylesheetURLs(resourceURL, baseURI, options) {
  1160. resourceURL = DomUtil.normalizeURL(resourceURL);
  1161. if (resourceURL && resourceURL != baseURI && resourceURL != ABOUT_BLANK_URI) {
  1162. const downloadOptions = { asDataURI: false, maxResourceSize: options.maxResourceSize, maxResourceSizeEnabled: options.maxResourceSizeEnabled, charset: options.charset };
  1163. const content = await docUtil.getContent(resourceURL, downloadOptions);
  1164. resourceURL = content.resourceURL;
  1165. let stylesheetContent = DomUtil.removeCssComments(content.data);
  1166. if (options.compressCSS) {
  1167. stylesheetContent = docUtil.compressCSS(stylesheetContent);
  1168. }
  1169. stylesheetContent = await DomProcessorHelper.resolveImportURLs(stylesheetContent, resourceURL, options);
  1170. return stylesheetContent;
  1171. }
  1172. }
  1173. static async processStylesheet(cssRules, baseURI, options, cssVariables, batchRequest) {
  1174. const promises = [];
  1175. const removedRules = [];
  1176. for (let cssRule = cssRules.head; cssRule; cssRule = cssRule.next) {
  1177. const ruleData = cssRule.data;
  1178. if (ruleData.type == "Atrule" && ruleData.name == "charset") {
  1179. removedRules.push(cssRule);
  1180. } else if (ruleData.block && ruleData.block.children) {
  1181. if (ruleData.type == "Rule") {
  1182. promises.push(this.processStyle(ruleData.block.children.toArray(), baseURI, options, cssVariables, batchRequest));
  1183. } else if (ruleData.type == "Atrule" && (ruleData.name == "media" || ruleData.name == "supports")) {
  1184. promises.push(this.processStylesheet(ruleData.block.children, baseURI, options, cssVariables, batchRequest));
  1185. } else if (ruleData.type == "Atrule" && ruleData.name == "font-face") {
  1186. promises.push(processFontFaceRule(ruleData));
  1187. }
  1188. }
  1189. }
  1190. removedRules.forEach(cssRule => cssRules.remove(cssRule));
  1191. await Promise.all(promises);
  1192. async function processFontFaceRule(cssRule) {
  1193. await Promise.all(cssRule.block.children.toArray().map(async declaration => {
  1194. if (declaration.type == "Declaration" && declaration.value.children) {
  1195. const urlFunctions = DomUtil.getUrlFunctions(DomUtil.getCSSValue(declaration.value));
  1196. await Promise.all(urlFunctions.map(async urlFunction => {
  1197. const originalResourceURL = DomUtil.matchURL(urlFunction);
  1198. const resourceURL = DomUtil.normalizeURL(originalResourceURL);
  1199. if (!DomUtil.testIgnoredPath(resourceURL)) {
  1200. if (DomUtil.testValidURL(resourceURL, baseURI, options.url)) {
  1201. let { content } = await batchRequest.addURL(resourceURL);
  1202. let validResource = content == EMPTY_DATA_URI || content.startsWith(PREFIX_DATA_URI_VND) || content.startsWith(PREFIX_DATA_URI_IMAGE_SVG);
  1203. if (!validResource) {
  1204. validResource = await docUtil.validFont(urlFunction);
  1205. }
  1206. if (!validResource) {
  1207. content = EMPTY_DATA_URI;
  1208. }
  1209. declaration.value.children.forEach(token => {
  1210. if (token.type == "Url" && docUtil.removeQuotes(DomUtil.getCSSValue(token.value)) == originalResourceURL) {
  1211. token.value.value = content;
  1212. }
  1213. });
  1214. }
  1215. }
  1216. }));
  1217. }
  1218. }));
  1219. }
  1220. }
  1221. static async processStyle(declarations, baseURI, options, cssVariables, batchRequest) {
  1222. await Promise.all(declarations.map(async declaration => {
  1223. if (declaration.type == "Declaration" && declaration.value.children) {
  1224. const urlFunctions = DomUtil.getUrlFunctions(DomUtil.getCSSValue(declaration.value));
  1225. await Promise.all(urlFunctions.map(async urlFunction => {
  1226. const originalResourceURL = DomUtil.matchURL(urlFunction);
  1227. const resourceURL = DomUtil.normalizeURL(originalResourceURL);
  1228. if (!DomUtil.testIgnoredPath(resourceURL)) {
  1229. if (DomUtil.testValidURL(resourceURL, baseURI, options.url)) {
  1230. let { content, indexResource, duplicate } = await batchRequest.addURL(resourceURL);
  1231. if (duplicate && options.groupDuplicateImages) {
  1232. const tokens = [];
  1233. findURLToken(originalResourceURL, declaration.value.children, (token, parent) => {
  1234. const value = cssTree.parse("var(" + SINGLE_FILE_VARIABLE_NAME_PREFIX + indexResource + ")", { context: "value" }).children.head;
  1235. tokens.push({ parent, token, value });
  1236. });
  1237. cssVariables.set(indexResource, content);
  1238. tokens.forEach(({ parent, token, value }) => parent.replace(token, value));
  1239. } else {
  1240. findURLToken(originalResourceURL, declaration.value.children, token => token.data.value.value = content);
  1241. }
  1242. }
  1243. }
  1244. }));
  1245. }
  1246. }));
  1247. function findURLToken(url, children, callback) {
  1248. for (let token = children.head; token; token = token.next) {
  1249. if (token.data.children) {
  1250. findURLToken(url, token.data.children, callback);
  1251. }
  1252. if (token.data.type == "Url" && docUtil.removeQuotes(DomUtil.getCSSValue(token.data.value)) == url) {
  1253. callback(token, children);
  1254. }
  1255. }
  1256. }
  1257. }
  1258. static async processAttribute(doc, resourceElements, attributeName, prefixDataURI, baseURI, options, cssVariables, styles, batchRequest, processDuplicates, removeElementIfMissing) {
  1259. await Promise.all(Array.from(resourceElements).map(async resourceElement => {
  1260. let resourceURL = resourceElement.getAttribute(attributeName);
  1261. resourceURL = DomUtil.normalizeURL(resourceURL);
  1262. if (!DomUtil.testIgnoredPath(resourceURL)) {
  1263. resourceElement.setAttribute(attributeName, EMPTY_IMAGE);
  1264. if (DomUtil.testValidPath(resourceURL, baseURI, options.url)) {
  1265. try {
  1266. resourceURL = docUtil.resolveURL(resourceURL, baseURI);
  1267. } catch (error) {
  1268. // ignored
  1269. }
  1270. if (DomUtil.testValidURL(resourceURL, baseURI, options.url)) {
  1271. const { content, indexResource, duplicate } = await batchRequest.addURL(resourceURL);
  1272. if (removeElementIfMissing && content == EMPTY_DATA_URI) {
  1273. resourceElement.remove();
  1274. } else {
  1275. if (content.startsWith(prefixDataURI) || content.startsWith(PREFIX_DATA_URI_NO_MIMETYPE) || content.match(PREFIX_DATA_URI_OCTET_STREAM) || content.match(PREFIX_DATA_URI_NULL_STREAM)) {
  1276. const isSVG = content.startsWith(PREFIX_DATA_URI_IMAGE_SVG);
  1277. if (processDuplicates && duplicate && options.groupDuplicateImages && !isSVG) {
  1278. if (DomUtil.replaceImageSource(resourceElement, SINGLE_FILE_VARIABLE_NAME_PREFIX + indexResource, options)) {
  1279. cssVariables.set(indexResource, content);
  1280. const declarationList = cssTree.parse(resourceElement.getAttribute("style"), { context: "declarationList" });
  1281. styles.set(resourceElement, declarationList);
  1282. } else {
  1283. resourceElement.setAttribute(attributeName, content);
  1284. }
  1285. } else {
  1286. resourceElement.setAttribute(attributeName, content);
  1287. }
  1288. }
  1289. }
  1290. }
  1291. }
  1292. }
  1293. }));
  1294. }
  1295. static async processXLinks(resourceElements, baseURI, options, batchRequest) {
  1296. const attributeName = "xlink:href";
  1297. await Promise.all(Array.from(resourceElements).map(async resourceElement => {
  1298. const originalResourceURL = resourceElement.getAttribute(attributeName);
  1299. let resourceURL = DomUtil.normalizeURL(originalResourceURL);
  1300. if (DomUtil.testValidPath(resourceURL, baseURI, options.url) && !DomUtil.testIgnoredPath(resourceURL)) {
  1301. resourceElement.setAttribute(attributeName, EMPTY_IMAGE);
  1302. try {
  1303. resourceURL = docUtil.resolveURL(resourceURL, baseURI);
  1304. } catch (error) {
  1305. // ignored
  1306. }
  1307. if (DomUtil.testValidURL(resourceURL, baseURI, options.url)) {
  1308. try {
  1309. const { content } = await batchRequest.addURL(resourceURL, false);
  1310. const DOMParser = docUtil.getParser();
  1311. if (DOMParser) {
  1312. const svgDoc = new DOMParser().parseFromString(content, "image/svg+xml");
  1313. const hashMatch = originalResourceURL.match(REGEXP_URL_HASH);
  1314. if (hashMatch && hashMatch[0]) {
  1315. const symbolElement = svgDoc.querySelector(hashMatch[0]);
  1316. if (symbolElement) {
  1317. resourceElement.setAttribute(attributeName, hashMatch[0]);
  1318. resourceElement.parentElement.insertBefore(symbolElement, resourceElement.parentElement.firstChild);
  1319. }
  1320. } else {
  1321. resourceElement.setAttribute(attributeName, "data:image/svg+xml," + content);
  1322. }
  1323. } else {
  1324. resourceElement.setAttribute(attributeName, "data:image/svg+xml," + content);
  1325. }
  1326. } catch (error) {
  1327. /* ignored */
  1328. }
  1329. }
  1330. } else if (resourceURL == options.url) {
  1331. resourceElement.setAttribute(attributeName, originalResourceURL.substring(resourceURL.length));
  1332. }
  1333. }));
  1334. }
  1335. static async processSrcset(resourceElements, attributeName, prefixDataURI, baseURI, options, batchRequest) {
  1336. await Promise.all(Array.from(resourceElements).map(async resourceElement => {
  1337. const srcset = docUtil.parseSrcset(resourceElement.getAttribute(attributeName));
  1338. const srcsetValues = await Promise.all(srcset.map(async srcsetValue => {
  1339. let resourceURL = DomUtil.normalizeURL(srcsetValue.url);
  1340. if (!DomUtil.testIgnoredPath(resourceURL)) {
  1341. if (DomUtil.testValidPath(resourceURL, baseURI, options.url)) {
  1342. try {
  1343. resourceURL = docUtil.resolveURL(resourceURL, baseURI);
  1344. } catch (error) {
  1345. // ignored
  1346. }
  1347. if (DomUtil.testValidURL(resourceURL, baseURI, options.url)) {
  1348. const { content } = await batchRequest.addURL(resourceURL);
  1349. if (!content.startsWith(prefixDataURI) && !content.startsWith(PREFIX_DATA_URI_NO_MIMETYPE) && !content.match(PREFIX_DATA_URI_OCTET_STREAM) && !content.match(PREFIX_DATA_URI_NULL_STREAM)) {
  1350. resourceElement.setAttribute(attributeName, EMPTY_IMAGE);
  1351. }
  1352. return content + (srcsetValue.w ? " " + srcsetValue.w + "w" : srcsetValue.d ? " " + srcsetValue.d + "x" : "");
  1353. } else {
  1354. return "";
  1355. }
  1356. } else {
  1357. return "";
  1358. }
  1359. } else {
  1360. return resourceURL + (srcsetValue.w ? " " + srcsetValue.w + "w" : srcsetValue.d ? " " + srcsetValue.d + "x" : "");
  1361. }
  1362. }));
  1363. resourceElement.setAttribute(attributeName, srcsetValues.join(", "));
  1364. }));
  1365. }
  1366. }
  1367. // -------
  1368. // DomUtil
  1369. // -------
  1370. const DATA_URI_PREFIX = "data:";
  1371. const BLOB_URI_PREFIX = "blob:";
  1372. const HTTP_URI_PREFIX = /^https?:\/\//;
  1373. const FILE_URI_PREFIX = /^file:\/\//;
  1374. const EMPTY_URL = /^https?:\/\/+\s*$/;
  1375. const ABOUT_BLANK_URI = "about:blank";
  1376. const NOT_EMPTY_URL = /^(https?:\/\/|file:\/\/|blob:).+/;
  1377. const REGEXP_URL_FN = /(url\s*\(\s*'(.*?)'\s*\))|(url\s*\(\s*"(.*?)"\s*\))|(url\s*\(\s*(.*?)\s*\))/gi;
  1378. const REGEXP_URL_SIMPLE_QUOTES_FN = /^url\s*\(\s*'(.*?)'\s*\)$/i;
  1379. const REGEXP_URL_DOUBLE_QUOTES_FN = /^url\s*\(\s*"(.*?)"\s*\)$/i;
  1380. const REGEXP_URL_NO_QUOTES_FN = /^url\s*\(\s*(.*?)\s*\)$/i;
  1381. const REGEXP_IMPORT_FN = /(@import\s*url\s*\(\s*'(.*?)'\s*\)\s*(.*?)(;|$|}))|(@import\s*url\s*\(\s*"(.*?)"\s*\)\s*(.*?)(;|$|}))|(@import\s*url\s*\(\s*(.*?)\s*\)\s*(.*?)(;|$|}))|(@import\s*'(.*?)'\s*(.*?)(;|$|}))|(@import\s*"(.*?)"\s*(.*?)(;|$|}))|(@import\s*(.*?)\s*(.*?)(;|$|}))/gi;
  1382. const REGEXP_IMPORT_URL_SIMPLE_QUOTES_FN = /@import\s*url\s*\(\s*'(.*?)'\s*\)\s*(.*?)(;|$|})/i;
  1383. const REGEXP_IMPORT_URL_DOUBLE_QUOTES_FN = /@import\s*url\s*\(\s*"(.*?)"\s*\)\s*(.*?)(;|$|})/i;
  1384. const REGEXP_IMPORT_URL_NO_QUOTES_FN = /@import\s*url\s*\(\s*(.*?)\s*\)\s*(.*?)(;|$|})/i;
  1385. const REGEXP_IMPORT_SIMPLE_QUOTES_FN = /@import\s*'(.*?)'\s*(.*?)(;|$|})/i;
  1386. const REGEXP_IMPORT_DOUBLE_QUOTES_FN = /@import\s*"(.*?)"\s*(.*?)(;|$|})/i;
  1387. const REGEXP_IMPORT_NO_QUOTES_FN = /@import\s*(.*?)\s*(.*?)(;|$|})/i;
  1388. const REGEXP_ESCAPE = /([{}()^$&.*?/+|[\\\\]|\]|-)/g;
  1389. class DomUtil {
  1390. static normalizeURL(url) {
  1391. if (!url || url.startsWith(DATA_URI_PREFIX)) {
  1392. return url;
  1393. } else {
  1394. return url.split("#")[0];
  1395. }
  1396. }
  1397. static getCSSValue(value) {
  1398. let result = "";
  1399. try {
  1400. result = cssTree.generate(value);
  1401. } catch (error) {
  1402. // ignored
  1403. }
  1404. return result;
  1405. }
  1406. static async evalTemplateVariable(template, variableName, valueGetter, dontReplaceSlash) {
  1407. const replaceRegExp = new RegExp("{\\s*" + variableName + "\\s*}", "g");
  1408. if (template.match(replaceRegExp)) {
  1409. let value = await valueGetter();
  1410. if (!dontReplaceSlash) {
  1411. value = value.replace(/\/+/g, "_");
  1412. }
  1413. return template.replace(replaceRegExp, value);
  1414. }
  1415. return template;
  1416. }
  1417. static getLastSegment(url) {
  1418. let lastSegmentMatch = url.pathname.match(/\/([^/]+)$/), lastSegment = lastSegmentMatch && lastSegmentMatch[0];
  1419. if (!lastSegment) {
  1420. lastSegmentMatch = url.href.match(/([^/]+)\/?$/);
  1421. lastSegment = lastSegmentMatch && lastSegmentMatch[0];
  1422. }
  1423. if (!lastSegment) {
  1424. lastSegmentMatch = lastSegment.match(/(.*)<\.[^.]+$/);
  1425. lastSegment = lastSegmentMatch && lastSegmentMatch[0];
  1426. }
  1427. if (!lastSegment) {
  1428. lastSegment = url.hostname.replace(/\/+/g, "_").replace(/\/$/, "");
  1429. }
  1430. lastSegment.replace(/\/$/, "").replace(/^\//, "");
  1431. return lastSegment;
  1432. }
  1433. static getRegExp(string) {
  1434. return new RegExp(string.replace(REGEXP_ESCAPE, "\\$1"), "gi");
  1435. }
  1436. static getUrlFunctions(stylesheetContent) {
  1437. return Array.from(new Set(stylesheetContent.match(REGEXP_URL_FN) || []));
  1438. }
  1439. static getImportFunctions(stylesheetContent) {
  1440. return stylesheetContent.match(REGEXP_IMPORT_FN) || [];
  1441. }
  1442. static findShortcutIcon(shortcutIcons) {
  1443. shortcutIcons = shortcutIcons.filter(linkElement => linkElement.href != EMPTY_IMAGE);
  1444. shortcutIcons.sort((linkElement1, linkElement2) => (parseInt(linkElement2.sizes, 10) || 16) - (parseInt(linkElement1.sizes, 10) || 16));
  1445. return shortcutIcons[0];
  1446. }
  1447. static matchURL(stylesheetContent) {
  1448. const match = stylesheetContent.match(REGEXP_URL_SIMPLE_QUOTES_FN) ||
  1449. stylesheetContent.match(REGEXP_URL_DOUBLE_QUOTES_FN) ||
  1450. stylesheetContent.match(REGEXP_URL_NO_QUOTES_FN);
  1451. return match && match[1];
  1452. }
  1453. static testIgnoredPath(resourceURL) {
  1454. return resourceURL && (resourceURL.startsWith(DATA_URI_PREFIX) || resourceURL == ABOUT_BLANK_URI);
  1455. }
  1456. static testValidPath(resourceURL, baseURI, docURL) {
  1457. return resourceURL && resourceURL != baseURI && resourceURL != docURL && !resourceURL.match(EMPTY_URL);
  1458. }
  1459. static testValidURL(resourceURL, baseURI, docURL) {
  1460. return DomUtil.testValidPath(resourceURL, baseURI, docURL) && (resourceURL.match(HTTP_URI_PREFIX) || resourceURL.match(FILE_URI_PREFIX) || resourceURL.startsWith(BLOB_URI_PREFIX)) && resourceURL.match(NOT_EMPTY_URL);
  1461. }
  1462. static matchImport(stylesheetContent) {
  1463. const match = stylesheetContent.match(REGEXP_IMPORT_URL_SIMPLE_QUOTES_FN) ||
  1464. stylesheetContent.match(REGEXP_IMPORT_URL_DOUBLE_QUOTES_FN) ||
  1465. stylesheetContent.match(REGEXP_IMPORT_URL_NO_QUOTES_FN) ||
  1466. stylesheetContent.match(REGEXP_IMPORT_SIMPLE_QUOTES_FN) ||
  1467. stylesheetContent.match(REGEXP_IMPORT_DOUBLE_QUOTES_FN) ||
  1468. stylesheetContent.match(REGEXP_IMPORT_NO_QUOTES_FN);
  1469. if (match) {
  1470. const [, resourceURL, media] = match;
  1471. return { resourceURL, media };
  1472. }
  1473. }
  1474. static removeCssComments(stylesheetContent) {
  1475. let start, end;
  1476. do {
  1477. start = stylesheetContent.indexOf("/*");
  1478. end = stylesheetContent.indexOf("*/", start + 2);
  1479. if (start != -1 && end != -1) {
  1480. stylesheetContent = stylesheetContent.substring(0, start) + stylesheetContent.substr(end + 2);
  1481. }
  1482. } while (start != -1 && end != -1);
  1483. return stylesheetContent;
  1484. }
  1485. static wrapMediaQuery(stylesheetContent, mediaQuery) {
  1486. if (mediaQuery) {
  1487. return "@media " + mediaQuery + "{ " + stylesheetContent + " }";
  1488. } else {
  1489. return stylesheetContent;
  1490. }
  1491. }
  1492. static replaceImageSource(imgElement, variableName, options) {
  1493. const dataAttributeName = docUtil.imagesAttributeName(options.sessionId);
  1494. if (imgElement.getAttribute(dataAttributeName) != null) {
  1495. const imgData = options.imageData[Number(imgElement.getAttribute(dataAttributeName))];
  1496. if (imgData.replaceable) {
  1497. imgElement.setAttribute("src", `data:image/svg+xml,<svg xmlns="http://www.w3.org/2000/svg" width="${imgData.size.pxWidth}" height="${imgData.size.pxHeight}"><rect fill-opacity="0"/></svg>`);
  1498. const backgroundStyle = {};
  1499. const backgroundSize = (imgData.objectFit == "content" || imgData.objectFit == "cover") && imgData.objectFit;
  1500. if (backgroundSize) {
  1501. backgroundStyle["background-size"] = imgData.objectFit;
  1502. }
  1503. if (imgData.objectPosition) {
  1504. backgroundStyle["background-position"] = imgData.objectPosition;
  1505. }
  1506. if (imgData.backgroundColor) {
  1507. backgroundStyle["background-color"] = imgData.backgroundColor;
  1508. }
  1509. DomProcessorHelper.setBackgroundImage(imgElement, "var(" + variableName + ")", backgroundStyle);
  1510. imgElement.removeAttribute(dataAttributeName);
  1511. return true;
  1512. }
  1513. }
  1514. }
  1515. }
  1516. function log(...args) {
  1517. console.log("S-File <core> ", ...args); // eslint-disable-line no-console
  1518. }
  1519. // -----
  1520. // Stats
  1521. // -----
  1522. const STATS_DEFAULT_VALUES = {
  1523. discarded: {
  1524. "HTML bytes": 0,
  1525. "hidden elements": 0,
  1526. "HTML imports": 0,
  1527. scripts: 0,
  1528. objects: 0,
  1529. "audio sources": 0,
  1530. "video sources": 0,
  1531. frames: 0,
  1532. "CSS rules": 0,
  1533. canvas: 0,
  1534. stylesheets: 0,
  1535. resources: 0,
  1536. medias: 0
  1537. },
  1538. processed: {
  1539. "HTML bytes": 0,
  1540. "hidden elements": 0,
  1541. "HTML imports": 0,
  1542. scripts: 0,
  1543. objects: 0,
  1544. "audio sources": 0,
  1545. "video sources": 0,
  1546. frames: 0,
  1547. "CSS rules": 0,
  1548. canvas: 0,
  1549. stylesheets: 0,
  1550. resources: 0,
  1551. medias: 0
  1552. }
  1553. };
  1554. class Stats {
  1555. constructor(options) {
  1556. this.options = options;
  1557. if (options.displayStats) {
  1558. this.data = JSON.parse(JSON.stringify(STATS_DEFAULT_VALUES));
  1559. }
  1560. }
  1561. set(type, subType, value) {
  1562. if (this.options.displayStats) {
  1563. this.data[type][subType] = value;
  1564. }
  1565. }
  1566. add(type, subType, value) {
  1567. if (this.options.displayStats) {
  1568. this.data[type][subType] += value;
  1569. }
  1570. }
  1571. addAll(pageData) {
  1572. if (this.options.displayStats) {
  1573. Object.keys(this.data.discarded).forEach(key => this.add("discarded", key, pageData.stats.discarded[key] || 0));
  1574. Object.keys(this.data.processed).forEach(key => this.add("processed", key, pageData.stats.processed[key] || 0));
  1575. }
  1576. }
  1577. }
  1578. return { getClass };
  1579. })();