single-file-core.js 70 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678
  1. /*
  2. * Copyright 2018 Gildas Lormeau
  3. * contact : gildas.lormeau <at> gmail.com
  4. *
  5. * This file is part of SingleFile.
  6. *
  7. * The code in this file is free software: you can redistribute it and/or
  8. * modify it under the terms of the GNU Affero General Public License
  9. * (GNU AGPL) as published by the Free Software Foundation, either version 3
  10. * of the License, or (at your option) any later version.
  11. *
  12. * The code in this file is distributed in the hope that it will be useful,
  13. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
  15. * General Public License for more details.
  16. *
  17. * As additional permission under GNU AGPL version 3 section 7, you may
  18. * distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU
  19. * AGPL normally required by section 4, provided you include this license
  20. * notice and a URL through which recipients can access the Corresponding
  21. * Source.
  22. */
  23. this.SingleFileCore = this.SingleFileCore || (() => {
  24. const SELECTED_CONTENT_ATTRIBUTE_NAME = "data-single-file-selected-content";
  25. const SELECTED_CONTENT_ROOT_ATTRIBUTE_NAME = "data-single-file-selected-content-root";
  26. const DEBUG = false;
  27. let Download, DOM, URL, cssTree, sessionId = 0;
  28. function getClass(...args) {
  29. [Download, DOM, URL, cssTree] = args;
  30. return SingleFileClass;
  31. }
  32. class SingleFileClass {
  33. constructor(options) {
  34. this.options = options;
  35. if (options.sessionId === undefined) {
  36. options.sessionId = sessionId;
  37. sessionId++;
  38. }
  39. }
  40. async initialize() {
  41. this.processor = new PageProcessor(this.options, true);
  42. await this.processor.loadPage();
  43. await this.processor.initialize();
  44. }
  45. async run() {
  46. await this.processor.run();
  47. }
  48. async getPageData() {
  49. return this.processor.getPageData();
  50. }
  51. }
  52. SingleFileClass.SELECTED_CONTENT_ATTRIBUTE_NAME = SELECTED_CONTENT_ATTRIBUTE_NAME;
  53. SingleFileClass.SELECTED_CONTENT_ROOT_ATTRIBUTE_NAME = SELECTED_CONTENT_ROOT_ATTRIBUTE_NAME;
  54. // -------------
  55. // ProgressEvent
  56. // -------------
  57. const PAGE_LOADING = "page-loading";
  58. const PAGE_LOADED = "page-loaded";
  59. const RESOURCES_INITIALIZING = "resource-initializing";
  60. const RESOURCES_INITIALIZED = "resources-initialized";
  61. const RESOURCE_LOADED = "resource-loaded";
  62. const PAGE_ENDED = "page-ended";
  63. const STAGE_STARTED = "stage-started";
  64. const STAGE_ENDED = "stage-ended";
  65. const STAGE_TASK_STARTED = "stage-task-started";
  66. const STAGE_TASK_ENDED = "stage-task-ended";
  67. class ProgressEvent {
  68. constructor(type, detail) {
  69. return { type, detail, PAGE_LOADING, PAGE_LOADED, RESOURCES_INITIALIZING, RESOURCES_INITIALIZED, RESOURCE_LOADED, PAGE_ENDED, STAGE_STARTED, STAGE_ENDED, STAGE_TASK_STARTED, STAGE_TASK_ENDED };
  70. }
  71. }
  72. // -------------
  73. // PageProcessor
  74. // -------------
  75. const RESOLVE_URLS_STAGE = 0;
  76. const REPLACE_DATA_STAGE = 1;
  77. const REPLACE_DOCS_STAGE = 2;
  78. const POST_PROCESS_STAGE = 3;
  79. const STAGES = [{
  80. sequential: [
  81. { action: "preProcessPage" },
  82. { action: "insertShadowRootContents" },
  83. { action: "replaceStyleContents" },
  84. { option: "selected", action: "removeUnselectedElements" },
  85. { option: "removeVideoSrc", action: "insertVideoPosters" },
  86. { option: "removeFrames", action: "removeFrames" },
  87. { option: "removeImports", action: "removeImports" },
  88. { option: "removeScripts", action: "removeScripts" },
  89. { action: "removeDiscardedResources" },
  90. { action: "resetCharsetMeta" },
  91. { action: "setInputValues" },
  92. { option: "insertFaviconLink", action: "insertFaviconLink" },
  93. { action: "replaceCanvasElements" },
  94. { action: "insertFonts" },
  95. { option: "removeHiddenElements", action: "removeHiddenElements" },
  96. { action: "resolveHrefs" },
  97. { action: "resolveStyleAttributeURLs" }
  98. ],
  99. parallel: [
  100. { action: "resolveStylesheetURLs" },
  101. { option: "!removeFrames", action: "resolveFrameURLs" },
  102. { option: "!removeImports", action: "resolveHtmlImportURLs" }
  103. ]
  104. }, {
  105. sequential: [
  106. { option: "removeUnusedStyles", action: "removeUnusedStyles" },
  107. { option: "removeAlternativeMedias", action: "removeAlternativeMedias" },
  108. { option: "removeUnusedStyles", action: "removeUnusedFonts" }
  109. ],
  110. parallel: [
  111. { action: "processStylesheets" },
  112. { action: "processStyleAttributes" },
  113. { action: "processPageResources" },
  114. { option: "!removeScripts", action: "processScripts" }
  115. ]
  116. }, {
  117. sequential: [
  118. { option: "removeAlternativeImages", action: "removeAlternativeImages" },
  119. { option: "removeAlternativeFonts", action: "removeAlternativeFonts" }
  120. ],
  121. parallel: [
  122. { option: "!removeFrames", action: "processFrames" },
  123. { option: "!removeImports", action: "processHtmlImports" },
  124. ]
  125. }, {
  126. sequential: [
  127. { action: "replaceStylesheets" },
  128. { action: "replaceStyleAttributes" },
  129. { action: "insertVariables" },
  130. { option: "compressHTML", action: "compressHTML" }
  131. ]
  132. }];
  133. class PageProcessor {
  134. constructor(options, root) {
  135. this.root = root;
  136. this.options = options;
  137. this.options.url = this.options.url || this.options.doc.location.href;
  138. this.options.baseURI = this.options.doc && this.options.doc.baseURI;
  139. this.batchRequest = new BatchRequest();
  140. this.processor = new DOMProcessor(options, this.batchRequest);
  141. if (this.options.doc) {
  142. const docData = DOM.preProcessDoc(this.options.doc, this.options.win, this.options);
  143. this.options.canvasData = docData.canvasData;
  144. this.options.fontsData = docData.fontsData;
  145. this.options.stylesheetContents = docData.stylesheetContents;
  146. this.options.imageData = docData.imageData;
  147. this.options.postersData = docData.postersData;
  148. this.options.usedFonts = docData.usedFonts;
  149. this.options.shadowRootContents = docData.shadowRootContents;
  150. }
  151. this.options.content = this.options.content || (this.options.doc ? DOM.serialize(this.options.doc, false) : null);
  152. this.onprogress = options.onprogress || (() => { });
  153. }
  154. async loadPage() {
  155. this.onprogress(new ProgressEvent(PAGE_LOADING, { pageURL: this.options.url, frame: !this.root }));
  156. await this.processor.loadPage(this.options.content);
  157. this.onprogress(new ProgressEvent(PAGE_LOADED, { pageURL: this.options.url, frame: !this.root }));
  158. }
  159. async initialize() {
  160. this.onprogress(new ProgressEvent(RESOURCES_INITIALIZING, { pageURL: this.options.url }));
  161. await this.executeStage(RESOLVE_URLS_STAGE);
  162. this.pendingPromises = this.executeStage(REPLACE_DATA_STAGE);
  163. if (this.options.doc) {
  164. DOM.postProcessDoc(this.options.doc, this.options);
  165. this.options.doc = null;
  166. this.options.win = null;
  167. }
  168. }
  169. async run() {
  170. if (this.root) {
  171. this.processor.initialize(this.batchRequest);
  172. this.onprogress(new ProgressEvent(RESOURCES_INITIALIZED, { pageURL: this.options.url, index: 0, max: this.processor.maxResources }));
  173. }
  174. await this.batchRequest.run(detail => {
  175. detail.pageURL = this.options.url;
  176. this.onprogress(new ProgressEvent(RESOURCE_LOADED, detail));
  177. }, this.options);
  178. await this.pendingPromises;
  179. await this.executeStage(REPLACE_DOCS_STAGE);
  180. await this.executeStage(POST_PROCESS_STAGE);
  181. await this.processor.end();
  182. }
  183. async getPageData() {
  184. if (this.root) {
  185. this.onprogress(new ProgressEvent(PAGE_ENDED, { pageURL: this.options.url }));
  186. }
  187. return this.processor.getPageData();
  188. }
  189. async executeStage(step) {
  190. if (DEBUG) {
  191. log("**** STARTED STAGE", step, "****");
  192. }
  193. const frame = !this.root;
  194. this.onprogress(new ProgressEvent(STAGE_STARTED, { pageURL: this.options.url, step, frame }));
  195. STAGES[step].sequential.forEach(task => {
  196. let startTime;
  197. if (DEBUG) {
  198. startTime = Date.now();
  199. log(" -- STARTED task =", task.action);
  200. }
  201. this.onprogress(new ProgressEvent(STAGE_TASK_STARTED, { pageURL: this.options.url, step, task: task.action, frame }));
  202. this.executeTask(task);
  203. this.onprogress(new ProgressEvent(STAGE_TASK_ENDED, { pageURL: this.options.url, step, task: task.action, frame }));
  204. if (DEBUG) {
  205. log(" -- ENDED task =", task.action, "delay =", Date.now() - startTime);
  206. }
  207. });
  208. let parallelTasksPromise;
  209. if (STAGES[step].parallel) {
  210. parallelTasksPromise = await Promise.all(STAGES[step].parallel.map(async task => {
  211. let startTime;
  212. if (DEBUG) {
  213. startTime = Date.now();
  214. log(" // STARTED task =", task.action);
  215. }
  216. this.onprogress(new ProgressEvent(STAGE_TASK_STARTED, { pageURL: this.options.url, step, task: task.action, frame }));
  217. await this.executeTask(task);
  218. this.onprogress(new ProgressEvent(STAGE_TASK_ENDED, { pageURL: this.options.url, step, task: task.action, frame }));
  219. if (DEBUG) {
  220. log(" // ENDED task =", task.action, "delay =", Date.now() - startTime);
  221. }
  222. }));
  223. } else {
  224. parallelTasksPromise = Promise.resolve();
  225. }
  226. this.onprogress(new ProgressEvent(STAGE_ENDED, { pageURL: this.options.url, step, frame }));
  227. if (DEBUG) {
  228. log("**** ENDED STAGE", step, "****");
  229. }
  230. return parallelTasksPromise;
  231. }
  232. executeTask(task) {
  233. if (!task.option || ((task.option.startsWith("!") && !this.options[task.option]) || this.options[task.option])) {
  234. return this.processor[task.action]();
  235. }
  236. }
  237. }
  238. // --------
  239. // BatchRequest
  240. // --------
  241. class BatchRequest {
  242. constructor() {
  243. this.requests = new Map();
  244. }
  245. async addURL(resourceURL, asDataURI = true) {
  246. return new Promise((resolve, reject) => {
  247. const requestKey = JSON.stringify([resourceURL, asDataURI]);
  248. const resourceRequests = this.requests.get(requestKey);
  249. if (resourceRequests) {
  250. resourceRequests.push({ resolve, reject });
  251. } else {
  252. this.requests.set(requestKey, [{ resolve, reject }]);
  253. }
  254. });
  255. }
  256. getMaxResources() {
  257. return Array.from(this.requests.keys()).length;
  258. }
  259. async run(onloadListener, options) {
  260. const resourceURLs = Array.from(this.requests.keys());
  261. let indexResource = 0;
  262. return Promise.all(resourceURLs.map(async requestKey => {
  263. const [resourceURL, asDataURI] = JSON.parse(requestKey);
  264. const resourceRequests = this.requests.get(requestKey);
  265. try {
  266. const resourceContent = await Download.getContent(resourceURL, { asDataURI, maxResourceSize: options.maxResourceSize, maxResourceSizeEnabled: options.maxResourceSizeEnabled });
  267. indexResource = indexResource + 1;
  268. onloadListener({ index: indexResource, url: resourceURL });
  269. resourceRequests.forEach(resourceRequest => resourceRequest.resolve({ content: resourceContent, indexResource, duplicate: Boolean(resourceRequests.length > 1) }));
  270. } catch (error) {
  271. indexResource = indexResource + 1;
  272. onloadListener({ index: indexResource, url: resourceURL });
  273. resourceRequests.forEach(resourceRequest => resourceRequest.reject(error));
  274. }
  275. this.requests.delete(requestKey);
  276. }));
  277. }
  278. }
  279. // ------------
  280. // DOMProcessor
  281. // ------------
  282. const EMPTY_DATA_URI = "data:base64,";
  283. const EMPTY_IMAGE = "data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==";
  284. const SCRIPT_TAG_FOUND = /<script/gi;
  285. const NOSCRIPT_TAG_FOUND = /<noscript/gi;
  286. class DOMProcessor {
  287. constructor(options, batchRequest) {
  288. this.options = options;
  289. this.stats = new Stats(options);
  290. this.baseURI = DomUtil.normalizeURL(options.baseURI || options.url);
  291. this.batchRequest = batchRequest;
  292. this.stylesheets = new Map();
  293. this.styles = new Map();
  294. this.cssVariables = new Map();
  295. }
  296. initialize() {
  297. this.maxResources = this.batchRequest.getMaxResources();
  298. if (!this.options.removeFrames) {
  299. this.options.framesData.forEach(frameData => this.maxResources += frameData.maxResources || 0);
  300. }
  301. this.stats.set("processed", "resources", this.maxResources);
  302. }
  303. async loadPage(pageContent) {
  304. if (!pageContent || this.options.saveRawPage) {
  305. pageContent = await Download.getContent(this.baseURI, { asDataURI: false, maxResourceSize: this.options.maxResourceSize, maxResourceSizeEnabled: this.options.maxResourceSizeEnabled });
  306. }
  307. this.doc = DOM.createDoc(pageContent, this.baseURI);
  308. this.onEventAttributeNames = DOM.getOnEventAttributeNames(this.doc);
  309. }
  310. async getPageData() {
  311. DOM.postProcessDoc(this.doc, this.options);
  312. const url = new URL(this.baseURI);
  313. if (this.options.insertSingleFileComment) {
  314. const infobarContent = (this.options.infobarContent || "").replace(/\\n/g, "\n").replace(/\\t/g, "\t");
  315. const commentNode = this.doc.createComment("\n Page saved with SingleFile" +
  316. " \n url: " + this.options.url +
  317. " \n saved date: " + new Date() +
  318. (infobarContent ? " \n info: " + infobarContent : "") + "\n");
  319. this.doc.documentElement.insertBefore(commentNode, this.doc.documentElement.firstChild);
  320. }
  321. let size;
  322. if (this.options.displayStats) {
  323. size = DOM.getContentSize(this.doc.documentElement.outerHTML);
  324. }
  325. const content = DOM.serialize(this.doc, this.options.compressHTML);
  326. if (this.options.displayStats) {
  327. const contentSize = DOM.getContentSize(content);
  328. this.stats.set("processed", "HTML bytes", contentSize);
  329. this.stats.add("discarded", "HTML bytes", size - contentSize);
  330. }
  331. let filename = await DomProcessorHelper.evalTemplate(this.options.filenameTemplate, this.options, content);
  332. filename = filename.replace(/[~\\?%*:|"<>\x00-\x1f\x7F]+/g, "_"); // eslint-disable-line no-control-regex
  333. filename = filename.replace(/\.\.\//g, "").replace(/^\/+/, "").replace(/\/+/g, "/").replace(/\/$/, "");
  334. if (!this.options.backgroundSave) {
  335. filename = filename.replace(/\//g, "_");
  336. }
  337. if (filename.length > 192) {
  338. const extensionMatch = filename.match(/(\.[^.]{3,4})$/);
  339. const extension = extensionMatch && extensionMatch[0] && extensionMatch[0].length > 1 ? extensionMatch[0] : "";
  340. filename = filename.substring(0, 192 - extension.length) + "…" + extension;
  341. }
  342. if (!filename) {
  343. filename = "Unnamed page";
  344. }
  345. const matchTitle = this.baseURI.match(/([^/]*)\/?(\.html?.*)$/) || this.baseURI.match(/\/\/([^/]*)\/?$/);
  346. return {
  347. stats: this.stats.data,
  348. title: this.options.title || (this.baseURI && matchTitle ? matchTitle[1] : (url.hostname ? url.hostname : "")),
  349. filename,
  350. content
  351. };
  352. }
  353. removeUnselectedElements() {
  354. const rootElement = this.doc.querySelector("[" + SELECTED_CONTENT_ROOT_ATTRIBUTE_NAME + "]");
  355. if (rootElement) {
  356. DomProcessorHelper.isolateElements(rootElement);
  357. rootElement.removeAttribute(SELECTED_CONTENT_ROOT_ATTRIBUTE_NAME);
  358. rootElement.removeAttribute(SELECTED_CONTENT_ATTRIBUTE_NAME);
  359. }
  360. }
  361. setInputValues() {
  362. this.doc.querySelectorAll("input").forEach(input => {
  363. const value = input.getAttribute(DOM.inputValueAttributeName(this.options.sessionId));
  364. input.setAttribute("value", value || "");
  365. });
  366. this.doc.querySelectorAll("input[type=radio], input[type=checkbox]").forEach(input => {
  367. const value = input.getAttribute(DOM.inputValueAttributeName(this.options.sessionId));
  368. if (value == "true") {
  369. input.setAttribute("checked", "");
  370. }
  371. });
  372. this.doc.querySelectorAll("textarea").forEach(textarea => {
  373. const value = textarea.getAttribute(DOM.inputValueAttributeName(this.options.sessionId));
  374. textarea.textContent = value || "";
  375. });
  376. this.doc.querySelectorAll("select").forEach(select => {
  377. select.querySelectorAll("option").forEach(option => {
  378. const selected = option.getAttribute(DOM.inputValueAttributeName(this.options.sessionId)) != null;
  379. if (selected) {
  380. option.setAttribute("selected", "");
  381. }
  382. });
  383. });
  384. }
  385. removeDiscardedResources() {
  386. const objectElements = this.doc.querySelectorAll("applet, meta[http-equiv=refresh], object[data]:not([type=\"image/svg+xml\"]):not([type=\"image/svg-xml\"]):not([type=\"text/html\"]), embed[src]:not([src*=\".svg\"])");
  387. this.stats.set("discarded", "objects", objectElements.length);
  388. this.stats.set("processed", "objects", objectElements.length);
  389. objectElements.forEach(element => element.remove());
  390. const replacedAttributeValue = this.doc.querySelectorAll("link[rel~=preconnect], link[rel~=prerender], link[rel~=dns-prefetch], link[rel~=preload], link[rel~=prefetch]");
  391. replacedAttributeValue.forEach(element => {
  392. const relValue = element.getAttribute("rel").replace(/(preconnect|prerender|dns-prefetch|preload|prefetch)/g, "").trim();
  393. if (relValue.length) {
  394. element.setAttribute("rel", relValue);
  395. } else {
  396. element.remove();
  397. }
  398. });
  399. this.doc.querySelectorAll("meta[http-equiv=\"content-security-policy\"]").forEach(element => element.remove());
  400. if (this.options.compressHTML) {
  401. this.doc.querySelectorAll("input[type=hidden]").forEach(element => element.remove());
  402. }
  403. this.doc.querySelectorAll("a[ping]").forEach(element => element.removeAttribute("ping"));
  404. if (this.options.removeScripts) {
  405. this.onEventAttributeNames.forEach(attributeName => this.doc.querySelectorAll("[" + attributeName + "]").forEach(element => element.removeAttribute(attributeName)));
  406. this.doc.querySelectorAll("[href]").forEach(element => {
  407. if (element.href && element.href.match && element.href.match(/^\s*javascript:/)) {
  408. element.setAttribute("href", "");
  409. }
  410. });
  411. this.doc.querySelectorAll("[src]").forEach(element => {
  412. if (element.src && element.src.match(/^\s*javascript:/)) {
  413. element.removeAttribute("src");
  414. }
  415. });
  416. }
  417. const audioSourceElements = this.doc.querySelectorAll("audio[src], audio > source[src]");
  418. this.stats.set("processed", "audio sources", audioSourceElements.length);
  419. if (this.options.removeAudioSrc) {
  420. this.stats.set("discarded", "audio sources", audioSourceElements.length);
  421. audioSourceElements.forEach(element => {
  422. if (element.tagName == "SOURCE") {
  423. element.remove();
  424. } else {
  425. audioSourceElements.forEach(element => element.removeAttribute("src"));
  426. }
  427. });
  428. }
  429. const videoSourceElements = this.doc.querySelectorAll("video[src], video > source[src]");
  430. this.stats.set("processed", "video sources", videoSourceElements.length);
  431. if (this.options.removeVideoSrc) {
  432. this.stats.set("discarded", "video sources", videoSourceElements.length);
  433. videoSourceElements.forEach(element => {
  434. if (element.tagName == "SOURCE") {
  435. element.remove();
  436. } else {
  437. videoSourceElements.forEach(element => element.removeAttribute("src"));
  438. }
  439. });
  440. }
  441. }
  442. async end() {
  443. const metaCharset = this.doc.head.querySelector("meta[charset]");
  444. if (metaCharset) {
  445. this.doc.head.insertBefore(metaCharset, this.doc.head.firstChild);
  446. }
  447. this.doc.querySelectorAll("base").forEach(element => element.remove());
  448. if (this.doc.head.querySelectorAll("*").length == 1 && metaCharset && this.doc.body.childNodes.length == 0) {
  449. this.doc.head.querySelector("meta[charset]").remove();
  450. }
  451. const titleElement = this.doc.querySelector("title");
  452. const descriptionElement = this.doc.querySelector("meta[name=description]");
  453. const authorElement = this.doc.querySelector("meta[name=author]");
  454. const creatorElement = this.doc.querySelector("meta[name=creator]");
  455. const publisherElement = this.doc.querySelector("meta[name=publisher]");
  456. this.options.title = titleElement ? titleElement.textContent.trim() : "";
  457. this.options.infobarContent = await DomProcessorHelper.evalTemplate(this.options.infobarTemplate, this.options, null, true);
  458. this.options.info = {
  459. description: descriptionElement ? descriptionElement.content.trim() : "",
  460. lang: this.doc.documentElement.lang,
  461. author: authorElement ? authorElement.content.trim() : "",
  462. creator: creatorElement ? creatorElement.content.trim() : "",
  463. publisher: publisherElement ? publisherElement.content.trim() : ""
  464. };
  465. }
  466. preProcessPage() {
  467. this.doc.querySelectorAll("singlefile-infobar, singlefile-mask, singlefile-logs-window").forEach(element => element.remove());
  468. if (this.options.win) {
  469. this.doc.body.querySelectorAll(":not(svg) title, meta, link[href][rel*=\"icon\"]").forEach(element => element instanceof this.options.win.HTMLElement && this.doc.head.appendChild(element));
  470. }
  471. if (this.options.imageData) {
  472. const dataAttributeName = DOM.imagesAttributeName(this.options.sessionId);
  473. this.doc.querySelectorAll("img").forEach(imgElement => {
  474. const imgData = this.options.imageData[Number(imgElement.getAttribute(dataAttributeName))];
  475. if (this.options.removeHiddenElements && imgData.size && !imgData.size.pxWidth && !imgData.size.pxHeight) {
  476. imgElement.setAttribute("src", EMPTY_IMAGE);
  477. } else if (imgData.currentSrc) {
  478. imgElement.setAttribute("src", imgData.currentSrc);
  479. }
  480. });
  481. if (this.options.lazyLoadImages) {
  482. this.doc.querySelectorAll("img[data-src]").forEach(imgElement => {
  483. const imgData = this.options.imageData[Number(imgElement.getAttribute(dataAttributeName))];
  484. if ((!imgElement.getAttribute("src") || imgElement.getAttribute("src") == EMPTY_IMAGE) && imgElement.getAttribute("data-src")) {
  485. imgData.src = imgElement.dataset.src;
  486. imgElement.setAttribute("src", imgElement.dataset.src);
  487. imgElement.removeAttribute("data-src");
  488. }
  489. });
  490. this.doc.querySelectorAll("img[data-srcset]").forEach(imgElement => {
  491. if (!imgElement.getAttribute("srcset") && imgElement.getAttribute("data-srcset")) {
  492. imgElement.setAttribute("srcset", imgElement.dataset.srcset);
  493. imgElement.removeAttribute("data-srcset");
  494. }
  495. });
  496. }
  497. }
  498. }
  499. removeScripts() {
  500. const scriptElements = this.doc.querySelectorAll("script:not([type=\"application/ld+json\"])");
  501. this.stats.set("discarded", "scripts", scriptElements.length);
  502. this.stats.set("processed", "scripts", scriptElements.length);
  503. scriptElements.forEach(element => element.remove());
  504. }
  505. removeFrames() {
  506. const frameElements = this.doc.querySelectorAll("iframe, frame, object[type=\"text/html\"][data]");
  507. this.stats.set("discarded", "frames", frameElements.length);
  508. this.stats.set("processed", "frames", frameElements.length);
  509. this.doc.querySelectorAll("iframe, frame, object[type=\"text/html\"][data]").forEach(element => element.remove());
  510. }
  511. removeImports() {
  512. const importElements = this.doc.querySelectorAll("link[rel=import]");
  513. this.stats.set("discarded", "HTML imports", importElements.length);
  514. this.stats.set("processed", "HTML imports", importElements.length);
  515. importElements.forEach(element => element.remove());
  516. }
  517. resetCharsetMeta() {
  518. let charSet;
  519. this.doc.querySelectorAll("meta[charset], meta[http-equiv=\"content-type\"]").forEach(element => {
  520. const charSetDeclaration = element.content.split(";")[1];
  521. if (charSetDeclaration && !charSet) {
  522. charSet = charSetDeclaration.split("=")[1];
  523. if (charSet) {
  524. this.charSet = charSet.trim().toLowerCase();
  525. }
  526. }
  527. element.remove();
  528. });
  529. const metaElement = this.doc.createElement("meta");
  530. metaElement.setAttribute("charset", "utf-8");
  531. if (this.doc.head.firstChild) {
  532. this.doc.head.insertBefore(metaElement, this.doc.head.firstChild);
  533. } else {
  534. this.doc.head.appendChild(metaElement);
  535. }
  536. }
  537. insertFaviconLink() {
  538. let faviconElement = this.doc.querySelector("link[href][rel=\"icon\"]");
  539. if (!faviconElement) {
  540. faviconElement = this.doc.querySelector("link[href][rel=\"shortcut icon\"]");
  541. }
  542. if (!faviconElement) {
  543. faviconElement = this.doc.createElement("link");
  544. faviconElement.setAttribute("type", "image/x-icon");
  545. faviconElement.setAttribute("rel", "shortcut icon");
  546. faviconElement.setAttribute("href", "/favicon.ico");
  547. }
  548. this.doc.head.appendChild(faviconElement);
  549. }
  550. resolveHrefs() {
  551. this.doc.querySelectorAll("[href]").forEach(element => {
  552. const href = element.getAttribute("href");
  553. if (!DomUtil.testIgnoredPath(href) && DomUtil.testValidPath(href, this.baseURI, this.options.url)) {
  554. const normalizedHref = DomUtil.normalizeURL(href);
  555. if (element.tagName == "LINK") {
  556. try {
  557. element.setAttribute("href", new URL(href, this.options.baseURI || this.options.url));
  558. } catch (error) {
  559. if (element.tagName == "LINK") {
  560. element.setAttribute("href", EMPTY_URL);
  561. }
  562. }
  563. } else if (normalizedHref == href) {
  564. try {
  565. element.setAttribute("href", new URL(normalizedHref, this.options.baseURI || this.options.url));
  566. } catch (error) {
  567. // ignored
  568. }
  569. }
  570. }
  571. });
  572. }
  573. removeUnusedStyles() {
  574. if (!this.mediaAllInfo) {
  575. this.mediaAllInfo = DOM.getMediaAllInfo(this.doc, this.stylesheets, this.styles);
  576. }
  577. const stats = DOM.minifyCSSRules(this.stylesheets, this.styles, this.mediaAllInfo);
  578. this.stats.set("processed", "CSS rules", stats.processed);
  579. this.stats.set("discarded", "CSS rules", stats.discarded);
  580. }
  581. removeUnusedFonts() {
  582. DOM.removeUnusedFonts(this.doc, this.stylesheets, this.styles, this.options);
  583. }
  584. removeAlternativeFonts() {
  585. DOM.removeAlternativeFonts(this.doc, this.stylesheets);
  586. }
  587. removeAlternativeImages() {
  588. DOM.removeAlternativeImages(this.doc);
  589. }
  590. removeHiddenElements() {
  591. const hiddenElements = this.doc.querySelectorAll("[" + DOM.removedContentAttributeName(this.options.sessionId) + "]");
  592. this.stats.set("discarded", "hidden elements", hiddenElements.length);
  593. this.stats.set("processed", "hidden elements", hiddenElements.length);
  594. hiddenElements.forEach(element => element.remove());
  595. }
  596. compressHTML() {
  597. let size;
  598. if (this.options.displayStats) {
  599. size = DOM.getContentSize(this.doc.documentElement.outerHTML);
  600. }
  601. DOM.minifyHTML(this.doc, { preservedSpaceAttributeName: DOM.preservedSpaceAttributeName(this.options.sessionId) });
  602. if (this.options.displayStats) {
  603. this.stats.add("discarded", "HTML bytes", size - DOM.getContentSize(this.doc.documentElement.outerHTML));
  604. }
  605. }
  606. removeAlternativeMedias() {
  607. const stats = DOM.minifyMedias(this.stylesheets);
  608. this.stats.set("processed", "medias", stats.processed);
  609. this.stats.set("discarded", "medias", stats.discarded);
  610. }
  611. replaceCanvasElements() {
  612. if (this.options.canvasData) {
  613. this.doc.querySelectorAll("canvas").forEach((canvasElement, indexCanvasElement) => {
  614. const canvasData = this.options.canvasData[indexCanvasElement];
  615. if (canvasData) {
  616. DomProcessorHelper.setBackgroundImage(canvasElement, "url(" + canvasData.dataURI + ")");
  617. this.stats.add("processed", "canvas", 1);
  618. }
  619. });
  620. }
  621. }
  622. insertFonts() {
  623. if (this.options.fontsData && this.options.fontsData.length) {
  624. let stylesheetContent = "";
  625. this.options.fontsData.forEach(fontStyles => {
  626. if (fontStyles["font-family"] && fontStyles.src) {
  627. stylesheetContent += "@font-face{";
  628. let stylesContent = "";
  629. Object.keys(fontStyles).forEach(fontStyle => {
  630. if (stylesContent) {
  631. stylesContent += ";";
  632. }
  633. stylesContent += fontStyle + ":" + fontStyles[fontStyle];
  634. });
  635. stylesheetContent += stylesContent + "}";
  636. }
  637. });
  638. if (stylesheetContent) {
  639. const styleElement = this.doc.createElement("style");
  640. styleElement.textContent = stylesheetContent;
  641. const existingStyleElement = this.doc.querySelector("style");
  642. if (existingStyleElement) {
  643. existingStyleElement.parentElement.insertBefore(styleElement, existingStyleElement);
  644. } else {
  645. this.doc.head.insertBefore(styleElement, this.doc.head.firstChild);
  646. }
  647. }
  648. }
  649. }
  650. replaceStyleContents() {
  651. if (this.options.stylesheetContents) {
  652. this.doc.querySelectorAll("style").forEach((styleElement, styleIndex) => {
  653. if (this.options.stylesheetContents[styleIndex]) {
  654. styleElement.textContent = this.options.stylesheetContents[styleIndex];
  655. }
  656. });
  657. }
  658. }
  659. insertShadowRootContents() {
  660. if (this.options.shadowRootContents) {
  661. this.doc.querySelectorAll("[" + DOM.shadowRootAttributeName(this.options.sessionId) + "]").forEach((element, elementIndex) => {
  662. const DOMParser = DOM.getParser();
  663. const elementInfo = this.options.shadowRootContents[elementIndex];
  664. if (DOMParser && elementInfo) {
  665. const frameElement = this.doc.createElement("iframe");
  666. frameElement.setAttribute("style", "all:initial!important;border:0!important;width:100%!important;height:" + elementInfo.height + "px!important");
  667. const windowId = "shadow-" + this.options.framesData.length;
  668. frameElement.setAttribute(DOM.windowIdAttributeName(this.options.sessionId), windowId);
  669. this.options.framesData.push({ windowId, content: elementInfo.content, baseURI: this.baseURI });
  670. element.appendChild(frameElement);
  671. }
  672. });
  673. }
  674. }
  675. insertVideoPosters() {
  676. if (this.options.postersData) {
  677. this.doc.querySelectorAll("video[src], video > source[src]").forEach((element, videoIndex) => {
  678. let videoElement;
  679. if (element.tagName == "VIDEO") {
  680. videoElement = element;
  681. } else {
  682. videoElement = element.parentElement;
  683. }
  684. if (!videoElement.poster && this.options.postersData[videoIndex]) {
  685. videoElement.setAttribute("poster", this.options.postersData[videoIndex]);
  686. }
  687. });
  688. }
  689. }
  690. async processPageResources() {
  691. const processAttributeArgs = [
  692. ["link[href][rel*=\"icon\"]", "href", "data:", false, true],
  693. ["object[type=\"image/svg+xml\"], object[type=\"image/svg-xml\"]", "data", PREFIX_DATA_URI_IMAGE_SVG],
  694. ["img[src], input[src][type=image]", "src", PREFIX_DATA_URI_IMAGE, true],
  695. ["embed[src*=\".svg\"]", "src", PREFIX_DATA_URI_IMAGE_SVG],
  696. ["video[poster]", "poster", PREFIX_DATA_URI_IMAGE],
  697. ["*[background]", "background", PREFIX_DATA_URI_IMAGE],
  698. ["image", "xlink:href", PREFIX_DATA_URI_IMAGE]
  699. ];
  700. let resourcePromises = processAttributeArgs.map(([selector, attributeName, prefixDataURI, processDuplicates, removeElementIfMissing]) =>
  701. DomProcessorHelper.processAttribute(this.doc, this.doc.querySelectorAll(selector), attributeName, prefixDataURI, this.baseURI, this.options, this.cssVariables, this.styles, this.batchRequest, processDuplicates, removeElementIfMissing)
  702. );
  703. resourcePromises = resourcePromises.concat([
  704. DomProcessorHelper.processXLinks(this.doc.querySelectorAll("use"), this.baseURI, this.options, this.batchRequest),
  705. DomProcessorHelper.processSrcset(this.doc.querySelectorAll("img[srcset], source[srcset]"), "srcset", PREFIX_DATA_URI_IMAGE, this.baseURI, this.options, this.batchRequest)
  706. ]);
  707. if (!this.options.removeAudioSrc) {
  708. resourcePromises.push(DomProcessorHelper.processAttribute(this.doc, this.doc.querySelectorAll("audio[src], audio > source[src]"), "src", PREFIX_DATA_URI_AUDIO, this.baseURI, this.options, this.cssVariables, this.styles, this.batchRequest));
  709. }
  710. if (!this.options.removeVideoSrc) {
  711. resourcePromises.push(DomProcessorHelper.processAttribute(this.doc, this.doc.querySelectorAll("video[src], video > source[src]"), "src", PREFIX_DATA_URI_VIDEO, this.baseURI, this.options, this.cssVariables, this.styles, this.batchRequest));
  712. }
  713. await Promise.all(resourcePromises);
  714. DomProcessorHelper.processShortcutIcons(this.doc);
  715. }
  716. async resolveStylesheetURLs() {
  717. await Promise.all(Array.from(this.doc.querySelectorAll("style, link[rel*=stylesheet]"))
  718. .map(async element => {
  719. let mediaText;
  720. if (element.media) {
  721. mediaText = element.media;
  722. }
  723. this.stylesheets.set(element, { mediaText });
  724. const options = { maxResourceSize: this.options.maxResourceSize, maxResourceSizeEnabled: this.options.maxResourceSizeEnabled, url: this.options.url, charSet: this.charSet, compressCSS: this.options.compressCSS };
  725. const isLinkTag = element.tagName.toLowerCase() == "link";
  726. if (isLinkTag && element.rel.includes("alternate") && element.title) {
  727. element.remove();
  728. } else {
  729. let stylesheetContent;
  730. if (isLinkTag) {
  731. if (element.charset) {
  732. options.charSet = element.charset;
  733. }
  734. stylesheetContent = await DomProcessorHelper.resolveLinkStylesheetURLs(element.href, this.baseURI, options);
  735. } else {
  736. stylesheetContent = await DomProcessorHelper.resolveImportURLs(element.textContent, this.baseURI, options);
  737. }
  738. let stylesheet;
  739. try {
  740. stylesheet = cssTree.parse(stylesheetContent);
  741. } catch (error) {
  742. // ignored
  743. }
  744. if (stylesheet && stylesheet.children) {
  745. if (this.options.compressCSS) {
  746. const removedRules = [];
  747. for (let cssRule = stylesheet.children.head; cssRule; cssRule = cssRule.next) {
  748. if (cssRule.data.type == "Raw" && cssRule.data.value && cssRule.data.value.trim().startsWith("//")) {
  749. removedRules.push(cssRule);
  750. }
  751. }
  752. removedRules.forEach(cssRule => stylesheet.children.remove(cssRule));
  753. }
  754. this.stylesheets.get(element).stylesheet = stylesheet;
  755. }
  756. }
  757. }));
  758. }
  759. async processStylesheets() {
  760. await Promise.all(Array.from(this.stylesheets).map(([, stylesheetInfo]) =>
  761. DomProcessorHelper.processStylesheet(stylesheetInfo.stylesheet.children, this.baseURI, this.options, this.cssVariables, this.batchRequest)
  762. ));
  763. }
  764. replaceStylesheets() {
  765. this.doc.querySelectorAll("style").forEach(styleElement => {
  766. const stylesheetInfo = this.stylesheets.get(styleElement);
  767. if (stylesheetInfo) {
  768. let stylesheetContent = cssTree.generate(stylesheetInfo.stylesheet);
  769. styleElement.textContent = stylesheetContent;
  770. if (stylesheetInfo.mediaText) {
  771. styleElement.media = stylesheetInfo.mediaText;
  772. }
  773. } else {
  774. styleElement.remove();
  775. }
  776. });
  777. this.doc.querySelectorAll("link[rel*=stylesheet]").forEach(linkElement => {
  778. const stylesheetInfo = this.stylesheets.get(linkElement);
  779. if (stylesheetInfo) {
  780. const styleElement = this.doc.createElement("style");
  781. if (stylesheetInfo.mediaText) {
  782. styleElement.media = stylesheetInfo.mediaText;
  783. }
  784. let stylesheetContent = cssTree.generate(stylesheetInfo.stylesheet);
  785. styleElement.textContent = stylesheetContent;
  786. linkElement.parentElement.replaceChild(styleElement, linkElement);
  787. } else {
  788. linkElement.remove();
  789. }
  790. });
  791. }
  792. insertVariables() {
  793. if (this.cssVariables.size) {
  794. const styleElement = this.doc.createElement("style");
  795. const firstStyleElement = this.doc.head.querySelector("style");
  796. if (firstStyleElement) {
  797. this.doc.head.insertBefore(styleElement, firstStyleElement);
  798. } else {
  799. this.doc.head.appendChild(styleElement);
  800. }
  801. let stylesheetContent = "";
  802. this.cssVariables.forEach((content, indexResource) => {
  803. if (stylesheetContent) {
  804. stylesheetContent += ";";
  805. }
  806. stylesheetContent += `${SINGLE_FILE_VARIABLE_NAME_PREFIX + indexResource}:url("${content}")`;
  807. });
  808. styleElement.textContent = ":root{" + stylesheetContent + "}";
  809. }
  810. }
  811. async processScripts() {
  812. await Promise.all(Array.from(this.doc.querySelectorAll("script[src]")).map(async scriptElement => {
  813. if (scriptElement.src) {
  814. this.stats.add("processed", "scripts", 1);
  815. const scriptContent = await Download.getContent(scriptElement.src, { asDataURI: false, maxResourceSize: this.options.maxResourceSize, maxResourceSizeEnabled: this.options.maxResourceSizeEnabled });
  816. scriptElement.textContent = scriptContent.replace(/<\//gi, "<\\/").replace(/\/>/gi, "\\/>");
  817. }
  818. scriptElement.removeAttribute("src");
  819. }));
  820. }
  821. async resolveFrameURLs() {
  822. if (this.options.framesData) {
  823. const frameElements = Array.from(this.doc.querySelectorAll("iframe, frame, object[type=\"text/html\"][data]"));
  824. await Promise.all(frameElements.map(async frameElement => {
  825. DomProcessorHelper.setFrameEmptySrc(frameElement);
  826. const frameWindowId = frameElement.getAttribute(DOM.windowIdAttributeName(this.options.sessionId));
  827. if (frameWindowId) {
  828. const frameData = this.options.framesData.find(frame => frame.windowId == frameWindowId);
  829. if (frameData) {
  830. await initializeProcessor(frameData, frameElement, frameWindowId, this.batchRequest, Object.create(this.options));
  831. }
  832. }
  833. }));
  834. }
  835. async function initializeProcessor(frameData, frameElement, frameWindowId, batchRequest, options) {
  836. options.insertSingleFileComment = false;
  837. options.insertFaviconLink = false;
  838. options.doc = null;
  839. options.win = null;
  840. options.url = frameData.baseURI;
  841. options.windowId = frameWindowId;
  842. if (frameData.content) {
  843. options.content = frameData.content;
  844. options.canvasData = frameData.canvasData;
  845. options.stylesheetContents = frameData.stylesheetContents;
  846. options.fontsData = frameData.fontsData;
  847. options.imageData = frameData.imageData;
  848. options.usedFonts = frameData.usedFonts;
  849. frameData.processor = new PageProcessor(options);
  850. frameData.frameElement = frameElement;
  851. await frameData.processor.loadPage();
  852. await frameData.processor.initialize();
  853. frameData.maxResources = batchRequest.getMaxResources();
  854. }
  855. }
  856. }
  857. async processFrames() {
  858. if (this.options.framesData) {
  859. const frameElements = Array.from(this.doc.querySelectorAll("iframe, frame, object[type=\"text/html\"][data]"));
  860. await Promise.all(frameElements.map(async frameElement => {
  861. const frameWindowId = frameElement.getAttribute(DOM.windowIdAttributeName(this.options.sessionId));
  862. if (frameWindowId) {
  863. const frameData = this.options.framesData.find(frame => frame.windowId == frameWindowId);
  864. if (frameData) {
  865. if (frameData.processor) {
  866. this.stats.add("processed", "frames", 1);
  867. await frameData.processor.run();
  868. const pageData = await frameData.processor.getPageData();
  869. frameElement.removeAttribute(DOM.windowIdAttributeName(this.options.sessionId));
  870. if (pageData.content.match(NOSCRIPT_TAG_FOUND) || pageData.content.match(SCRIPT_TAG_FOUND)) {
  871. frameElement.setAttribute("sandbox", "allow-scripts allow-same-origin");
  872. } else {
  873. frameElement.setAttribute("sandbox", "");
  874. }
  875. DomProcessorHelper.setFrameContent(frameElement, pageData.content);
  876. this.stats.addAll(pageData);
  877. } else {
  878. this.stats.add("discarded", "frames", 1);
  879. }
  880. }
  881. }
  882. }));
  883. }
  884. }
  885. async resolveHtmlImportURLs() {
  886. const linkElements = Array.from(this.doc.querySelectorAll("link[rel=import][href]"));
  887. if (!this.relImportProcessors) {
  888. this.relImportProcessors = new Map();
  889. }
  890. await Promise.all(linkElements.map(async linkElement => {
  891. const resourceURL = linkElement.href;
  892. linkElement.removeAttribute("href");
  893. const options = Object.create(this.options);
  894. options.insertSingleFileComment = false;
  895. options.insertFaviconLink = false;
  896. options.doc = null;
  897. options.win = null;
  898. options.url = resourceURL;
  899. if (!DomUtil.testIgnoredPath(resourceURL) && DomUtil.testValidPath(resourceURL, this.baseURI, this.options.url)) {
  900. const processor = new PageProcessor(options);
  901. this.relImportProcessors.set(linkElement, processor);
  902. await processor.loadPage();
  903. return processor.initialize();
  904. }
  905. }));
  906. }
  907. async processHtmlImports() {
  908. const linkElements = Array.from(this.doc.querySelectorAll("link[rel=import][href]"));
  909. await Promise.all(linkElements.map(async linkElement => {
  910. const processor = this.relImportProcessors.get(linkElement);
  911. if (processor) {
  912. this.stats.add("processed", "HTML imports", 1);
  913. this.relImportProcessors.delete(linkElement);
  914. const pageData = await processor.getPageData();
  915. linkElement.setAttribute("href", "data:text/html," + pageData.content);
  916. this.stats.addAll(pageData);
  917. } else {
  918. this.stats.add("discarded", "HTML imports", 1);
  919. }
  920. }));
  921. }
  922. resolveStyleAttributeURLs() {
  923. this.doc.querySelectorAll("[style]").forEach(element => {
  924. let styleContent = element.getAttribute("style");
  925. if (this.options.compressCSS) {
  926. styleContent = DOM.compressCSS(styleContent);
  927. }
  928. styleContent = DomProcessorHelper.resolveStylesheetURLs(styleContent, this.baseURI, this.options);
  929. const declarationList = cssTree.parse(styleContent, { context: "declarationList" });
  930. this.styles.set(element, declarationList);
  931. });
  932. }
  933. async processStyleAttributes() {
  934. return Promise.all(Array.from(this.styles).map(([, declarationList]) =>
  935. DomProcessorHelper.processStyle(declarationList.children.toArray(), this.baseURI, this.options, this.cssVariables, this.batchRequest)
  936. ));
  937. }
  938. replaceStyleAttributes() {
  939. this.doc.querySelectorAll("[style]").forEach(async element => {
  940. const declarations = this.styles.get(element);
  941. if (declarations) {
  942. let styleContent = cssTree.generate(declarations);
  943. element.setAttribute("style", styleContent);
  944. } else {
  945. element.setAttribute("style", "");
  946. }
  947. });
  948. }
  949. }
  950. // ---------
  951. // DomHelper
  952. // ---------
  953. const REGEXP_AMP = /&/g;
  954. const REGEXP_NBSP = /\u00a0/g;
  955. const REGEXP_START_TAG = /</g;
  956. const REGEXP_END_TAG = />/g;
  957. const REGEXP_URL_HASH = /(#.+?)$/;
  958. const PREFIX_DATA_URI_IMAGE = "data:image/";
  959. const PREFIX_DATA_URI_AUDIO = "data:audio/";
  960. const PREFIX_DATA_URI_VIDEO = "data:video/";
  961. const PREFIX_DATA_URI_IMAGE_SVG = "data:image/svg+xml";
  962. const PREFIX_DATA_URI_NO_MIMETYPE = "data:;";
  963. const PREFIX_DATA_URI_OCTET_STREAM = /^data:(application|binary)\/octet-stream/;
  964. const PREFIX_DATA_URI_NULL_STREAM = /^data:null;/;
  965. const PREFIX_DATA_URI_VND = "data:application/vnd.";
  966. const SINGLE_FILE_VARIABLE_NAME_PREFIX = "--sf-img-";
  967. class DomProcessorHelper {
  968. static async evalTemplate(template, options, content, dontReplaceSlash) {
  969. const date = new Date();
  970. const url = new URL(options.url);
  971. template = await DomUtil.evalTemplateVariable(template, "page-title", () => options.title || "No title", dontReplaceSlash);
  972. template = await DomUtil.evalTemplateVariable(template, "page-language", () => options.info.lang || "No language", dontReplaceSlash);
  973. template = await DomUtil.evalTemplateVariable(template, "page-description", () => options.info.description || "No description", dontReplaceSlash);
  974. template = await DomUtil.evalTemplateVariable(template, "page-author", () => options.info.author || "No author", dontReplaceSlash);
  975. template = await DomUtil.evalTemplateVariable(template, "page-creator", () => options.info.creator || "No creator", dontReplaceSlash);
  976. template = await DomUtil.evalTemplateVariable(template, "page-publisher", () => options.info.publisher || "No publisher", dontReplaceSlash);
  977. template = await DomUtil.evalTemplateVariable(template, "datetime-iso", () => date.toISOString(), dontReplaceSlash);
  978. template = await DomUtil.evalTemplateVariable(template, "date-iso", () => date.toISOString().split("T")[0], dontReplaceSlash);
  979. template = await DomUtil.evalTemplateVariable(template, "time-iso", () => date.toISOString().split("T")[1].split("Z")[0], dontReplaceSlash);
  980. template = await DomUtil.evalTemplateVariable(template, "date-locale", () => date.toLocaleDateString(), dontReplaceSlash);
  981. template = await DomUtil.evalTemplateVariable(template, "time-locale", () => date.toLocaleTimeString(), dontReplaceSlash);
  982. template = await DomUtil.evalTemplateVariable(template, "day-locale", () => String(date.getDate()).padStart(2, "0"), dontReplaceSlash);
  983. template = await DomUtil.evalTemplateVariable(template, "month-locale", () => String(date.getMonth() + 1).padStart(2, "0"), dontReplaceSlash);
  984. template = await DomUtil.evalTemplateVariable(template, "year-locale", () => String(date.getFullYear()), dontReplaceSlash);
  985. template = await DomUtil.evalTemplateVariable(template, "datetime-locale", () => date.toLocaleString(), dontReplaceSlash);
  986. template = await DomUtil.evalTemplateVariable(template, "datetime-utc", () => date.toUTCString(), dontReplaceSlash);
  987. template = await DomUtil.evalTemplateVariable(template, "day-utc", () => String(date.getUTCDate()).padStart(2, "0"), dontReplaceSlash);
  988. template = await DomUtil.evalTemplateVariable(template, "month-utc", () => String(date.getUTCMonth() + 1).padStart(2, "0"), dontReplaceSlash);
  989. template = await DomUtil.evalTemplateVariable(template, "year-utc", () => String(date.getUTCFullYear()), dontReplaceSlash);
  990. template = await DomUtil.evalTemplateVariable(template, "hours-locale", () => String(date.getHours()).padStart(2, "0"), dontReplaceSlash);
  991. template = await DomUtil.evalTemplateVariable(template, "minutes-locale", () => String(date.getMinutes()).padStart(2, "0"), dontReplaceSlash);
  992. template = await DomUtil.evalTemplateVariable(template, "seconds-locale", () => String(date.getSeconds()).padStart(2, "0"), dontReplaceSlash);
  993. template = await DomUtil.evalTemplateVariable(template, "hours-utc", () => String(date.getUTCHours()).padStart(2, "0"), dontReplaceSlash);
  994. template = await DomUtil.evalTemplateVariable(template, "minutes-utc", () => String(date.getUTCMinutes()).padStart(2, "0"), dontReplaceSlash);
  995. template = await DomUtil.evalTemplateVariable(template, "seconds-utc", () => String(date.getUTCSeconds()).padStart(2, "0"), dontReplaceSlash);
  996. template = await DomUtil.evalTemplateVariable(template, "url-hash", () => url.hash.substring(1), dontReplaceSlash);
  997. template = await DomUtil.evalTemplateVariable(template, "url-host", () => url.host.replace(/\/$/, ""), dontReplaceSlash);
  998. template = await DomUtil.evalTemplateVariable(template, "url-hostname", () => url.hostname.replace(/\/$/, ""), dontReplaceSlash);
  999. template = await DomUtil.evalTemplateVariable(template, "url-href", () => url.href, dontReplaceSlash);
  1000. template = await DomUtil.evalTemplateVariable(template, "url-password", () => url.password, dontReplaceSlash);
  1001. template = await DomUtil.evalTemplateVariable(template, "url-pathname", () => url.pathname.replace(/^\//, "").replace(/\/$/, ""), dontReplaceSlash === undefined ? true : dontReplaceSlash);
  1002. template = await DomUtil.evalTemplateVariable(template, "url-port", () => url.port, dontReplaceSlash);
  1003. template = await DomUtil.evalTemplateVariable(template, "url-protocol", () => url.protocol, dontReplaceSlash);
  1004. template = await DomUtil.evalTemplateVariable(template, "url-search", () => url.search.substring(1), dontReplaceSlash);
  1005. template = await DomUtil.evalTemplateVariable(template, "url-username", () => url.username, dontReplaceSlash);
  1006. template = await DomUtil.evalTemplateVariable(template, "tab-id", () => String(options.tabId || "No tab id"), dontReplaceSlash);
  1007. template = await DomUtil.evalTemplateVariable(template, "url-last-segment", () => DomUtil.getLastSegment(url), dontReplaceSlash);
  1008. if (content) {
  1009. template = await DomUtil.evalTemplateVariable(template, "digest-sha-256", async () => DOM.digest("SHA-256", content), dontReplaceSlash);
  1010. template = await DomUtil.evalTemplateVariable(template, "digest-sha-384", async () => DOM.digest("SHA-384", content), dontReplaceSlash);
  1011. template = await DomUtil.evalTemplateVariable(template, "digest-sha-512", async () => DOM.digest("SHA-512", content), dontReplaceSlash);
  1012. }
  1013. return template;
  1014. }
  1015. static setBackgroundImage(element, url, style) {
  1016. element.style.setProperty("background-blend-mode", "normal", "important");
  1017. element.style.setProperty("background-clip", "content-box", "important");
  1018. element.style.setProperty("background-position", style && style["background-position"] ? style["background-position"] : "center", "important");
  1019. element.style.setProperty("background-color", style && style["background-color"] ? style["background-color"] : "transparent", "important");
  1020. element.style.setProperty("background-image", url, "important");
  1021. element.style.setProperty("background-size", style && style["background-size"] ? style["background-size"] : "100% 100%", "important");
  1022. element.style.setProperty("background-origin", "content-box", "important");
  1023. element.style.setProperty("background-repeat", "no-repeat", "important");
  1024. }
  1025. static processShortcutIcons(doc) {
  1026. let shortcutIcon = DomUtil.findShortcutIcon(Array.from(doc.querySelectorAll("link[href][rel=\"icon\"], link[href][rel=\"shortcut icon\"]")));
  1027. if (!shortcutIcon) {
  1028. shortcutIcon = DomUtil.findShortcutIcon(Array.from(doc.querySelectorAll("link[href][rel*=\"icon\"]")));
  1029. if (shortcutIcon) {
  1030. shortcutIcon.rel = "icon";
  1031. }
  1032. }
  1033. if (shortcutIcon) {
  1034. doc.querySelectorAll("link[href][rel*=\"icon\"]").forEach(linkElement => {
  1035. if (linkElement != shortcutIcon) {
  1036. linkElement.remove();
  1037. }
  1038. });
  1039. }
  1040. }
  1041. static setFrameEmptySrc(frameElement) {
  1042. if (frameElement.tagName == "OBJECT") {
  1043. frameElement.setAttribute("data", "data:text/html,");
  1044. } else {
  1045. frameElement.removeAttribute("src");
  1046. frameElement.removeAttribute("srcdoc");
  1047. }
  1048. }
  1049. static setFrameContent(frameElement, content) {
  1050. if (frameElement.tagName == "OBJECT") {
  1051. frameElement.setAttribute("data", "data:text/html," + content);
  1052. } else {
  1053. frameElement.setAttribute("srcdoc", content);
  1054. frameElement.removeAttribute("src");
  1055. }
  1056. }
  1057. static isolateElements(rootElement) {
  1058. rootElement.querySelectorAll("*:not(style)").forEach(element => {
  1059. if (element.getAttribute(SELECTED_CONTENT_ATTRIBUTE_NAME) == "") {
  1060. element.removeAttribute(SELECTED_CONTENT_ATTRIBUTE_NAME);
  1061. } else if (!element.querySelector("[" + SELECTED_CONTENT_ATTRIBUTE_NAME + "]")) {
  1062. element.remove();
  1063. }
  1064. });
  1065. isolateParentElements(rootElement.parentElement, rootElement);
  1066. function isolateParentElements(parentElement, element) {
  1067. let elementFound = false;
  1068. Array.from(parentElement.childNodes).forEach(node => {
  1069. elementFound = elementFound || (node == element);
  1070. const tagName = node.tagName && node.tagName.toLowerCase();
  1071. if (node != element && tagName != "svg" && tagName != "style" && tagName != "link") {
  1072. if (elementFound) {
  1073. node.remove();
  1074. } else {
  1075. node.hidden = true;
  1076. if (node.childNodes && node.childNodes.length) {
  1077. Array.from(node.childNodes).forEach(node => node.remove());
  1078. }
  1079. }
  1080. }
  1081. });
  1082. element = element.parentElement;
  1083. if (element && element.parentElement && element.parentElement.tagName != "HTML") {
  1084. isolateParentElements(element.parentElement, element);
  1085. }
  1086. }
  1087. }
  1088. static async resolveImportURLs(stylesheetContent, baseURI, options) {
  1089. stylesheetContent = DomProcessorHelper.resolveStylesheetURLs(stylesheetContent, baseURI, options);
  1090. const imports = DomUtil.getImportFunctions(stylesheetContent);
  1091. await Promise.all(imports.map(async cssImport => {
  1092. const match = DomUtil.matchImport(cssImport);
  1093. if (match) {
  1094. let resourceURL = DomUtil.normalizeURL(match.resourceURL);
  1095. if (!DomUtil.testIgnoredPath(resourceURL) && DomUtil.testValidPath(resourceURL, baseURI, options.url)) {
  1096. try {
  1097. resourceURL = new URL(match.resourceURL, baseURI).href;
  1098. } catch (error) {
  1099. // ignored
  1100. }
  1101. if (DomUtil.testValidURL(resourceURL, baseURI, options.url)) {
  1102. const downloadOptions = { asDataURI: false, maxResourceSize: options.maxResourceSize, maxResourceSizeEnabled: options.maxResourceSizeEnabled, validateTextContentType: true };
  1103. let importedStylesheetContent = await Download.getContent(resourceURL, downloadOptions);
  1104. importedStylesheetContent = DomUtil.removeCssComments(importedStylesheetContent);
  1105. if (options.compressCSS) {
  1106. importedStylesheetContent = DOM.compressCSS(importedStylesheetContent);
  1107. }
  1108. importedStylesheetContent = DomUtil.wrapMediaQuery(importedStylesheetContent, match.media);
  1109. if (stylesheetContent.includes(cssImport)) {
  1110. importedStylesheetContent = await DomProcessorHelper.resolveImportURLs(importedStylesheetContent, resourceURL, options);
  1111. stylesheetContent = stylesheetContent.replace(DomUtil.getRegExp(cssImport), importedStylesheetContent);
  1112. }
  1113. }
  1114. }
  1115. }
  1116. }));
  1117. return stylesheetContent;
  1118. }
  1119. static resolveStylesheetURLs(stylesheetContent, baseURI, options) {
  1120. const urlFunctions = DomUtil.getUrlFunctions(stylesheetContent);
  1121. urlFunctions.map(urlFunction => {
  1122. const originalResourceURL = DomUtil.matchURL(urlFunction);
  1123. const resourceURL = DomUtil.normalizeURL(originalResourceURL);
  1124. if (!DomUtil.testIgnoredPath(resourceURL)) {
  1125. if (DomUtil.testValidPath(resourceURL, baseURI, options.url)) {
  1126. let resolvedURL;
  1127. try {
  1128. resolvedURL = new URL(resourceURL, baseURI).href;
  1129. } catch (error) {
  1130. // ignored
  1131. }
  1132. if (DomUtil.testValidURL(resolvedURL, baseURI, options.url) && resourceURL != resolvedURL && stylesheetContent.includes(urlFunction)) {
  1133. stylesheetContent = stylesheetContent.replace(DomUtil.getRegExp(urlFunction), urlFunction.replace(originalResourceURL, resolvedURL));
  1134. }
  1135. } else {
  1136. let newUrlFunction;
  1137. if (originalResourceURL) {
  1138. newUrlFunction = urlFunction.replace(originalResourceURL, EMPTY_DATA_URI);
  1139. } else {
  1140. newUrlFunction = "url(" + EMPTY_DATA_URI + ")";
  1141. }
  1142. stylesheetContent = stylesheetContent.replace(DomUtil.getRegExp(urlFunction), newUrlFunction);
  1143. }
  1144. } else {
  1145. if (resourceURL.startsWith(DATA_URI_PREFIX)) {
  1146. const escapedResourceURL = resourceURL.replace(REGEXP_AMP, "&amp;").replace(REGEXP_NBSP, "&nbsp;").replace(REGEXP_START_TAG, "&lt;").replace(REGEXP_END_TAG, "&gt;");
  1147. if (escapedResourceURL != resourceURL && stylesheetContent.includes(urlFunction)) {
  1148. stylesheetContent = stylesheetContent.replace(DomUtil.getRegExp(urlFunction), urlFunction.replace(originalResourceURL, escapedResourceURL));
  1149. }
  1150. }
  1151. }
  1152. });
  1153. return stylesheetContent;
  1154. }
  1155. static async resolveLinkStylesheetURLs(resourceURL, baseURI, options) {
  1156. resourceURL = DomUtil.normalizeURL(resourceURL);
  1157. if (resourceURL && resourceURL != baseURI && resourceURL != ABOUT_BLANK_URI) {
  1158. const downloadOptions = { asDataURI: false, maxResourceSize: options.maxResourceSize, maxResourceSizeEnabled: options.maxResourceSizeEnabled, charSet: options.charSet };
  1159. let stylesheetContent = await Download.getContent(resourceURL, downloadOptions);
  1160. stylesheetContent = DomUtil.removeCssComments(stylesheetContent);
  1161. if (options.compressCSS) {
  1162. stylesheetContent = DOM.compressCSS(stylesheetContent);
  1163. }
  1164. stylesheetContent = await DomProcessorHelper.resolveImportURLs(stylesheetContent, resourceURL, options);
  1165. return stylesheetContent;
  1166. }
  1167. }
  1168. static async processStylesheet(cssRules, baseURI, options, cssVariables, batchRequest) {
  1169. const promises = [];
  1170. const removedRules = [];
  1171. for (let cssRule = cssRules.head; cssRule; cssRule = cssRule.next) {
  1172. const ruleData = cssRule.data;
  1173. if (ruleData.type == "Atrule" && ruleData.name == "charset") {
  1174. removedRules.push(cssRule);
  1175. } else if (ruleData.block && ruleData.block.children) {
  1176. if (ruleData.type == "Rule") {
  1177. promises.push(this.processStyle(ruleData.block.children.toArray(), baseURI, options, cssVariables, batchRequest));
  1178. } else if (ruleData.type == "Atrule" && (ruleData.name == "media" || ruleData.name == "supports")) {
  1179. promises.push(this.processStylesheet(ruleData.block.children, baseURI, options, cssVariables, batchRequest));
  1180. } else if (ruleData.type == "Atrule" && ruleData.name == "font-face") {
  1181. promises.push(processFontFaceRule(ruleData));
  1182. }
  1183. }
  1184. }
  1185. removedRules.forEach(cssRule => cssRules.remove(cssRule));
  1186. await Promise.all(promises);
  1187. async function processFontFaceRule(cssRule) {
  1188. await Promise.all(cssRule.block.children.toArray().map(async declaration => {
  1189. if (declaration.type == "Declaration" && declaration.value.children) {
  1190. const urlFunctions = DomUtil.getUrlFunctions(DomUtil.getCSSValue(declaration.value));
  1191. await Promise.all(urlFunctions.map(async urlFunction => {
  1192. const originalResourceURL = DomUtil.matchURL(urlFunction);
  1193. const resourceURL = DomUtil.normalizeURL(originalResourceURL);
  1194. if (!DomUtil.testIgnoredPath(resourceURL)) {
  1195. if (DomUtil.testValidURL(resourceURL, baseURI, options.url)) {
  1196. let { content } = await batchRequest.addURL(resourceURL);
  1197. let validResource = content == EMPTY_DATA_URI || content.startsWith(PREFIX_DATA_URI_VND) || content.startsWith(PREFIX_DATA_URI_IMAGE_SVG);
  1198. if (!validResource) {
  1199. validResource = await DOM.validFont(urlFunction);
  1200. }
  1201. if (!validResource) {
  1202. content = EMPTY_DATA_URI;
  1203. }
  1204. declaration.value.children.forEach(token => {
  1205. if (token.type == "Url" && DOM.removeQuotes(DomUtil.getCSSValue(token.value)) == originalResourceURL) {
  1206. token.value.value = content;
  1207. }
  1208. });
  1209. }
  1210. }
  1211. }));
  1212. }
  1213. }));
  1214. }
  1215. }
  1216. static async processStyle(declarations, baseURI, options, cssVariables, batchRequest) {
  1217. await Promise.all(declarations.map(async declaration => {
  1218. if (declaration.type == "Declaration" && declaration.value.children) {
  1219. const urlFunctions = DomUtil.getUrlFunctions(DomUtil.getCSSValue(declaration.value));
  1220. await Promise.all(urlFunctions.map(async urlFunction => {
  1221. const originalResourceURL = DomUtil.matchURL(urlFunction);
  1222. const resourceURL = DomUtil.normalizeURL(originalResourceURL);
  1223. if (!DomUtil.testIgnoredPath(resourceURL)) {
  1224. if (DomUtil.testValidURL(resourceURL, baseURI, options.url)) {
  1225. let { content, indexResource, duplicate } = await batchRequest.addURL(resourceURL);
  1226. if (duplicate && options.groupDuplicateImages) {
  1227. const tokens = [];
  1228. findURLToken(originalResourceURL, declaration.value.children, (token, parent) => {
  1229. const value = cssTree.parse("var(" + SINGLE_FILE_VARIABLE_NAME_PREFIX + indexResource + ")", { context: "value" }).children.head;
  1230. tokens.push({ parent, token, value });
  1231. });
  1232. cssVariables.set(indexResource, content);
  1233. tokens.forEach(({ parent, token, value }) => parent.replace(token, value));
  1234. } else {
  1235. findURLToken(originalResourceURL, declaration.value.children, token => token.data.value.value = content);
  1236. }
  1237. }
  1238. }
  1239. }));
  1240. }
  1241. }));
  1242. function findURLToken(url, children, callback) {
  1243. for (let token = children.head; token; token = token.next) {
  1244. if (token.data.children) {
  1245. findURLToken(url, token.data.children, callback);
  1246. }
  1247. if (token.data.type == "Url" && DOM.removeQuotes(DomUtil.getCSSValue(token.data.value)) == url) {
  1248. callback(token, children);
  1249. }
  1250. }
  1251. }
  1252. }
  1253. static async processAttribute(doc, resourceElements, attributeName, prefixDataURI, baseURI, options, cssVariables, styles, batchRequest, processDuplicates, removeElementIfMissing) {
  1254. await Promise.all(Array.from(resourceElements).map(async resourceElement => {
  1255. let resourceURL = resourceElement.getAttribute(attributeName);
  1256. resourceURL = DomUtil.normalizeURL(resourceURL);
  1257. if (!DomUtil.testIgnoredPath(resourceURL)) {
  1258. resourceElement.setAttribute(attributeName, EMPTY_IMAGE);
  1259. if (DomUtil.testValidPath(resourceURL, baseURI, options.url)) {
  1260. try {
  1261. resourceURL = new URL(resourceURL, baseURI).href;
  1262. } catch (error) {
  1263. // ignored
  1264. }
  1265. if (DomUtil.testValidURL(resourceURL, baseURI, options.url)) {
  1266. const { content, indexResource, duplicate } = await batchRequest.addURL(resourceURL);
  1267. if (removeElementIfMissing && content == EMPTY_DATA_URI) {
  1268. resourceElement.remove();
  1269. } else {
  1270. if (content.startsWith(prefixDataURI) || content.startsWith(PREFIX_DATA_URI_NO_MIMETYPE) || content.match(PREFIX_DATA_URI_OCTET_STREAM) || content.match(PREFIX_DATA_URI_NULL_STREAM)) {
  1271. const isSVG = content.startsWith(PREFIX_DATA_URI_IMAGE_SVG);
  1272. if (processDuplicates && duplicate && options.groupDuplicateImages && !isSVG) {
  1273. if (DomUtil.replaceImageSource(resourceElement, SINGLE_FILE_VARIABLE_NAME_PREFIX + indexResource, options)) {
  1274. cssVariables.set(indexResource, content);
  1275. const declarationList = cssTree.parse(resourceElement.getAttribute("style"), { context: "declarationList" });
  1276. styles.set(resourceElement, declarationList);
  1277. } else {
  1278. resourceElement.setAttribute(attributeName, content);
  1279. }
  1280. } else {
  1281. resourceElement.setAttribute(attributeName, content);
  1282. }
  1283. }
  1284. }
  1285. }
  1286. }
  1287. }
  1288. }));
  1289. }
  1290. static async processXLinks(resourceElements, baseURI, options, batchRequest) {
  1291. const attributeName = "xlink:href";
  1292. await Promise.all(Array.from(resourceElements).map(async resourceElement => {
  1293. const originalResourceURL = resourceElement.getAttribute(attributeName);
  1294. let resourceURL = DomUtil.normalizeURL(originalResourceURL);
  1295. if (DomUtil.testValidPath(resourceURL, baseURI, options.url) && !DomUtil.testIgnoredPath(resourceURL)) {
  1296. resourceElement.setAttribute(attributeName, EMPTY_IMAGE);
  1297. try {
  1298. resourceURL = new URL(resourceURL, baseURI).href;
  1299. } catch (error) {
  1300. // ignored
  1301. }
  1302. if (DomUtil.testValidURL(resourceURL, baseURI, options.url)) {
  1303. try {
  1304. const { content } = await batchRequest.addURL(resourceURL, false);
  1305. const DOMParser = DOM.getParser();
  1306. if (DOMParser) {
  1307. const svgDoc = new DOMParser().parseFromString(content, "image/svg+xml");
  1308. const hashMatch = originalResourceURL.match(REGEXP_URL_HASH);
  1309. if (hashMatch && hashMatch[0]) {
  1310. const symbolElement = svgDoc.querySelector(hashMatch[0]);
  1311. if (symbolElement) {
  1312. resourceElement.setAttribute(attributeName, hashMatch[0]);
  1313. resourceElement.parentElement.insertBefore(symbolElement, resourceElement.parentElement.firstChild);
  1314. }
  1315. } else {
  1316. resourceElement.setAttribute(attributeName, "data:image/svg+xml," + content);
  1317. }
  1318. } else {
  1319. resourceElement.setAttribute(attributeName, "data:image/svg+xml," + content);
  1320. }
  1321. } catch (error) {
  1322. /* ignored */
  1323. }
  1324. }
  1325. } else if (resourceURL == options.url) {
  1326. resourceElement.setAttribute(attributeName, originalResourceURL.substring(resourceURL.length));
  1327. }
  1328. }));
  1329. }
  1330. static async processSrcset(resourceElements, attributeName, prefixDataURI, baseURI, options, batchRequest) {
  1331. await Promise.all(Array.from(resourceElements).map(async resourceElement => {
  1332. const srcset = DOM.parseSrcset(resourceElement.getAttribute(attributeName));
  1333. const srcsetValues = await Promise.all(srcset.map(async srcsetValue => {
  1334. let resourceURL = DomUtil.normalizeURL(srcsetValue.url);
  1335. if (!DomUtil.testIgnoredPath(resourceURL)) {
  1336. if (DomUtil.testValidPath(resourceURL, baseURI, options.url)) {
  1337. try {
  1338. resourceURL = new URL(resourceURL, baseURI).href;
  1339. } catch (error) {
  1340. // ignored
  1341. }
  1342. if (DomUtil.testValidURL(resourceURL, baseURI, options.url)) {
  1343. const { content } = await batchRequest.addURL(resourceURL);
  1344. if (!content.startsWith(prefixDataURI) && !content.startsWith(PREFIX_DATA_URI_NO_MIMETYPE) && !content.match(PREFIX_DATA_URI_OCTET_STREAM) && !content.match(PREFIX_DATA_URI_NULL_STREAM)) {
  1345. resourceElement.setAttribute(attributeName, EMPTY_IMAGE);
  1346. }
  1347. return content + (srcsetValue.w ? " " + srcsetValue.w + "w" : srcsetValue.d ? " " + srcsetValue.d + "x" : "");
  1348. } else {
  1349. return "";
  1350. }
  1351. } else {
  1352. return "";
  1353. }
  1354. } else {
  1355. return resourceURL + (srcsetValue.w ? " " + srcsetValue.w + "w" : srcsetValue.d ? " " + srcsetValue.d + "x" : "");
  1356. }
  1357. }));
  1358. resourceElement.setAttribute(attributeName, srcsetValues.join(", "));
  1359. }));
  1360. }
  1361. }
  1362. // -------
  1363. // DomUtil
  1364. // -------
  1365. const DATA_URI_PREFIX = "data:";
  1366. const BLOB_URI_PREFIX = "blob:";
  1367. const HTTP_URI_PREFIX = /^https?:\/\//;
  1368. const FILE_URI_PREFIX = /^file:\/\//;
  1369. const EMPTY_URL = /^https?:\/\/+\s*$/;
  1370. const ABOUT_BLANK_URI = "about:blank";
  1371. const NOT_EMPTY_URL = /^(https?:\/\/|file:\/\/|blob:).+/;
  1372. const REGEXP_URL_FN = /(url\s*\(\s*'(.*?)'\s*\))|(url\s*\(\s*"(.*?)"\s*\))|(url\s*\(\s*(.*?)\s*\))/gi;
  1373. const REGEXP_URL_SIMPLE_QUOTES_FN = /^url\s*\(\s*'(.*?)'\s*\)$/i;
  1374. const REGEXP_URL_DOUBLE_QUOTES_FN = /^url\s*\(\s*"(.*?)"\s*\)$/i;
  1375. const REGEXP_URL_NO_QUOTES_FN = /^url\s*\(\s*(.*?)\s*\)$/i;
  1376. const REGEXP_IMPORT_FN = /(@import\s*url\s*\(\s*'(.*?)'\s*\)\s*(.*?)(;|$|}))|(@import\s*url\s*\(\s*"(.*?)"\s*\)\s*(.*?)(;|$|}))|(@import\s*url\s*\(\s*(.*?)\s*\)\s*(.*?)(;|$|}))|(@import\s*'(.*?)'\s*(.*?)(;|$|}))|(@import\s*"(.*?)"\s*(.*?)(;|$|}))|(@import\s*(.*?)\s*(.*?)(;|$|}))/gi;
  1377. const REGEXP_IMPORT_URL_SIMPLE_QUOTES_FN = /@import\s*url\s*\(\s*'(.*?)'\s*\)\s*(.*?)(;|$|})/i;
  1378. const REGEXP_IMPORT_URL_DOUBLE_QUOTES_FN = /@import\s*url\s*\(\s*"(.*?)"\s*\)\s*(.*?)(;|$|})/i;
  1379. const REGEXP_IMPORT_URL_NO_QUOTES_FN = /@import\s*url\s*\(\s*(.*?)\s*\)\s*(.*?)(;|$|})/i;
  1380. const REGEXP_IMPORT_SIMPLE_QUOTES_FN = /@import\s*'(.*?)'\s*(.*?)(;|$|})/i;
  1381. const REGEXP_IMPORT_DOUBLE_QUOTES_FN = /@import\s*"(.*?)"\s*(.*?)(;|$|})/i;
  1382. const REGEXP_IMPORT_NO_QUOTES_FN = /@import\s*(.*?)\s*(.*?)(;|$|})/i;
  1383. const REGEXP_ESCAPE = /([{}()^$&.*?/+|[\\\\]|\]|-)/g;
  1384. class DomUtil {
  1385. static normalizeURL(url) {
  1386. if (!url || url.startsWith(DATA_URI_PREFIX)) {
  1387. return url;
  1388. } else {
  1389. return url.split("#")[0];
  1390. }
  1391. }
  1392. static getCSSValue(value) {
  1393. let result = "";
  1394. try {
  1395. result = cssTree.generate(value);
  1396. } catch (error) {
  1397. // ignored
  1398. }
  1399. return result;
  1400. }
  1401. static async evalTemplateVariable(template, variableName, valueGetter, dontReplaceSlash) {
  1402. const replaceRegExp = new RegExp("{\\s*" + variableName + "\\s*}", "g");
  1403. if (template.match(replaceRegExp)) {
  1404. let value = await valueGetter();
  1405. if (!dontReplaceSlash) {
  1406. value = value.replace(/\/+/g, "_");
  1407. }
  1408. return template.replace(replaceRegExp, value);
  1409. }
  1410. return template;
  1411. }
  1412. static getLastSegment(url) {
  1413. let lastSegmentMatch = url.pathname.match(/\/([^/]+)$/), lastSegment = lastSegmentMatch && lastSegmentMatch[0];
  1414. if (!lastSegment) {
  1415. lastSegmentMatch = url.href.match(/([^/]+)\/?$/);
  1416. lastSegment = lastSegmentMatch && lastSegmentMatch[0];
  1417. }
  1418. if (!lastSegment) {
  1419. lastSegmentMatch = lastSegment.match(/(.*)<\.[^.]+$/);
  1420. lastSegment = lastSegmentMatch && lastSegmentMatch[0];
  1421. }
  1422. if (!lastSegment) {
  1423. lastSegment = url.hostname.replace(/\/+/g, "_").replace(/\/$/, "");
  1424. }
  1425. lastSegment.replace(/\/$/, "").replace(/^\//, "");
  1426. return lastSegment;
  1427. }
  1428. static getRegExp(string) {
  1429. return new RegExp(string.replace(REGEXP_ESCAPE, "\\$1"), "gi");
  1430. }
  1431. static getUrlFunctions(stylesheetContent) {
  1432. return Array.from(new Set(stylesheetContent.match(REGEXP_URL_FN) || []));
  1433. }
  1434. static getImportFunctions(stylesheetContent) {
  1435. return stylesheetContent.match(REGEXP_IMPORT_FN) || [];
  1436. }
  1437. static findShortcutIcon(shortcutIcons) {
  1438. shortcutIcons = shortcutIcons.filter(linkElement => linkElement.href != EMPTY_IMAGE);
  1439. shortcutIcons.sort((linkElement1, linkElement2) => (parseInt(linkElement2.sizes, 10) || 16) - (parseInt(linkElement1.sizes, 10) || 16));
  1440. return shortcutIcons[0];
  1441. }
  1442. static matchURL(stylesheetContent) {
  1443. const match = stylesheetContent.match(REGEXP_URL_SIMPLE_QUOTES_FN) ||
  1444. stylesheetContent.match(REGEXP_URL_DOUBLE_QUOTES_FN) ||
  1445. stylesheetContent.match(REGEXP_URL_NO_QUOTES_FN);
  1446. return match && match[1];
  1447. }
  1448. static testIgnoredPath(resourceURL) {
  1449. return resourceURL && (resourceURL.startsWith(DATA_URI_PREFIX) || resourceURL == ABOUT_BLANK_URI);
  1450. }
  1451. static testValidPath(resourceURL, baseURI, docURL) {
  1452. return resourceURL && resourceURL != baseURI && resourceURL != docURL && !resourceURL.match(EMPTY_URL);
  1453. }
  1454. static testValidURL(resourceURL, baseURI, docURL) {
  1455. return DomUtil.testValidPath(resourceURL, baseURI, docURL) && (resourceURL.match(HTTP_URI_PREFIX) || resourceURL.match(FILE_URI_PREFIX) || resourceURL.startsWith(BLOB_URI_PREFIX)) && resourceURL.match(NOT_EMPTY_URL);
  1456. }
  1457. static matchImport(stylesheetContent) {
  1458. const match = stylesheetContent.match(REGEXP_IMPORT_URL_SIMPLE_QUOTES_FN) ||
  1459. stylesheetContent.match(REGEXP_IMPORT_URL_DOUBLE_QUOTES_FN) ||
  1460. stylesheetContent.match(REGEXP_IMPORT_URL_NO_QUOTES_FN) ||
  1461. stylesheetContent.match(REGEXP_IMPORT_SIMPLE_QUOTES_FN) ||
  1462. stylesheetContent.match(REGEXP_IMPORT_DOUBLE_QUOTES_FN) ||
  1463. stylesheetContent.match(REGEXP_IMPORT_NO_QUOTES_FN);
  1464. if (match) {
  1465. const [, resourceURL, media] = match;
  1466. return { resourceURL, media };
  1467. }
  1468. }
  1469. static removeCssComments(stylesheetContent) {
  1470. let start, end;
  1471. do {
  1472. start = stylesheetContent.indexOf("/*");
  1473. end = stylesheetContent.indexOf("*/", start + 2);
  1474. if (start != -1 && end != -1) {
  1475. stylesheetContent = stylesheetContent.substring(0, start) + stylesheetContent.substr(end + 2);
  1476. }
  1477. } while (start != -1 && end != -1);
  1478. return stylesheetContent;
  1479. }
  1480. static wrapMediaQuery(stylesheetContent, mediaQuery) {
  1481. if (mediaQuery) {
  1482. return "@media " + mediaQuery + "{ " + stylesheetContent + " }";
  1483. } else {
  1484. return stylesheetContent;
  1485. }
  1486. }
  1487. static replaceImageSource(imgElement, variableName, options) {
  1488. const dataAttributeName = DOM.imagesAttributeName(options.sessionId);
  1489. if (imgElement.getAttribute(dataAttributeName) != null) {
  1490. const imgData = options.imageData[Number(imgElement.getAttribute(dataAttributeName))];
  1491. if (imgData.replaceable) {
  1492. imgElement.setAttribute("src", `data:image/svg+xml,<svg xmlns="http://www.w3.org/2000/svg" width="${imgData.size.pxWidth}" height="${imgData.size.pxHeight}"><rect fill-opacity="0"/></svg>`);
  1493. const backgroundStyle = {};
  1494. const backgroundSize = (imgData.objectFit == "content" || imgData.objectFit == "cover") && imgData.objectFit;
  1495. if (backgroundSize) {
  1496. backgroundStyle["background-size"] = imgData.objectFit;
  1497. }
  1498. if (imgData.objectPosition) {
  1499. backgroundStyle["background-position"] = imgData.objectPosition;
  1500. }
  1501. if (imgData.backgroundColor) {
  1502. backgroundStyle["background-color"] = imgData.backgroundColor;
  1503. }
  1504. DomProcessorHelper.setBackgroundImage(imgElement, "var(" + variableName + ")", backgroundStyle);
  1505. imgElement.removeAttribute(dataAttributeName);
  1506. return true;
  1507. }
  1508. }
  1509. }
  1510. }
  1511. function log(...args) {
  1512. console.log("S-File <core> ", ...args); // eslint-disable-line no-console
  1513. }
  1514. // -----
  1515. // Stats
  1516. // -----
  1517. const STATS_DEFAULT_VALUES = {
  1518. discarded: {
  1519. "HTML bytes": 0,
  1520. "hidden elements": 0,
  1521. "HTML imports": 0,
  1522. scripts: 0,
  1523. objects: 0,
  1524. "audio sources": 0,
  1525. "video sources": 0,
  1526. frames: 0,
  1527. "CSS rules": 0,
  1528. canvas: 0,
  1529. stylesheets: 0,
  1530. resources: 0,
  1531. medias: 0
  1532. },
  1533. processed: {
  1534. "HTML bytes": 0,
  1535. "hidden elements": 0,
  1536. "HTML imports": 0,
  1537. scripts: 0,
  1538. objects: 0,
  1539. "audio sources": 0,
  1540. "video sources": 0,
  1541. frames: 0,
  1542. "CSS rules": 0,
  1543. canvas: 0,
  1544. stylesheets: 0,
  1545. resources: 0,
  1546. medias: 0
  1547. }
  1548. };
  1549. class Stats {
  1550. constructor(options) {
  1551. this.options = options;
  1552. if (options.displayStats) {
  1553. this.data = JSON.parse(JSON.stringify(STATS_DEFAULT_VALUES));
  1554. }
  1555. }
  1556. set(type, subType, value) {
  1557. if (this.options.displayStats) {
  1558. this.data[type][subType] = value;
  1559. }
  1560. }
  1561. add(type, subType, value) {
  1562. if (this.options.displayStats) {
  1563. this.data[type][subType] += value;
  1564. }
  1565. }
  1566. addAll(pageData) {
  1567. if (this.options.displayStats) {
  1568. Object.keys(this.data.discarded).forEach(key => this.add("discarded", key, pageData.stats.discarded[key] || 0));
  1569. Object.keys(this.data.processed).forEach(key => this.add("processed", key, pageData.stats.processed[key] || 0));
  1570. }
  1571. }
  1572. }
  1573. return { getClass };
  1574. })();