single-file-core.js 71 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711
  1. /*
  2. * Copyright 2018 Gildas Lormeau
  3. * contact : gildas.lormeau <at> gmail.com
  4. *
  5. * This file is part of SingleFile.
  6. *
  7. * The code in this file is free software: you can redistribute it and/or
  8. * modify it under the terms of the GNU Affero General Public License
  9. * (GNU AGPL) as published by the Free Software Foundation, either version 3
  10. * of the License, or (at your option) any later version.
  11. *
  12. * The code in this file is distributed in the hope that it will be useful,
  13. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
  15. * General Public License for more details.
  16. *
  17. * As additional permission under GNU AGPL version 3 section 7, you may
  18. * distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU
  19. * AGPL normally required by section 4, provided you include this license
  20. * notice and a URL through which recipients can access the Corresponding
  21. * Source.
  22. */
  23. this.SingleFileCore = this.SingleFileCore || (() => {
  24. const SELECTED_CONTENT_ATTRIBUTE_NAME = "data-single-file-selected-content";
  25. const SELECTED_CONTENT_ROOT_ATTRIBUTE_NAME = "data-single-file-selected-content-root";
  26. const DEBUG = false;
  27. let DocUtil, cssTree, sessionId = 0;
  28. function getClass(...args) {
  29. [DocUtil, cssTree] = args;
  30. return SingleFileClass;
  31. }
  32. class SingleFileClass {
  33. constructor(options) {
  34. this.options = options;
  35. if (options.sessionId === undefined) {
  36. options.sessionId = sessionId;
  37. sessionId++;
  38. }
  39. }
  40. async initialize() {
  41. this.runner = new Runner(this.options, true);
  42. await this.runner.loadPage();
  43. await this.runner.initialize();
  44. }
  45. async run() {
  46. await this.runner.run();
  47. }
  48. async getPageData() {
  49. return this.runner.getPageData();
  50. }
  51. }
  52. SingleFileClass.SELECTED_CONTENT_ATTRIBUTE_NAME = SELECTED_CONTENT_ATTRIBUTE_NAME;
  53. SingleFileClass.SELECTED_CONTENT_ROOT_ATTRIBUTE_NAME = SELECTED_CONTENT_ROOT_ATTRIBUTE_NAME;
  54. // -------------
  55. // ProgressEvent
  56. // -------------
  57. const PAGE_LOADING = "page-loading";
  58. const PAGE_LOADED = "page-loaded";
  59. const RESOURCES_INITIALIZING = "resource-initializing";
  60. const RESOURCES_INITIALIZED = "resources-initialized";
  61. const RESOURCE_LOADED = "resource-loaded";
  62. const PAGE_ENDED = "page-ended";
  63. const STAGE_STARTED = "stage-started";
  64. const STAGE_ENDED = "stage-ended";
  65. const STAGE_TASK_STARTED = "stage-task-started";
  66. const STAGE_TASK_ENDED = "stage-task-ended";
  67. class ProgressEvent {
  68. constructor(type, detail) {
  69. return { type, detail, PAGE_LOADING, PAGE_LOADED, RESOURCES_INITIALIZING, RESOURCES_INITIALIZED, RESOURCE_LOADED, PAGE_ENDED, STAGE_STARTED, STAGE_ENDED, STAGE_TASK_STARTED, STAGE_TASK_ENDED };
  70. }
  71. }
  72. // ------
  73. // Runner
  74. // ------
  75. const RESOLVE_URLS_STAGE = 0;
  76. const REPLACE_DATA_STAGE = 1;
  77. const REPLACE_DOCS_STAGE = 2;
  78. const POST_PROCESS_STAGE = 3;
  79. const STAGES = [{
  80. sequential: [
  81. { action: "preProcessPage" },
  82. { action: "insertShadowRootContents" },
  83. { action: "replaceStyleContents" },
  84. { option: "selected", action: "removeUnselectedElements" },
  85. { option: "removeVideoSrc", action: "insertVideoPosters" },
  86. { option: "removeFrames", action: "removeFrames" },
  87. { option: "removeImports", action: "removeImports" },
  88. { option: "removeScripts", action: "removeScripts" },
  89. { action: "removeDiscardedResources" },
  90. { action: "resetCharsetMeta" },
  91. { action: "setInputValues" },
  92. { option: "insertFaviconLink", action: "insertFaviconLink" },
  93. { action: "replaceCanvasElements" },
  94. { action: "insertFonts" },
  95. { option: "removeHiddenElements", action: "removeHiddenElements" },
  96. { action: "resolveHrefs" },
  97. { action: "resolveStyleAttributeURLs" }
  98. ],
  99. parallel: [
  100. { action: "resolveStylesheetURLs" },
  101. { option: "!removeFrames", action: "resolveFrameURLs" },
  102. { option: "!removeImports", action: "resolveHtmlImportURLs" }
  103. ]
  104. }, {
  105. sequential: [
  106. { option: "removeUnusedStyles", action: "removeUnusedStyles" },
  107. { option: "removeAlternativeMedias", action: "removeAlternativeMedias" },
  108. { option: "removeUnusedStyles", action: "removeUnusedFonts" }
  109. ],
  110. parallel: [
  111. { action: "processStylesheets" },
  112. { action: "processStyleAttributes" },
  113. { action: "processPageResources" },
  114. { option: "!removeScripts", action: "processScripts" }
  115. ]
  116. }, {
  117. sequential: [
  118. { option: "removeAlternativeImages", action: "removeAlternativeImages" },
  119. { option: "removeAlternativeFonts", action: "removeAlternativeFonts" }
  120. ],
  121. parallel: [
  122. { option: "!removeFrames", action: "processFrames" },
  123. { option: "!removeImports", action: "processHtmlImports" },
  124. ]
  125. }, {
  126. sequential: [
  127. { action: "replaceStylesheets" },
  128. { action: "replaceStyleAttributes" },
  129. { action: "insertVariables" },
  130. { option: "compressHTML", action: "compressHTML" }
  131. ]
  132. }];
  133. class Runner {
  134. constructor(options, root) {
  135. this.root = root;
  136. this.options = options;
  137. this.options.url = this.options.url || this.options.doc.location.href;
  138. this.options.baseURI = this.options.doc && this.options.doc.baseURI;
  139. this.batchRequest = new BatchRequest();
  140. this.processor = new Processor(options, this.batchRequest);
  141. if (this.options.doc) {
  142. const docData = DocUtil.preProcessDoc(this.options.doc, this.options.win, this.options);
  143. this.options.canvasData = docData.canvasData;
  144. this.options.fontsData = docData.fontsData;
  145. this.options.stylesheetContents = docData.stylesheetContents;
  146. this.options.imageData = docData.imageData;
  147. this.options.postersData = docData.postersData;
  148. this.options.usedFonts = docData.usedFonts;
  149. this.options.shadowRootContents = docData.shadowRootContents;
  150. }
  151. this.options.content = this.options.content || (this.options.doc ? DocUtil.serialize(this.options.doc, false) : null);
  152. this.onprogress = options.onprogress || (() => { });
  153. }
  154. async loadPage() {
  155. this.onprogress(new ProgressEvent(PAGE_LOADING, { pageURL: this.options.url, frame: !this.root }));
  156. await this.processor.loadPage(this.options.content);
  157. this.onprogress(new ProgressEvent(PAGE_LOADED, { pageURL: this.options.url, frame: !this.root }));
  158. }
  159. async initialize() {
  160. this.onprogress(new ProgressEvent(RESOURCES_INITIALIZING, { pageURL: this.options.url }));
  161. await this.executeStage(RESOLVE_URLS_STAGE);
  162. this.pendingPromises = this.executeStage(REPLACE_DATA_STAGE);
  163. if (this.options.doc) {
  164. DocUtil.postProcessDoc(this.options.doc, this.options);
  165. this.options.doc = null;
  166. this.options.win = null;
  167. }
  168. }
  169. async run() {
  170. if (this.root) {
  171. this.processor.initialize(this.batchRequest);
  172. this.onprogress(new ProgressEvent(RESOURCES_INITIALIZED, { pageURL: this.options.url, index: 0, max: this.processor.maxResources }));
  173. }
  174. await this.batchRequest.run(detail => {
  175. detail.pageURL = this.options.url;
  176. this.onprogress(new ProgressEvent(RESOURCE_LOADED, detail));
  177. }, this.options);
  178. await this.pendingPromises;
  179. await this.executeStage(REPLACE_DOCS_STAGE);
  180. await this.executeStage(POST_PROCESS_STAGE);
  181. await this.processor.end();
  182. }
  183. async getPageData() {
  184. if (this.root) {
  185. this.onprogress(new ProgressEvent(PAGE_ENDED, { pageURL: this.options.url }));
  186. }
  187. return this.processor.getPageData();
  188. }
  189. async executeStage(step) {
  190. if (DEBUG) {
  191. log("**** STARTED STAGE", step, "****");
  192. }
  193. const frame = !this.root;
  194. this.onprogress(new ProgressEvent(STAGE_STARTED, { pageURL: this.options.url, step, frame }));
  195. STAGES[step].sequential.forEach(task => {
  196. let startTime;
  197. if (DEBUG) {
  198. startTime = Date.now();
  199. log(" -- STARTED task =", task.action);
  200. }
  201. this.onprogress(new ProgressEvent(STAGE_TASK_STARTED, { pageURL: this.options.url, step, task: task.action, frame }));
  202. this.executeTask(task);
  203. this.onprogress(new ProgressEvent(STAGE_TASK_ENDED, { pageURL: this.options.url, step, task: task.action, frame }));
  204. if (DEBUG) {
  205. log(" -- ENDED task =", task.action, "delay =", Date.now() - startTime);
  206. }
  207. });
  208. let parallelTasksPromise;
  209. if (STAGES[step].parallel) {
  210. parallelTasksPromise = await Promise.all(STAGES[step].parallel.map(async task => {
  211. let startTime;
  212. if (DEBUG) {
  213. startTime = Date.now();
  214. log(" // STARTED task =", task.action);
  215. }
  216. this.onprogress(new ProgressEvent(STAGE_TASK_STARTED, { pageURL: this.options.url, step, task: task.action, frame }));
  217. await this.executeTask(task);
  218. this.onprogress(new ProgressEvent(STAGE_TASK_ENDED, { pageURL: this.options.url, step, task: task.action, frame }));
  219. if (DEBUG) {
  220. log(" // ENDED task =", task.action, "delay =", Date.now() - startTime);
  221. }
  222. }));
  223. } else {
  224. parallelTasksPromise = Promise.resolve();
  225. }
  226. this.onprogress(new ProgressEvent(STAGE_ENDED, { pageURL: this.options.url, step, frame }));
  227. if (DEBUG) {
  228. log("**** ENDED STAGE", step, "****");
  229. }
  230. return parallelTasksPromise;
  231. }
  232. executeTask(task) {
  233. if (!task.option || ((task.option.startsWith("!") && !this.options[task.option]) || this.options[task.option])) {
  234. return this.processor[task.action]();
  235. }
  236. }
  237. }
  238. // ------------
  239. // BatchRequest
  240. // ------------
  241. class BatchRequest {
  242. constructor() {
  243. this.requests = new Map();
  244. this.duplicates = new Map();
  245. }
  246. async addURL(resourceURL, asDataURI, groupDuplicates) {
  247. return new Promise((resolve, reject) => {
  248. const requestKey = JSON.stringify([resourceURL, asDataURI]);
  249. let resourceRequests = this.requests.get(requestKey);
  250. if (!resourceRequests) {
  251. resourceRequests = [];
  252. this.requests.set(requestKey, resourceRequests);
  253. }
  254. const callbacks = { resolve, reject };
  255. resourceRequests.push(callbacks);
  256. if (groupDuplicates) {
  257. let duplicateRequests = this.duplicates.get(requestKey);
  258. if (!duplicateRequests) {
  259. duplicateRequests = [];
  260. this.duplicates.set(requestKey, duplicateRequests);
  261. }
  262. duplicateRequests.push(callbacks);
  263. }
  264. });
  265. }
  266. getMaxResources() {
  267. return Array.from(this.requests.keys()).length;
  268. }
  269. async run(onloadListener, options) {
  270. const resourceURLs = Array.from(this.requests.keys());
  271. let indexResource = 0;
  272. return Promise.all(resourceURLs.map(async requestKey => {
  273. const [resourceURL, asDataURI] = JSON.parse(requestKey);
  274. const resourceRequests = this.requests.get(requestKey);
  275. try {
  276. const content = await DocUtil.getContent(resourceURL, { asDataURI, maxResourceSize: options.maxResourceSize, maxResourceSizeEnabled: options.maxResourceSizeEnabled });
  277. indexResource = indexResource + 1;
  278. onloadListener({ index: indexResource, url: resourceURL });
  279. resourceRequests.forEach(callbacks => {
  280. const duplicateCallbacks = this.duplicates.get(requestKey);
  281. const duplicate = duplicateCallbacks && duplicateCallbacks.length > 1 && duplicateCallbacks.includes(callbacks);
  282. callbacks.resolve({ content: content.data, indexResource, duplicate });
  283. });
  284. } catch (error) {
  285. indexResource = indexResource + 1;
  286. onloadListener({ index: indexResource, url: resourceURL });
  287. resourceRequests.forEach(resourceRequest => resourceRequest.reject(error));
  288. }
  289. this.requests.delete(requestKey);
  290. }));
  291. }
  292. }
  293. // ---------
  294. // Processor
  295. // ---------
  296. const EMPTY_DATA_URI = "data:base64,";
  297. const EMPTY_IMAGE = "data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==";
  298. const SCRIPT_TAG_FOUND = /<script/gi;
  299. const NOSCRIPT_TAG_FOUND = /<noscript/gi;
  300. class Processor {
  301. constructor(options, batchRequest) {
  302. this.options = options;
  303. this.stats = new Stats(options);
  304. this.baseURI = Util.normalizeURL(options.baseURI || options.url);
  305. this.batchRequest = batchRequest;
  306. this.stylesheets = new Map();
  307. this.styles = new Map();
  308. this.cssVariables = new Map();
  309. }
  310. initialize() {
  311. this.maxResources = this.batchRequest.getMaxResources();
  312. if (!this.options.removeFrames) {
  313. this.options.framesData.forEach(frameData => this.maxResources += frameData.maxResources || 0);
  314. }
  315. this.stats.set("processed", "resources", this.maxResources);
  316. }
  317. async loadPage(pageContent) {
  318. let resourceURL;
  319. if (!pageContent || this.options.saveRawPage) {
  320. const content = await DocUtil.getContent(this.baseURI, { asDataURI: false, maxResourceSize: this.options.maxResourceSize, maxResourceSizeEnabled: this.options.maxResourceSizeEnabled });
  321. pageContent = content.data;
  322. this.baseURI = this.options.url = resourceURL;
  323. }
  324. this.doc = DocUtil.createDoc(pageContent, this.baseURI);
  325. this.onEventAttributeNames = Util.getOnEventAttributeNames(this.doc);
  326. }
  327. async getPageData() {
  328. DocUtil.postProcessDoc(this.doc, this.options);
  329. const url = DocUtil.parseURL(this.baseURI);
  330. if (this.options.insertSingleFileComment) {
  331. const infobarContent = (this.options.infobarContent || "").replace(/\\n/g, "\n").replace(/\\t/g, "\t");
  332. const commentNode = this.doc.createComment("\n Page saved with SingleFile" +
  333. " \n url: " + this.options.url +
  334. " \n saved date: " + new Date() +
  335. (infobarContent ? " \n info: " + infobarContent : "") + "\n");
  336. this.doc.documentElement.insertBefore(commentNode, this.doc.documentElement.firstChild);
  337. }
  338. let size;
  339. if (this.options.displayStats) {
  340. size = DocUtil.getContentSize(this.doc.documentElement.outerHTML);
  341. }
  342. const content = DocUtil.serialize(this.doc, this.options.compressHTML);
  343. if (this.options.displayStats) {
  344. const contentSize = DocUtil.getContentSize(content);
  345. this.stats.set("processed", "HTML bytes", contentSize);
  346. this.stats.add("discarded", "HTML bytes", size - contentSize);
  347. }
  348. let filename = await ProcessorHelper.evalTemplate(this.options.filenameTemplate, this.options, content);
  349. filename = filename.replace(/[~\\?%*:|"<>\x00-\x1f\x7F]+/g, "_"); // eslint-disable-line no-control-regex
  350. filename = filename.replace(/\.\.\//g, "").replace(/^\/+/, "").replace(/\/+/g, "/").replace(/\/$/, "");
  351. if (!this.options.backgroundSave) {
  352. filename = filename.replace(/\//g, "_");
  353. }
  354. if (filename.length > 192) {
  355. const extensionMatch = filename.match(/(\.[^.]{3,4})$/);
  356. const extension = extensionMatch && extensionMatch[0] && extensionMatch[0].length > 1 ? extensionMatch[0] : "";
  357. filename = filename.substring(0, 192 - extension.length) + "…" + extension;
  358. }
  359. if (!filename) {
  360. filename = "Unnamed page";
  361. }
  362. const matchTitle = this.baseURI.match(/([^/]*)\/?(\.html?.*)$/) || this.baseURI.match(/\/\/([^/]*)\/?$/);
  363. return {
  364. stats: this.stats.data,
  365. title: this.options.title || (this.baseURI && matchTitle ? matchTitle[1] : (url.hostname ? url.hostname : "")),
  366. filename,
  367. content
  368. };
  369. }
  370. removeUnselectedElements() {
  371. const rootElement = this.doc.querySelector("[" + SELECTED_CONTENT_ROOT_ATTRIBUTE_NAME + "]");
  372. if (rootElement) {
  373. ProcessorHelper.isolateElements(rootElement);
  374. rootElement.removeAttribute(SELECTED_CONTENT_ROOT_ATTRIBUTE_NAME);
  375. rootElement.removeAttribute(SELECTED_CONTENT_ATTRIBUTE_NAME);
  376. }
  377. }
  378. setInputValues() {
  379. this.doc.querySelectorAll("input").forEach(input => {
  380. const value = input.getAttribute(DocUtil.inputValueAttributeName(this.options.sessionId));
  381. input.setAttribute("value", value || "");
  382. });
  383. this.doc.querySelectorAll("input[type=radio], input[type=checkbox]").forEach(input => {
  384. const value = input.getAttribute(DocUtil.inputValueAttributeName(this.options.sessionId));
  385. if (value == "true") {
  386. input.setAttribute("checked", "");
  387. }
  388. });
  389. this.doc.querySelectorAll("textarea").forEach(textarea => {
  390. const value = textarea.getAttribute(DocUtil.inputValueAttributeName(this.options.sessionId));
  391. textarea.textContent = value || "";
  392. });
  393. this.doc.querySelectorAll("select").forEach(select => {
  394. select.querySelectorAll("option").forEach(option => {
  395. const selected = option.getAttribute(DocUtil.inputValueAttributeName(this.options.sessionId)) != null;
  396. if (selected) {
  397. option.setAttribute("selected", "");
  398. }
  399. });
  400. });
  401. }
  402. removeDiscardedResources() {
  403. const objectElements = this.doc.querySelectorAll("applet, meta[http-equiv=refresh], object[data]:not([type=\"image/svg+xml\"]):not([type=\"image/svg-xml\"]):not([type=\"text/html\"]), embed[src]:not([src*=\".svg\"])");
  404. this.stats.set("discarded", "objects", objectElements.length);
  405. this.stats.set("processed", "objects", objectElements.length);
  406. objectElements.forEach(element => element.remove());
  407. const replacedAttributeValue = this.doc.querySelectorAll("link[rel~=preconnect], link[rel~=prerender], link[rel~=dns-prefetch], link[rel~=preload], link[rel~=prefetch]");
  408. replacedAttributeValue.forEach(element => {
  409. const relValue = element.getAttribute("rel").replace(/(preconnect|prerender|dns-prefetch|preload|prefetch)/g, "").trim();
  410. if (relValue.length) {
  411. element.setAttribute("rel", relValue);
  412. } else {
  413. element.remove();
  414. }
  415. });
  416. this.doc.querySelectorAll("meta[http-equiv=\"content-security-policy\"]").forEach(element => element.remove());
  417. if (this.options.compressHTML) {
  418. this.doc.querySelectorAll("input[type=hidden]").forEach(element => element.remove());
  419. }
  420. this.doc.querySelectorAll("a[ping]").forEach(element => element.removeAttribute("ping"));
  421. if (this.options.removeScripts) {
  422. this.onEventAttributeNames.forEach(attributeName => this.doc.querySelectorAll("[" + attributeName + "]").forEach(element => element.removeAttribute(attributeName)));
  423. this.doc.querySelectorAll("[href]").forEach(element => {
  424. if (element.href && element.href.match && element.href.match(/^\s*javascript:/)) {
  425. element.setAttribute("href", "");
  426. }
  427. });
  428. this.doc.querySelectorAll("[src]").forEach(element => {
  429. if (element.src && element.src.match(/^\s*javascript:/)) {
  430. element.removeAttribute("src");
  431. }
  432. });
  433. }
  434. const audioSourceElements = this.doc.querySelectorAll("audio[src], audio > source[src]");
  435. this.stats.set("processed", "audio sources", audioSourceElements.length);
  436. if (this.options.removeAudioSrc) {
  437. this.stats.set("discarded", "audio sources", audioSourceElements.length);
  438. audioSourceElements.forEach(element => {
  439. if (element.tagName == "SOURCE") {
  440. element.remove();
  441. } else {
  442. audioSourceElements.forEach(element => element.removeAttribute("src"));
  443. }
  444. });
  445. }
  446. const videoSourceElements = this.doc.querySelectorAll("video[src], video > source[src]");
  447. this.stats.set("processed", "video sources", videoSourceElements.length);
  448. if (this.options.removeVideoSrc) {
  449. this.stats.set("discarded", "video sources", videoSourceElements.length);
  450. videoSourceElements.forEach(element => {
  451. if (element.tagName == "SOURCE") {
  452. element.remove();
  453. } else {
  454. videoSourceElements.forEach(element => element.removeAttribute("src"));
  455. }
  456. });
  457. }
  458. }
  459. async end() {
  460. const metaCharset = this.doc.head.querySelector("meta[charset]");
  461. if (metaCharset) {
  462. this.doc.head.insertBefore(metaCharset, this.doc.head.firstChild);
  463. }
  464. this.doc.querySelectorAll("base").forEach(element => element.remove());
  465. if (this.doc.head.querySelectorAll("*").length == 1 && metaCharset && this.doc.body.childNodes.length == 0) {
  466. this.doc.head.querySelector("meta[charset]").remove();
  467. }
  468. const titleElement = this.doc.querySelector("title");
  469. const descriptionElement = this.doc.querySelector("meta[name=description]");
  470. const authorElement = this.doc.querySelector("meta[name=author]");
  471. const creatorElement = this.doc.querySelector("meta[name=creator]");
  472. const publisherElement = this.doc.querySelector("meta[name=publisher]");
  473. this.options.title = titleElement ? titleElement.textContent.trim() : "";
  474. this.options.infobarContent = await ProcessorHelper.evalTemplate(this.options.infobarTemplate, this.options, null, true);
  475. this.options.info = {
  476. description: descriptionElement ? descriptionElement.content.trim() : "",
  477. lang: this.doc.documentElement.lang,
  478. author: authorElement ? authorElement.content.trim() : "",
  479. creator: creatorElement ? creatorElement.content.trim() : "",
  480. publisher: publisherElement ? publisherElement.content.trim() : ""
  481. };
  482. }
  483. preProcessPage() {
  484. this.doc.querySelectorAll("singlefile-infobar, singlefile-mask, singlefile-logs-window").forEach(element => element.remove());
  485. if (this.options.win) {
  486. this.doc.body.querySelectorAll(":not(svg) title, meta, link[href][rel*=\"icon\"]").forEach(element => element instanceof this.options.win.HTMLElement && this.doc.head.appendChild(element));
  487. }
  488. if (this.options.imageData) {
  489. const dataAttributeName = DocUtil.imagesAttributeName(this.options.sessionId);
  490. this.doc.querySelectorAll("img").forEach(imgElement => {
  491. const imgData = this.options.imageData[Number(imgElement.getAttribute(dataAttributeName))];
  492. if (this.options.removeHiddenElements && imgData.size && !imgData.size.pxWidth && !imgData.size.pxHeight) {
  493. imgElement.setAttribute("src", EMPTY_IMAGE);
  494. } else if (imgData.currentSrc) {
  495. imgElement.setAttribute("src", imgData.currentSrc);
  496. }
  497. });
  498. if (this.options.lazyLoadImages) {
  499. this.doc.querySelectorAll("img[data-src]").forEach(imgElement => {
  500. const imgData = this.options.imageData[Number(imgElement.getAttribute(dataAttributeName))];
  501. if ((!imgElement.getAttribute("src") || imgElement.getAttribute("src") == EMPTY_IMAGE) && imgElement.getAttribute("data-src")) {
  502. imgData.src = imgElement.dataset.src;
  503. imgElement.setAttribute("src", imgElement.dataset.src);
  504. imgElement.removeAttribute("data-src");
  505. }
  506. });
  507. this.doc.querySelectorAll("img[data-srcset]").forEach(imgElement => {
  508. if (!imgElement.getAttribute("srcset") && imgElement.getAttribute("data-srcset")) {
  509. imgElement.setAttribute("srcset", imgElement.dataset.srcset);
  510. imgElement.removeAttribute("data-srcset");
  511. }
  512. });
  513. }
  514. }
  515. }
  516. removeScripts() {
  517. const scriptElements = this.doc.querySelectorAll("script:not([type=\"application/ld+json\"])");
  518. this.stats.set("discarded", "scripts", scriptElements.length);
  519. this.stats.set("processed", "scripts", scriptElements.length);
  520. scriptElements.forEach(element => element.remove());
  521. }
  522. removeFrames() {
  523. const frameElements = this.doc.querySelectorAll("iframe, frame, object[type=\"text/html\"][data]");
  524. this.stats.set("discarded", "frames", frameElements.length);
  525. this.stats.set("processed", "frames", frameElements.length);
  526. this.doc.querySelectorAll("iframe, frame, object[type=\"text/html\"][data]").forEach(element => element.remove());
  527. }
  528. removeImports() {
  529. const importElements = this.doc.querySelectorAll("link[rel=import]");
  530. this.stats.set("discarded", "HTML imports", importElements.length);
  531. this.stats.set("processed", "HTML imports", importElements.length);
  532. importElements.forEach(element => element.remove());
  533. }
  534. resetCharsetMeta() {
  535. let charset;
  536. this.doc.querySelectorAll("meta[charset], meta[http-equiv=\"content-type\"]").forEach(element => {
  537. const charsetDeclaration = element.content.split(";")[1];
  538. if (charsetDeclaration && !charset) {
  539. charset = charsetDeclaration.split("=")[1];
  540. if (charset) {
  541. this.charset = charset.trim().toLowerCase();
  542. }
  543. }
  544. element.remove();
  545. });
  546. const metaElement = this.doc.createElement("meta");
  547. metaElement.setAttribute("charset", "utf-8");
  548. if (this.doc.head.firstChild) {
  549. this.doc.head.insertBefore(metaElement, this.doc.head.firstChild);
  550. } else {
  551. this.doc.head.appendChild(metaElement);
  552. }
  553. }
  554. insertFaviconLink() {
  555. let faviconElement = this.doc.querySelector("link[href][rel=\"icon\"]");
  556. if (!faviconElement) {
  557. faviconElement = this.doc.querySelector("link[href][rel=\"shortcut icon\"]");
  558. }
  559. if (!faviconElement) {
  560. faviconElement = this.doc.createElement("link");
  561. faviconElement.setAttribute("type", "image/x-icon");
  562. faviconElement.setAttribute("rel", "shortcut icon");
  563. faviconElement.setAttribute("href", "/favicon.ico");
  564. }
  565. this.doc.head.appendChild(faviconElement);
  566. }
  567. resolveHrefs() {
  568. this.doc.querySelectorAll("[href]").forEach(element => {
  569. const href = element.getAttribute("href");
  570. if (!Util.testIgnoredPath(href) && Util.testValidPath(href, this.baseURI, this.options.url)) {
  571. const normalizedHref = Util.normalizeURL(href);
  572. if (element.tagName == "LINK") {
  573. try {
  574. element.setAttribute("href", DocUtil.resolveURL(href, this.options.baseURI || this.options.url));
  575. } catch (error) {
  576. if (element.tagName == "LINK") {
  577. element.setAttribute("href", EMPTY_URL);
  578. }
  579. }
  580. } else if (normalizedHref == href) {
  581. try {
  582. element.setAttribute("href", DocUtil.resolveURL(normalizedHref, this.options.baseURI || this.options.url));
  583. } catch (error) {
  584. // ignored
  585. }
  586. }
  587. }
  588. });
  589. }
  590. removeUnusedStyles() {
  591. if (!this.mediaAllInfo) {
  592. this.mediaAllInfo = DocUtil.getMediaAllInfo(this.doc, this.stylesheets, this.styles);
  593. }
  594. const stats = DocUtil.minifyCSSRules(this.stylesheets, this.styles, this.mediaAllInfo);
  595. this.stats.set("processed", "CSS rules", stats.processed);
  596. this.stats.set("discarded", "CSS rules", stats.discarded);
  597. }
  598. removeUnusedFonts() {
  599. DocUtil.removeUnusedFonts(this.doc, this.stylesheets, this.styles, this.options);
  600. }
  601. removeAlternativeFonts() {
  602. DocUtil.removeAlternativeFonts(this.doc, this.stylesheets);
  603. }
  604. removeAlternativeImages() {
  605. DocUtil.removeAlternativeImages(this.doc);
  606. }
  607. removeHiddenElements() {
  608. const hiddenElements = this.doc.querySelectorAll("[" + DocUtil.removedContentAttributeName(this.options.sessionId) + "]");
  609. this.stats.set("discarded", "hidden elements", hiddenElements.length);
  610. this.stats.set("processed", "hidden elements", hiddenElements.length);
  611. hiddenElements.forEach(element => element.remove());
  612. }
  613. compressHTML() {
  614. let size;
  615. if (this.options.displayStats) {
  616. size = DocUtil.getContentSize(this.doc.documentElement.outerHTML);
  617. }
  618. DocUtil.minifyHTML(this.doc, { preservedSpaceAttributeName: DocUtil.preservedSpaceAttributeName(this.options.sessionId) });
  619. if (this.options.displayStats) {
  620. this.stats.add("discarded", "HTML bytes", size - DocUtil.getContentSize(this.doc.documentElement.outerHTML));
  621. }
  622. }
  623. removeAlternativeMedias() {
  624. const stats = DocUtil.minifyMedias(this.stylesheets);
  625. this.stats.set("processed", "medias", stats.processed);
  626. this.stats.set("discarded", "medias", stats.discarded);
  627. }
  628. replaceCanvasElements() {
  629. if (this.options.canvasData) {
  630. this.doc.querySelectorAll("canvas").forEach((canvasElement, indexCanvasElement) => {
  631. const canvasData = this.options.canvasData[indexCanvasElement];
  632. if (canvasData) {
  633. ProcessorHelper.setBackgroundImage(canvasElement, "url(" + canvasData.dataURI + ")");
  634. this.stats.add("processed", "canvas", 1);
  635. }
  636. });
  637. }
  638. }
  639. insertFonts() {
  640. if (this.options.fontsData && this.options.fontsData.length) {
  641. let stylesheetContent = "";
  642. this.options.fontsData.forEach(fontStyles => {
  643. if (fontStyles["font-family"] && fontStyles.src) {
  644. stylesheetContent += "@font-face{";
  645. let stylesContent = "";
  646. Object.keys(fontStyles).forEach(fontStyle => {
  647. if (stylesContent) {
  648. stylesContent += ";";
  649. }
  650. stylesContent += fontStyle + ":" + fontStyles[fontStyle];
  651. });
  652. stylesheetContent += stylesContent + "}";
  653. }
  654. });
  655. if (stylesheetContent) {
  656. const styleElement = this.doc.createElement("style");
  657. styleElement.textContent = stylesheetContent;
  658. const existingStyleElement = this.doc.querySelector("style");
  659. if (existingStyleElement) {
  660. existingStyleElement.parentElement.insertBefore(styleElement, existingStyleElement);
  661. } else {
  662. this.doc.head.insertBefore(styleElement, this.doc.head.firstChild);
  663. }
  664. }
  665. }
  666. }
  667. replaceStyleContents() {
  668. if (this.options.stylesheetContents) {
  669. this.doc.querySelectorAll("style").forEach((styleElement, styleIndex) => {
  670. if (this.options.stylesheetContents[styleIndex]) {
  671. styleElement.textContent = this.options.stylesheetContents[styleIndex];
  672. }
  673. });
  674. }
  675. }
  676. insertShadowRootContents() {
  677. if (this.options.shadowRootContents) {
  678. this.doc.querySelectorAll("[" + DocUtil.shadowRootAttributeName(this.options.sessionId) + "]").forEach((element, elementIndex) => {
  679. const elementInfo = this.options.shadowRootContents[elementIndex];
  680. if (elementInfo) {
  681. const frameElement = this.doc.createElement("iframe");
  682. frameElement.setAttribute("style", "all:initial!important;border:0!important;width:100%!important;height:" + elementInfo.height + "px!important");
  683. const windowId = "shadow-" + this.options.framesData.length;
  684. frameElement.setAttribute(DocUtil.windowIdAttributeName(this.options.sessionId), windowId);
  685. this.options.framesData.push({ windowId, content: elementInfo.content, baseURI: this.baseURI });
  686. element.appendChild(frameElement);
  687. }
  688. });
  689. }
  690. }
  691. insertVideoPosters() {
  692. if (this.options.postersData) {
  693. this.doc.querySelectorAll("video[src], video > source[src]").forEach((element, videoIndex) => {
  694. let videoElement;
  695. if (element.tagName == "VIDEO") {
  696. videoElement = element;
  697. } else {
  698. videoElement = element.parentElement;
  699. }
  700. if (!videoElement.poster && this.options.postersData[videoIndex]) {
  701. videoElement.setAttribute("poster", this.options.postersData[videoIndex]);
  702. }
  703. });
  704. }
  705. }
  706. async processPageResources() {
  707. const processAttributeArgs = [
  708. ["link[href][rel*=\"icon\"]", "href", "data:", false, true],
  709. ["object[type=\"image/svg+xml\"], object[type=\"image/svg-xml\"]", "data", PREFIX_DATA_URI_IMAGE_SVG],
  710. ["img[src], input[src][type=image]", "src", PREFIX_DATA_URI_IMAGE, true],
  711. ["embed[src*=\".svg\"]", "src", PREFIX_DATA_URI_IMAGE_SVG],
  712. ["video[poster]", "poster", PREFIX_DATA_URI_IMAGE],
  713. ["*[background]", "background", PREFIX_DATA_URI_IMAGE],
  714. ["image", "xlink:href", PREFIX_DATA_URI_IMAGE]
  715. ];
  716. let resourcePromises = processAttributeArgs.map(([selector, attributeName, prefixDataURI, processDuplicates, removeElementIfMissing]) =>
  717. ProcessorHelper.processAttribute(this.doc, this.doc.querySelectorAll(selector), attributeName, prefixDataURI, this.baseURI, this.options, this.cssVariables, this.styles, this.batchRequest, processDuplicates, removeElementIfMissing)
  718. );
  719. resourcePromises = resourcePromises.concat([
  720. ProcessorHelper.processXLinks(this.doc.querySelectorAll("use"), this.baseURI, this.options, this.batchRequest),
  721. ProcessorHelper.processSrcset(this.doc.querySelectorAll("img[srcset], source[srcset]"), "srcset", PREFIX_DATA_URI_IMAGE, this.baseURI, this.options, this.batchRequest)
  722. ]);
  723. if (!this.options.removeAudioSrc) {
  724. resourcePromises.push(ProcessorHelper.processAttribute(this.doc, this.doc.querySelectorAll("audio[src], audio > source[src]"), "src", PREFIX_DATA_URI_AUDIO, this.baseURI, this.options, this.cssVariables, this.styles, this.batchRequest));
  725. }
  726. if (!this.options.removeVideoSrc) {
  727. resourcePromises.push(ProcessorHelper.processAttribute(this.doc, this.doc.querySelectorAll("video[src], video > source[src]"), "src", PREFIX_DATA_URI_VIDEO, this.baseURI, this.options, this.cssVariables, this.styles, this.batchRequest));
  728. }
  729. await Promise.all(resourcePromises);
  730. ProcessorHelper.processShortcutIcons(this.doc);
  731. }
  732. async resolveStylesheetURLs() {
  733. await Promise.all(Array.from(this.doc.querySelectorAll("style, link[rel*=stylesheet]"))
  734. .map(async element => {
  735. let mediaText;
  736. if (element.media) {
  737. mediaText = element.media;
  738. }
  739. this.stylesheets.set(element, { mediaText });
  740. const options = { maxResourceSize: this.options.maxResourceSize, maxResourceSizeEnabled: this.options.maxResourceSizeEnabled, url: this.options.url, charset: this.charset, compressCSS: this.options.compressCSS };
  741. const isLinkTag = element.tagName.toLowerCase() == "link";
  742. if (isLinkTag && element.rel.includes("alternate") && element.title) {
  743. element.remove();
  744. } else {
  745. let stylesheetContent;
  746. if (isLinkTag) {
  747. if (element.charset) {
  748. options.charset = element.charset;
  749. }
  750. stylesheetContent = await ProcessorHelper.resolveLinkStylesheetURLs(element.href, this.baseURI, options);
  751. } else {
  752. stylesheetContent = await ProcessorHelper.resolveImportURLs(element.textContent, this.baseURI, options);
  753. }
  754. let stylesheet;
  755. try {
  756. stylesheet = cssTree.parse(stylesheetContent);
  757. } catch (error) {
  758. // ignored
  759. }
  760. if (stylesheet && stylesheet.children) {
  761. if (this.options.compressCSS) {
  762. const removedRules = [];
  763. for (let cssRule = stylesheet.children.head; cssRule; cssRule = cssRule.next) {
  764. if (cssRule.data.type == "Raw" && cssRule.data.value && cssRule.data.value.trim().startsWith("//")) {
  765. removedRules.push(cssRule);
  766. }
  767. }
  768. removedRules.forEach(cssRule => stylesheet.children.remove(cssRule));
  769. }
  770. this.stylesheets.get(element).stylesheet = stylesheet;
  771. }
  772. }
  773. }));
  774. }
  775. async processStylesheets() {
  776. await Promise.all(Array.from(this.stylesheets).map(([, stylesheetInfo]) =>
  777. ProcessorHelper.processStylesheet(stylesheetInfo.stylesheet.children, this.baseURI, this.options, this.cssVariables, this.batchRequest)
  778. ));
  779. }
  780. replaceStylesheets() {
  781. this.doc.querySelectorAll("style").forEach(styleElement => {
  782. const stylesheetInfo = this.stylesheets.get(styleElement);
  783. if (stylesheetInfo) {
  784. let stylesheetContent = cssTree.generate(stylesheetInfo.stylesheet);
  785. styleElement.textContent = stylesheetContent;
  786. if (stylesheetInfo.mediaText) {
  787. styleElement.media = stylesheetInfo.mediaText;
  788. }
  789. } else {
  790. styleElement.remove();
  791. }
  792. });
  793. this.doc.querySelectorAll("link[rel*=stylesheet]").forEach(linkElement => {
  794. const stylesheetInfo = this.stylesheets.get(linkElement);
  795. if (stylesheetInfo) {
  796. const styleElement = this.doc.createElement("style");
  797. if (stylesheetInfo.mediaText) {
  798. styleElement.media = stylesheetInfo.mediaText;
  799. }
  800. let stylesheetContent = cssTree.generate(stylesheetInfo.stylesheet);
  801. styleElement.textContent = stylesheetContent;
  802. linkElement.parentElement.replaceChild(styleElement, linkElement);
  803. } else {
  804. linkElement.remove();
  805. }
  806. });
  807. }
  808. insertVariables() {
  809. if (this.cssVariables.size) {
  810. const styleElement = this.doc.createElement("style");
  811. const firstStyleElement = this.doc.head.querySelector("style");
  812. if (firstStyleElement) {
  813. this.doc.head.insertBefore(styleElement, firstStyleElement);
  814. } else {
  815. this.doc.head.appendChild(styleElement);
  816. }
  817. let stylesheetContent = "";
  818. this.cssVariables.forEach((content, indexResource) => {
  819. if (stylesheetContent) {
  820. stylesheetContent += ";";
  821. }
  822. stylesheetContent += `${SINGLE_FILE_VARIABLE_NAME_PREFIX + indexResource}:url("${content}")`;
  823. });
  824. styleElement.textContent = ":root{" + stylesheetContent + "}";
  825. }
  826. }
  827. async processScripts() {
  828. await Promise.all(Array.from(this.doc.querySelectorAll("script[src]")).map(async scriptElement => {
  829. if (scriptElement.src) {
  830. this.stats.add("processed", "scripts", 1);
  831. const content = await DocUtil.getContent(scriptElement.src, { asDataURI: false, maxResourceSize: this.options.maxResourceSize, maxResourceSizeEnabled: this.options.maxResourceSizeEnabled });
  832. scriptElement.textContent = content.data.replace(/<\//gi, "<\\/").replace(/\/>/gi, "\\/>");
  833. }
  834. scriptElement.removeAttribute("src");
  835. }));
  836. }
  837. async resolveFrameURLs() {
  838. if (this.options.framesData) {
  839. const frameElements = Array.from(this.doc.querySelectorAll("iframe, frame, object[type=\"text/html\"][data]"));
  840. await Promise.all(frameElements.map(async frameElement => {
  841. ProcessorHelper.setFrameEmptySrc(frameElement);
  842. const frameWindowId = frameElement.getAttribute(DocUtil.windowIdAttributeName(this.options.sessionId));
  843. if (frameWindowId) {
  844. const frameData = this.options.framesData.find(frame => frame.windowId == frameWindowId);
  845. if (frameData) {
  846. await initializeProcessor(frameData, frameElement, frameWindowId, this.batchRequest, Object.create(this.options));
  847. }
  848. }
  849. }));
  850. }
  851. async function initializeProcessor(frameData, frameElement, frameWindowId, batchRequest, options) {
  852. options.insertSingleFileComment = false;
  853. options.insertFaviconLink = false;
  854. options.doc = null;
  855. options.win = null;
  856. options.url = frameData.baseURI;
  857. options.windowId = frameWindowId;
  858. if (frameData.content) {
  859. options.content = frameData.content;
  860. options.canvasData = frameData.canvasData;
  861. options.stylesheetContents = frameData.stylesheetContents;
  862. options.fontsData = frameData.fontsData;
  863. options.imageData = frameData.imageData;
  864. options.usedFonts = frameData.usedFonts;
  865. frameData.processor = new Runner(options);
  866. frameData.frameElement = frameElement;
  867. await frameData.processor.loadPage();
  868. await frameData.processor.initialize();
  869. frameData.maxResources = batchRequest.getMaxResources();
  870. }
  871. }
  872. }
  873. async processFrames() {
  874. if (this.options.framesData) {
  875. const frameElements = Array.from(this.doc.querySelectorAll("iframe, frame, object[type=\"text/html\"][data]"));
  876. await Promise.all(frameElements.map(async frameElement => {
  877. const frameWindowId = frameElement.getAttribute(DocUtil.windowIdAttributeName(this.options.sessionId));
  878. if (frameWindowId) {
  879. const frameData = this.options.framesData.find(frame => frame.windowId == frameWindowId);
  880. if (frameData) {
  881. if (frameData.processor) {
  882. this.stats.add("processed", "frames", 1);
  883. await frameData.processor.run();
  884. const pageData = await frameData.processor.getPageData();
  885. frameElement.removeAttribute(DocUtil.windowIdAttributeName(this.options.sessionId));
  886. if (pageData.content.match(NOSCRIPT_TAG_FOUND) || pageData.content.match(SCRIPT_TAG_FOUND)) {
  887. frameElement.setAttribute("sandbox", "allow-scripts allow-same-origin");
  888. } else {
  889. frameElement.setAttribute("sandbox", "");
  890. }
  891. ProcessorHelper.setFrameContent(frameElement, pageData.content);
  892. this.stats.addAll(pageData);
  893. } else {
  894. this.stats.add("discarded", "frames", 1);
  895. }
  896. }
  897. }
  898. }));
  899. }
  900. }
  901. async resolveHtmlImportURLs() {
  902. const linkElements = Array.from(this.doc.querySelectorAll("link[rel=import][href]"));
  903. if (!this.relImportProcessors) {
  904. this.relImportProcessors = new Map();
  905. }
  906. await Promise.all(linkElements.map(async linkElement => {
  907. const resourceURL = linkElement.href;
  908. linkElement.removeAttribute("href");
  909. const options = Object.create(this.options);
  910. options.insertSingleFileComment = false;
  911. options.insertFaviconLink = false;
  912. options.doc = null;
  913. options.win = null;
  914. options.url = resourceURL;
  915. if (!Util.testIgnoredPath(resourceURL) && Util.testValidPath(resourceURL, this.baseURI, this.options.url)) {
  916. const processor = new Runner(options);
  917. this.relImportProcessors.set(linkElement, processor);
  918. await processor.loadPage();
  919. return processor.initialize();
  920. }
  921. }));
  922. }
  923. async processHtmlImports() {
  924. const linkElements = Array.from(this.doc.querySelectorAll("link[rel=import][href]"));
  925. await Promise.all(linkElements.map(async linkElement => {
  926. const processor = this.relImportProcessors.get(linkElement);
  927. if (processor) {
  928. this.stats.add("processed", "HTML imports", 1);
  929. this.relImportProcessors.delete(linkElement);
  930. const pageData = await processor.getPageData();
  931. linkElement.setAttribute("href", "data:text/html," + pageData.content);
  932. this.stats.addAll(pageData);
  933. } else {
  934. this.stats.add("discarded", "HTML imports", 1);
  935. }
  936. }));
  937. }
  938. resolveStyleAttributeURLs() {
  939. this.doc.querySelectorAll("[style]").forEach(element => {
  940. let styleContent = element.getAttribute("style");
  941. if (this.options.compressCSS) {
  942. styleContent = DocUtil.compressCSS(styleContent);
  943. }
  944. styleContent = ProcessorHelper.resolveStylesheetURLs(styleContent, this.baseURI, this.options);
  945. const declarationList = cssTree.parse(styleContent, { context: "declarationList" });
  946. this.styles.set(element, declarationList);
  947. });
  948. }
  949. async processStyleAttributes() {
  950. return Promise.all(Array.from(this.styles).map(([, declarationList]) =>
  951. ProcessorHelper.processStyle(declarationList.children.toArray(), this.baseURI, this.options, this.cssVariables, this.batchRequest)
  952. ));
  953. }
  954. replaceStyleAttributes() {
  955. this.doc.querySelectorAll("[style]").forEach(async element => {
  956. const declarations = this.styles.get(element);
  957. if (declarations) {
  958. let styleContent = cssTree.generate(declarations);
  959. element.setAttribute("style", styleContent);
  960. } else {
  961. element.setAttribute("style", "");
  962. }
  963. });
  964. }
  965. }
  966. // ---------------
  967. // ProcessorHelper
  968. // ---------------
  969. const REGEXP_AMP = /&/g;
  970. const REGEXP_NBSP = /\u00a0/g;
  971. const REGEXP_START_TAG = /</g;
  972. const REGEXP_END_TAG = />/g;
  973. const REGEXP_URL_HASH = /(#.+?)$/;
  974. const PREFIX_DATA_URI_IMAGE = "data:image/";
  975. const PREFIX_DATA_URI_AUDIO = "data:audio/";
  976. const PREFIX_DATA_URI_VIDEO = "data:video/";
  977. const PREFIX_DATA_URI_IMAGE_SVG = "data:image/svg+xml";
  978. const PREFIX_DATA_URI_NO_MIMETYPE = "data:;";
  979. const PREFIX_DATA_URI_OCTET_STREAM = /^data:(application|binary)\/octet-stream/;
  980. const PREFIX_DATA_URI_NULL_STREAM = /^data:null;/;
  981. const PREFIX_DATA_URI_VND = "data:application/vnd.";
  982. const SINGLE_FILE_VARIABLE_NAME_PREFIX = "--sf-img-";
  983. class ProcessorHelper {
  984. static async evalTemplate(template, options, content, dontReplaceSlash) {
  985. const date = new Date();
  986. const url = DocUtil.parseURL(options.url);
  987. template = await Util.evalTemplateVariable(template, "page-title", () => options.title || "No title", dontReplaceSlash);
  988. template = await Util.evalTemplateVariable(template, "page-language", () => options.info.lang || "No language", dontReplaceSlash);
  989. template = await Util.evalTemplateVariable(template, "page-description", () => options.info.description || "No description", dontReplaceSlash);
  990. template = await Util.evalTemplateVariable(template, "page-author", () => options.info.author || "No author", dontReplaceSlash);
  991. template = await Util.evalTemplateVariable(template, "page-creator", () => options.info.creator || "No creator", dontReplaceSlash);
  992. template = await Util.evalTemplateVariable(template, "page-publisher", () => options.info.publisher || "No publisher", dontReplaceSlash);
  993. template = await Util.evalTemplateVariable(template, "datetime-iso", () => date.toISOString(), dontReplaceSlash);
  994. template = await Util.evalTemplateVariable(template, "date-iso", () => date.toISOString().split("T")[0], dontReplaceSlash);
  995. template = await Util.evalTemplateVariable(template, "time-iso", () => date.toISOString().split("T")[1].split("Z")[0], dontReplaceSlash);
  996. template = await Util.evalTemplateVariable(template, "date-locale", () => date.toLocaleDateString(), dontReplaceSlash);
  997. template = await Util.evalTemplateVariable(template, "time-locale", () => date.toLocaleTimeString(), dontReplaceSlash);
  998. template = await Util.evalTemplateVariable(template, "day-locale", () => String(date.getDate()).padStart(2, "0"), dontReplaceSlash);
  999. template = await Util.evalTemplateVariable(template, "month-locale", () => String(date.getMonth() + 1).padStart(2, "0"), dontReplaceSlash);
  1000. template = await Util.evalTemplateVariable(template, "year-locale", () => String(date.getFullYear()), dontReplaceSlash);
  1001. template = await Util.evalTemplateVariable(template, "datetime-locale", () => date.toLocaleString(), dontReplaceSlash);
  1002. template = await Util.evalTemplateVariable(template, "datetime-utc", () => date.toUTCString(), dontReplaceSlash);
  1003. template = await Util.evalTemplateVariable(template, "day-utc", () => String(date.getUTCDate()).padStart(2, "0"), dontReplaceSlash);
  1004. template = await Util.evalTemplateVariable(template, "month-utc", () => String(date.getUTCMonth() + 1).padStart(2, "0"), dontReplaceSlash);
  1005. template = await Util.evalTemplateVariable(template, "year-utc", () => String(date.getUTCFullYear()), dontReplaceSlash);
  1006. template = await Util.evalTemplateVariable(template, "hours-locale", () => String(date.getHours()).padStart(2, "0"), dontReplaceSlash);
  1007. template = await Util.evalTemplateVariable(template, "minutes-locale", () => String(date.getMinutes()).padStart(2, "0"), dontReplaceSlash);
  1008. template = await Util.evalTemplateVariable(template, "seconds-locale", () => String(date.getSeconds()).padStart(2, "0"), dontReplaceSlash);
  1009. template = await Util.evalTemplateVariable(template, "hours-utc", () => String(date.getUTCHours()).padStart(2, "0"), dontReplaceSlash);
  1010. template = await Util.evalTemplateVariable(template, "minutes-utc", () => String(date.getUTCMinutes()).padStart(2, "0"), dontReplaceSlash);
  1011. template = await Util.evalTemplateVariable(template, "seconds-utc", () => String(date.getUTCSeconds()).padStart(2, "0"), dontReplaceSlash);
  1012. template = await Util.evalTemplateVariable(template, "url-hash", () => url.hash.substring(1), dontReplaceSlash);
  1013. template = await Util.evalTemplateVariable(template, "url-host", () => url.host.replace(/\/$/, ""), dontReplaceSlash);
  1014. template = await Util.evalTemplateVariable(template, "url-hostname", () => url.hostname.replace(/\/$/, ""), dontReplaceSlash);
  1015. template = await Util.evalTemplateVariable(template, "url-href", () => url.href, dontReplaceSlash);
  1016. template = await Util.evalTemplateVariable(template, "url-password", () => url.password, dontReplaceSlash);
  1017. template = await Util.evalTemplateVariable(template, "url-pathname", () => url.pathname.replace(/^\//, "").replace(/\/$/, ""), dontReplaceSlash === undefined ? true : dontReplaceSlash);
  1018. template = await Util.evalTemplateVariable(template, "url-port", () => url.port, dontReplaceSlash);
  1019. template = await Util.evalTemplateVariable(template, "url-protocol", () => url.protocol, dontReplaceSlash);
  1020. template = await Util.evalTemplateVariable(template, "url-search", () => url.search.substring(1), dontReplaceSlash);
  1021. template = await Util.evalTemplateVariable(template, "url-username", () => url.username, dontReplaceSlash);
  1022. template = await Util.evalTemplateVariable(template, "tab-id", () => String(options.tabId || "No tab id"), dontReplaceSlash);
  1023. template = await Util.evalTemplateVariable(template, "url-last-segment", () => Util.getLastSegment(url), dontReplaceSlash);
  1024. if (content) {
  1025. template = await Util.evalTemplateVariable(template, "digest-sha-256", async () => DocUtil.digest("SHA-256", content), dontReplaceSlash);
  1026. template = await Util.evalTemplateVariable(template, "digest-sha-384", async () => DocUtil.digest("SHA-384", content), dontReplaceSlash);
  1027. template = await Util.evalTemplateVariable(template, "digest-sha-512", async () => DocUtil.digest("SHA-512", content), dontReplaceSlash);
  1028. }
  1029. return template;
  1030. }
  1031. static setBackgroundImage(element, url, style) {
  1032. element.style.setProperty("background-blend-mode", "normal", "important");
  1033. element.style.setProperty("background-clip", "content-box", "important");
  1034. element.style.setProperty("background-position", style && style["background-position"] ? style["background-position"] : "center", "important");
  1035. element.style.setProperty("background-color", style && style["background-color"] ? style["background-color"] : "transparent", "important");
  1036. element.style.setProperty("background-image", url, "important");
  1037. element.style.setProperty("background-size", style && style["background-size"] ? style["background-size"] : "100% 100%", "important");
  1038. element.style.setProperty("background-origin", "content-box", "important");
  1039. element.style.setProperty("background-repeat", "no-repeat", "important");
  1040. }
  1041. static processShortcutIcons(doc) {
  1042. let shortcutIcon = Util.findShortcutIcon(Array.from(doc.querySelectorAll("link[href][rel=\"icon\"], link[href][rel=\"shortcut icon\"]")));
  1043. if (!shortcutIcon) {
  1044. shortcutIcon = Util.findShortcutIcon(Array.from(doc.querySelectorAll("link[href][rel*=\"icon\"]")));
  1045. if (shortcutIcon) {
  1046. shortcutIcon.rel = "icon";
  1047. }
  1048. }
  1049. if (shortcutIcon) {
  1050. doc.querySelectorAll("link[href][rel*=\"icon\"]").forEach(linkElement => {
  1051. if (linkElement != shortcutIcon) {
  1052. linkElement.remove();
  1053. }
  1054. });
  1055. }
  1056. }
  1057. static setFrameEmptySrc(frameElement) {
  1058. if (frameElement.tagName == "OBJECT") {
  1059. frameElement.setAttribute("data", "data:text/html,");
  1060. } else {
  1061. frameElement.removeAttribute("src");
  1062. frameElement.removeAttribute("srcdoc");
  1063. }
  1064. }
  1065. static setFrameContent(frameElement, content) {
  1066. if (frameElement.tagName == "OBJECT") {
  1067. frameElement.setAttribute("data", "data:text/html," + content);
  1068. } else {
  1069. frameElement.setAttribute("srcdoc", content);
  1070. frameElement.removeAttribute("src");
  1071. }
  1072. }
  1073. static isolateElements(rootElement) {
  1074. rootElement.querySelectorAll("*:not(style)").forEach(element => {
  1075. if (element.getAttribute(SELECTED_CONTENT_ATTRIBUTE_NAME) == "") {
  1076. element.removeAttribute(SELECTED_CONTENT_ATTRIBUTE_NAME);
  1077. } else if (!element.querySelector("[" + SELECTED_CONTENT_ATTRIBUTE_NAME + "]")) {
  1078. element.remove();
  1079. }
  1080. });
  1081. isolateParentElements(rootElement.parentElement, rootElement);
  1082. function isolateParentElements(parentElement, element) {
  1083. let elementFound = false;
  1084. Array.from(parentElement.childNodes).forEach(node => {
  1085. elementFound = elementFound || (node == element);
  1086. const tagName = node.tagName && node.tagName.toLowerCase();
  1087. if (node != element && tagName != "svg" && tagName != "style" && tagName != "link") {
  1088. if (elementFound) {
  1089. node.remove();
  1090. } else {
  1091. node.hidden = true;
  1092. if (node.childNodes && node.childNodes.length) {
  1093. Array.from(node.childNodes).forEach(node => node.remove());
  1094. }
  1095. }
  1096. }
  1097. });
  1098. element = element.parentElement;
  1099. if (element && element.parentElement && element.parentElement.tagName != "HTML") {
  1100. isolateParentElements(element.parentElement, element);
  1101. }
  1102. }
  1103. }
  1104. static async resolveImportURLs(stylesheetContent, baseURI, options) {
  1105. stylesheetContent = ProcessorHelper.resolveStylesheetURLs(stylesheetContent, baseURI, options);
  1106. const imports = Util.getImportFunctions(stylesheetContent);
  1107. await Promise.all(imports.map(async cssImport => {
  1108. const match = Util.matchImport(cssImport);
  1109. if (match) {
  1110. let resourceURL = Util.normalizeURL(match.resourceURL);
  1111. if (!Util.testIgnoredPath(resourceURL) && Util.testValidPath(resourceURL, baseURI, options.url)) {
  1112. try {
  1113. resourceURL = DocUtil.resolveURL(match.resourceURL, baseURI);
  1114. } catch (error) {
  1115. // ignored
  1116. }
  1117. if (Util.testValidURL(resourceURL, baseURI, options.url)) {
  1118. const downloadOptions = { asDataURI: false, maxResourceSize: options.maxResourceSize, maxResourceSizeEnabled: options.maxResourceSizeEnabled, validateTextContentType: true };
  1119. const content = await DocUtil.getContent(resourceURL, downloadOptions);
  1120. resourceURL = content.resourceURL;
  1121. let importedStylesheetContent = Util.removeCssComments(content.data);
  1122. if (options.compressCSS) {
  1123. importedStylesheetContent = DocUtil.compressCSS(importedStylesheetContent);
  1124. }
  1125. importedStylesheetContent = Util.wrapMediaQuery(importedStylesheetContent, match.media);
  1126. if (stylesheetContent.includes(cssImport)) {
  1127. importedStylesheetContent = await ProcessorHelper.resolveImportURLs(importedStylesheetContent, resourceURL, options);
  1128. stylesheetContent = stylesheetContent.replace(Util.getRegExp(cssImport), importedStylesheetContent);
  1129. }
  1130. }
  1131. }
  1132. }
  1133. }));
  1134. return stylesheetContent;
  1135. }
  1136. static resolveStylesheetURLs(stylesheetContent, baseURI, options) {
  1137. const urlFunctions = Util.getUrlFunctions(stylesheetContent);
  1138. urlFunctions.map(urlFunction => {
  1139. const originalResourceURL = Util.matchURL(urlFunction);
  1140. const resourceURL = Util.normalizeURL(originalResourceURL);
  1141. if (!Util.testIgnoredPath(resourceURL)) {
  1142. if (!resourceURL || Util.testValidPath(resourceURL, baseURI, options.url)) {
  1143. let resolvedURL;
  1144. try {
  1145. resolvedURL = DocUtil.resolveURL(resourceURL, baseURI);
  1146. } catch (error) {
  1147. // ignored
  1148. }
  1149. if (Util.testValidURL(resolvedURL, baseURI, options.url) && resourceURL != resolvedURL && stylesheetContent.includes(urlFunction)) {
  1150. stylesheetContent = stylesheetContent.replace(Util.getRegExp(urlFunction), urlFunction.replace(originalResourceURL, resolvedURL));
  1151. }
  1152. } else {
  1153. let newUrlFunction;
  1154. if (originalResourceURL) {
  1155. newUrlFunction = urlFunction.replace(originalResourceURL, EMPTY_DATA_URI);
  1156. } else {
  1157. newUrlFunction = "url(" + EMPTY_DATA_URI + ")";
  1158. }
  1159. stylesheetContent = stylesheetContent.replace(Util.getRegExp(urlFunction), newUrlFunction);
  1160. }
  1161. } else {
  1162. if (resourceURL.startsWith(DATA_URI_PREFIX)) {
  1163. const escapedResourceURL = resourceURL.replace(REGEXP_AMP, "&amp;").replace(REGEXP_NBSP, "&nbsp;").replace(REGEXP_START_TAG, "&lt;").replace(REGEXP_END_TAG, "&gt;");
  1164. if (escapedResourceURL != resourceURL && stylesheetContent.includes(urlFunction)) {
  1165. stylesheetContent = stylesheetContent.replace(Util.getRegExp(urlFunction), urlFunction.replace(originalResourceURL, escapedResourceURL));
  1166. }
  1167. }
  1168. }
  1169. });
  1170. return stylesheetContent;
  1171. }
  1172. static async resolveLinkStylesheetURLs(resourceURL, baseURI, options) {
  1173. resourceURL = Util.normalizeURL(resourceURL);
  1174. if (resourceURL && resourceURL != baseURI && resourceURL != ABOUT_BLANK_URI) {
  1175. const downloadOptions = { asDataURI: false, maxResourceSize: options.maxResourceSize, maxResourceSizeEnabled: options.maxResourceSizeEnabled, charset: options.charset };
  1176. const content = await DocUtil.getContent(resourceURL, downloadOptions);
  1177. resourceURL = content.resourceURL;
  1178. let stylesheetContent = Util.removeCssComments(content.data);
  1179. if (options.compressCSS) {
  1180. stylesheetContent = DocUtil.compressCSS(stylesheetContent);
  1181. }
  1182. stylesheetContent = await ProcessorHelper.resolveImportURLs(stylesheetContent, resourceURL, options);
  1183. return stylesheetContent;
  1184. }
  1185. }
  1186. static async processStylesheet(cssRules, baseURI, options, cssVariables, batchRequest) {
  1187. const promises = [];
  1188. const removedRules = [];
  1189. for (let cssRule = cssRules.head; cssRule; cssRule = cssRule.next) {
  1190. const ruleData = cssRule.data;
  1191. if (ruleData.type == "Atrule" && ruleData.name == "charset") {
  1192. removedRules.push(cssRule);
  1193. } else if (ruleData.block && ruleData.block.children) {
  1194. if (ruleData.type == "Rule") {
  1195. promises.push(this.processStyle(ruleData.block.children.toArray(), baseURI, options, cssVariables, batchRequest));
  1196. } else if (ruleData.type == "Atrule" && (ruleData.name == "media" || ruleData.name == "supports")) {
  1197. promises.push(this.processStylesheet(ruleData.block.children, baseURI, options, cssVariables, batchRequest));
  1198. } else if (ruleData.type == "Atrule" && ruleData.name == "font-face") {
  1199. promises.push(processFontFaceRule(ruleData));
  1200. }
  1201. }
  1202. }
  1203. removedRules.forEach(cssRule => cssRules.remove(cssRule));
  1204. await Promise.all(promises);
  1205. async function processFontFaceRule(cssRule) {
  1206. await Promise.all(cssRule.block.children.toArray().map(async declaration => {
  1207. if (declaration.type == "Declaration" && declaration.value.children) {
  1208. const urlFunctions = Util.getUrlFunctions(Util.getCSSValue(declaration.value));
  1209. await Promise.all(urlFunctions.map(async urlFunction => {
  1210. const originalResourceURL = Util.matchURL(urlFunction);
  1211. const resourceURL = Util.normalizeURL(originalResourceURL);
  1212. if (!Util.testIgnoredPath(resourceURL)) {
  1213. if (Util.testValidURL(resourceURL, baseURI, options.url)) {
  1214. let { content } = await batchRequest.addURL(resourceURL, true);
  1215. let validResource = content == EMPTY_DATA_URI || content.startsWith(PREFIX_DATA_URI_VND) || content.startsWith(PREFIX_DATA_URI_IMAGE_SVG);
  1216. if (!validResource) {
  1217. validResource = await DocUtil.validFont(urlFunction);
  1218. }
  1219. if (!validResource) {
  1220. content = EMPTY_DATA_URI;
  1221. }
  1222. declaration.value.children.forEach(token => {
  1223. if (token.type == "Url" && DocUtil.removeQuotes(Util.getCSSValue(token.value)) == originalResourceURL) {
  1224. token.value.value = content;
  1225. }
  1226. });
  1227. }
  1228. }
  1229. }));
  1230. }
  1231. }));
  1232. }
  1233. }
  1234. static async processStyle(declarations, baseURI, options, cssVariables, batchRequest) {
  1235. await Promise.all(declarations.map(async declaration => {
  1236. if (declaration.type == "Declaration" && declaration.value.children) {
  1237. const urlFunctions = Util.getUrlFunctions(Util.getCSSValue(declaration.value));
  1238. await Promise.all(urlFunctions.map(async urlFunction => {
  1239. const originalResourceURL = Util.matchURL(urlFunction);
  1240. const resourceURL = Util.normalizeURL(originalResourceURL);
  1241. if (!Util.testIgnoredPath(resourceURL)) {
  1242. if (Util.testValidURL(resourceURL, baseURI, options.url)) {
  1243. let { content, indexResource, duplicate } = await batchRequest.addURL(resourceURL, true, true);
  1244. let variableDefined;
  1245. const tokens = [];
  1246. findURLToken(originalResourceURL, declaration.value.children, (token, parent, rootFunction) => {
  1247. if (duplicate && options.groupDuplicateImages && rootFunction) {
  1248. const value = cssTree.parse("var(" + SINGLE_FILE_VARIABLE_NAME_PREFIX + indexResource + ")", { context: "value" }).children.head;
  1249. tokens.push({ parent, token, value });
  1250. variableDefined = true;
  1251. } else {
  1252. token.data.value.value = content;
  1253. }
  1254. });
  1255. if (variableDefined) {
  1256. cssVariables.set(indexResource, content);
  1257. tokens.forEach(({ parent, token, value }) => parent.replace(token, value));
  1258. }
  1259. }
  1260. }
  1261. }));
  1262. }
  1263. }));
  1264. function findURLToken(url, children, callback, depth = 0) {
  1265. for (let token = children.head; token; token = token.next) {
  1266. if (token.data.children) {
  1267. findURLToken(url, token.data.children, callback, depth + 1);
  1268. }
  1269. if (token.data.type == "Url" && DocUtil.removeQuotes(Util.getCSSValue(token.data.value)) == url) {
  1270. callback(token, children, depth == 0);
  1271. }
  1272. }
  1273. }
  1274. }
  1275. static async processAttribute(doc, resourceElements, attributeName, prefixDataURI, baseURI, options, cssVariables, styles, batchRequest, processDuplicates, removeElementIfMissing) {
  1276. await Promise.all(Array.from(resourceElements).map(async resourceElement => {
  1277. let resourceURL = resourceElement.getAttribute(attributeName);
  1278. resourceURL = Util.normalizeURL(resourceURL);
  1279. if (!Util.testIgnoredPath(resourceURL)) {
  1280. resourceElement.setAttribute(attributeName, EMPTY_IMAGE);
  1281. if (Util.testValidPath(resourceURL, baseURI, options.url)) {
  1282. try {
  1283. resourceURL = DocUtil.resolveURL(resourceURL, baseURI);
  1284. } catch (error) {
  1285. // ignored
  1286. }
  1287. if (Util.testValidURL(resourceURL, baseURI, options.url)) {
  1288. const { content, indexResource, duplicate } = await batchRequest.addURL(resourceURL, true, resourceElement.tagName == "IMG" && attributeName == "src");
  1289. if (removeElementIfMissing && content == EMPTY_DATA_URI) {
  1290. resourceElement.remove();
  1291. } else {
  1292. if (content.startsWith(prefixDataURI) || content.startsWith(PREFIX_DATA_URI_NO_MIMETYPE) || content.match(PREFIX_DATA_URI_OCTET_STREAM) || content.match(PREFIX_DATA_URI_NULL_STREAM)) {
  1293. const isSVG = content.startsWith(PREFIX_DATA_URI_IMAGE_SVG);
  1294. if (processDuplicates && duplicate && options.groupDuplicateImages && !isSVG) {
  1295. if (Util.replaceImageSource(resourceElement, SINGLE_FILE_VARIABLE_NAME_PREFIX + indexResource, options)) {
  1296. cssVariables.set(indexResource, content);
  1297. const declarationList = cssTree.parse(resourceElement.getAttribute("style"), { context: "declarationList" });
  1298. styles.set(resourceElement, declarationList);
  1299. } else {
  1300. resourceElement.setAttribute(attributeName, content);
  1301. }
  1302. } else {
  1303. resourceElement.setAttribute(attributeName, content);
  1304. }
  1305. }
  1306. }
  1307. }
  1308. }
  1309. }
  1310. }));
  1311. }
  1312. static async processXLinks(resourceElements, baseURI, options, batchRequest) {
  1313. const attributeName = "xlink:href";
  1314. await Promise.all(Array.from(resourceElements).map(async resourceElement => {
  1315. const originalResourceURL = resourceElement.getAttribute(attributeName);
  1316. let resourceURL = Util.normalizeURL(originalResourceURL);
  1317. if (Util.testValidPath(resourceURL, baseURI, options.url) && !Util.testIgnoredPath(resourceURL)) {
  1318. resourceElement.setAttribute(attributeName, EMPTY_IMAGE);
  1319. try {
  1320. resourceURL = DocUtil.resolveURL(resourceURL, baseURI);
  1321. } catch (error) {
  1322. // ignored
  1323. }
  1324. if (Util.testValidURL(resourceURL, baseURI, options.url)) {
  1325. try {
  1326. const { content } = await batchRequest.addURL(resourceURL);
  1327. const DOMParser = DocUtil.getParser();
  1328. if (DOMParser) {
  1329. const svgDoc = new DOMParser().parseFromString(content, "image/svg+xml");
  1330. const hashMatch = originalResourceURL.match(REGEXP_URL_HASH);
  1331. if (hashMatch && hashMatch[0]) {
  1332. const symbolElement = svgDoc.querySelector(hashMatch[0]);
  1333. if (symbolElement) {
  1334. resourceElement.setAttribute(attributeName, hashMatch[0]);
  1335. resourceElement.parentElement.insertBefore(symbolElement, resourceElement.parentElement.firstChild);
  1336. }
  1337. } else {
  1338. resourceElement.setAttribute(attributeName, "data:image/svg+xml," + content);
  1339. }
  1340. } else {
  1341. resourceElement.setAttribute(attributeName, "data:image/svg+xml," + content);
  1342. }
  1343. } catch (error) {
  1344. /* ignored */
  1345. }
  1346. }
  1347. } else if (resourceURL == options.url) {
  1348. resourceElement.setAttribute(attributeName, originalResourceURL.substring(resourceURL.length));
  1349. }
  1350. }));
  1351. }
  1352. static async processSrcset(resourceElements, attributeName, prefixDataURI, baseURI, options, batchRequest) {
  1353. await Promise.all(Array.from(resourceElements).map(async resourceElement => {
  1354. const srcset = DocUtil.parseSrcset(resourceElement.getAttribute(attributeName));
  1355. const srcsetValues = await Promise.all(srcset.map(async srcsetValue => {
  1356. let resourceURL = Util.normalizeURL(srcsetValue.url);
  1357. if (!Util.testIgnoredPath(resourceURL)) {
  1358. if (Util.testValidPath(resourceURL, baseURI, options.url)) {
  1359. try {
  1360. resourceURL = DocUtil.resolveURL(resourceURL, baseURI);
  1361. } catch (error) {
  1362. // ignored
  1363. }
  1364. if (Util.testValidURL(resourceURL, baseURI, options.url)) {
  1365. const { content } = await batchRequest.addURL(resourceURL, true);
  1366. if (!content.startsWith(prefixDataURI) && !content.startsWith(PREFIX_DATA_URI_NO_MIMETYPE) && !content.match(PREFIX_DATA_URI_OCTET_STREAM) && !content.match(PREFIX_DATA_URI_NULL_STREAM)) {
  1367. resourceElement.setAttribute(attributeName, EMPTY_IMAGE);
  1368. }
  1369. return content + (srcsetValue.w ? " " + srcsetValue.w + "w" : srcsetValue.d ? " " + srcsetValue.d + "x" : "");
  1370. } else {
  1371. return "";
  1372. }
  1373. } else {
  1374. return "";
  1375. }
  1376. } else {
  1377. return resourceURL + (srcsetValue.w ? " " + srcsetValue.w + "w" : srcsetValue.d ? " " + srcsetValue.d + "x" : "");
  1378. }
  1379. }));
  1380. resourceElement.setAttribute(attributeName, srcsetValues.join(", "));
  1381. }));
  1382. }
  1383. }
  1384. // ----
  1385. // Util
  1386. // ----
  1387. const DATA_URI_PREFIX = "data:";
  1388. const BLOB_URI_PREFIX = "blob:";
  1389. const HTTP_URI_PREFIX = /^https?:\/\//;
  1390. const FILE_URI_PREFIX = /^file:\/\//;
  1391. const EMPTY_URL = /^https?:\/\/+\s*$/;
  1392. const ABOUT_BLANK_URI = "about:blank";
  1393. const NOT_EMPTY_URL = /^(https?:\/\/|file:\/\/|blob:).+/;
  1394. const REGEXP_URL_FN = /(url\s*\(\s*'(.*?)'\s*\))|(url\s*\(\s*"(.*?)"\s*\))|(url\s*\(\s*(.*?)\s*\))/gi;
  1395. const REGEXP_URL_SIMPLE_QUOTES_FN = /^url\s*\(\s*'(.*?)'\s*\)$/i;
  1396. const REGEXP_URL_DOUBLE_QUOTES_FN = /^url\s*\(\s*"(.*?)"\s*\)$/i;
  1397. const REGEXP_URL_NO_QUOTES_FN = /^url\s*\(\s*(.*?)\s*\)$/i;
  1398. const REGEXP_IMPORT_FN = /(@import\s*url\s*\(\s*'(.*?)'\s*\)\s*(.*?)(;|$|}))|(@import\s*url\s*\(\s*"(.*?)"\s*\)\s*(.*?)(;|$|}))|(@import\s*url\s*\(\s*(.*?)\s*\)\s*(.*?)(;|$|}))|(@import\s*'(.*?)'\s*(.*?)(;|$|}))|(@import\s*"(.*?)"\s*(.*?)(;|$|}))|(@import\s*(.*?)\s*(.*?)(;|$|}))/gi;
  1399. const REGEXP_IMPORT_URL_SIMPLE_QUOTES_FN = /@import\s*url\s*\(\s*'(.*?)'\s*\)\s*(.*?)(;|$|})/i;
  1400. const REGEXP_IMPORT_URL_DOUBLE_QUOTES_FN = /@import\s*url\s*\(\s*"(.*?)"\s*\)\s*(.*?)(;|$|})/i;
  1401. const REGEXP_IMPORT_URL_NO_QUOTES_FN = /@import\s*url\s*\(\s*(.*?)\s*\)\s*(.*?)(;|$|})/i;
  1402. const REGEXP_IMPORT_SIMPLE_QUOTES_FN = /@import\s*'(.*?)'\s*(.*?)(;|$|})/i;
  1403. const REGEXP_IMPORT_DOUBLE_QUOTES_FN = /@import\s*"(.*?)"\s*(.*?)(;|$|})/i;
  1404. const REGEXP_IMPORT_NO_QUOTES_FN = /@import\s*(.*?)\s*(.*?)(;|$|})/i;
  1405. const REGEXP_ESCAPE = /([{}()^$&.*?/+|[\\\\]|\]|-)/g;
  1406. class Util {
  1407. static normalizeURL(url) {
  1408. if (!url || url.startsWith(DATA_URI_PREFIX)) {
  1409. return url;
  1410. } else {
  1411. return url.split("#")[0];
  1412. }
  1413. }
  1414. static getCSSValue(value) {
  1415. let result = "";
  1416. try {
  1417. result = cssTree.generate(value);
  1418. } catch (error) {
  1419. // ignored
  1420. }
  1421. return result;
  1422. }
  1423. static getOnEventAttributeNames(doc) {
  1424. const element = doc.createElement("div");
  1425. const attributeNames = [];
  1426. for (const propertyName in element) {
  1427. if (propertyName.startsWith("on")) {
  1428. attributeNames.push(propertyName);
  1429. }
  1430. }
  1431. return attributeNames;
  1432. }
  1433. static async evalTemplateVariable(template, variableName, valueGetter, dontReplaceSlash) {
  1434. const replaceRegExp = new RegExp("{\\s*" + variableName + "\\s*}", "g");
  1435. if (template.match(replaceRegExp)) {
  1436. let value = await valueGetter();
  1437. if (!dontReplaceSlash) {
  1438. value = value.replace(/\/+/g, "_");
  1439. }
  1440. return template.replace(replaceRegExp, value);
  1441. }
  1442. return template;
  1443. }
  1444. static getLastSegment(url) {
  1445. let lastSegmentMatch = url.pathname.match(/\/([^/]+)$/), lastSegment = lastSegmentMatch && lastSegmentMatch[0];
  1446. if (!lastSegment) {
  1447. lastSegmentMatch = url.href.match(/([^/]+)\/?$/);
  1448. lastSegment = lastSegmentMatch && lastSegmentMatch[0];
  1449. }
  1450. if (!lastSegment) {
  1451. lastSegmentMatch = lastSegment.match(/(.*)<\.[^.]+$/);
  1452. lastSegment = lastSegmentMatch && lastSegmentMatch[0];
  1453. }
  1454. if (!lastSegment) {
  1455. lastSegment = url.hostname.replace(/\/+/g, "_").replace(/\/$/, "");
  1456. }
  1457. lastSegment.replace(/\/$/, "").replace(/^\//, "");
  1458. return lastSegment;
  1459. }
  1460. static getRegExp(string) {
  1461. return new RegExp(string.replace(REGEXP_ESCAPE, "\\$1"), "gi");
  1462. }
  1463. static getUrlFunctions(stylesheetContent) {
  1464. return Array.from(new Set(stylesheetContent.match(REGEXP_URL_FN) || []));
  1465. }
  1466. static getImportFunctions(stylesheetContent) {
  1467. return stylesheetContent.match(REGEXP_IMPORT_FN) || [];
  1468. }
  1469. static findShortcutIcon(shortcutIcons) {
  1470. shortcutIcons = shortcutIcons.filter(linkElement => linkElement.href != EMPTY_IMAGE);
  1471. shortcutIcons.sort((linkElement1, linkElement2) => (parseInt(linkElement2.sizes, 10) || 16) - (parseInt(linkElement1.sizes, 10) || 16));
  1472. return shortcutIcons[0];
  1473. }
  1474. static matchURL(stylesheetContent) {
  1475. const match = stylesheetContent.match(REGEXP_URL_SIMPLE_QUOTES_FN) ||
  1476. stylesheetContent.match(REGEXP_URL_DOUBLE_QUOTES_FN) ||
  1477. stylesheetContent.match(REGEXP_URL_NO_QUOTES_FN);
  1478. return match && match[1];
  1479. }
  1480. static testIgnoredPath(resourceURL) {
  1481. return resourceURL && (resourceURL.startsWith(DATA_URI_PREFIX) || resourceURL == ABOUT_BLANK_URI);
  1482. }
  1483. static testValidPath(resourceURL, baseURI, docURL) {
  1484. return resourceURL && resourceURL != baseURI && resourceURL != docURL && !resourceURL.match(EMPTY_URL);
  1485. }
  1486. static testValidURL(resourceURL, baseURI, docURL) {
  1487. return Util.testValidPath(resourceURL, baseURI, docURL) && (resourceURL.match(HTTP_URI_PREFIX) || resourceURL.match(FILE_URI_PREFIX) || resourceURL.startsWith(BLOB_URI_PREFIX)) && resourceURL.match(NOT_EMPTY_URL);
  1488. }
  1489. static matchImport(stylesheetContent) {
  1490. const match = stylesheetContent.match(REGEXP_IMPORT_URL_SIMPLE_QUOTES_FN) ||
  1491. stylesheetContent.match(REGEXP_IMPORT_URL_DOUBLE_QUOTES_FN) ||
  1492. stylesheetContent.match(REGEXP_IMPORT_URL_NO_QUOTES_FN) ||
  1493. stylesheetContent.match(REGEXP_IMPORT_SIMPLE_QUOTES_FN) ||
  1494. stylesheetContent.match(REGEXP_IMPORT_DOUBLE_QUOTES_FN) ||
  1495. stylesheetContent.match(REGEXP_IMPORT_NO_QUOTES_FN);
  1496. if (match) {
  1497. const [, resourceURL, media] = match;
  1498. return { resourceURL, media };
  1499. }
  1500. }
  1501. static removeCssComments(stylesheetContent) {
  1502. let start, end;
  1503. do {
  1504. start = stylesheetContent.indexOf("/*");
  1505. end = stylesheetContent.indexOf("*/", start + 2);
  1506. if (start != -1 && end != -1) {
  1507. stylesheetContent = stylesheetContent.substring(0, start) + stylesheetContent.substr(end + 2);
  1508. }
  1509. } while (start != -1 && end != -1);
  1510. return stylesheetContent;
  1511. }
  1512. static wrapMediaQuery(stylesheetContent, mediaQuery) {
  1513. if (mediaQuery) {
  1514. return "@media " + mediaQuery + "{ " + stylesheetContent + " }";
  1515. } else {
  1516. return stylesheetContent;
  1517. }
  1518. }
  1519. static replaceImageSource(imgElement, variableName, options) {
  1520. const dataAttributeName = DocUtil.imagesAttributeName(options.sessionId);
  1521. if (imgElement.getAttribute(dataAttributeName) != null) {
  1522. const imgData = options.imageData[Number(imgElement.getAttribute(dataAttributeName))];
  1523. if (imgData.replaceable) {
  1524. imgElement.setAttribute("src", `data:image/svg+xml,<svg xmlns="http://www.w3.org/2000/svg" width="${imgData.size.pxWidth}" height="${imgData.size.pxHeight}"><rect fill-opacity="0"/></svg>`);
  1525. const backgroundStyle = {};
  1526. const backgroundSize = (imgData.objectFit == "content" || imgData.objectFit == "cover") && imgData.objectFit;
  1527. if (backgroundSize) {
  1528. backgroundStyle["background-size"] = imgData.objectFit;
  1529. }
  1530. if (imgData.objectPosition) {
  1531. backgroundStyle["background-position"] = imgData.objectPosition;
  1532. }
  1533. if (imgData.backgroundColor) {
  1534. backgroundStyle["background-color"] = imgData.backgroundColor;
  1535. }
  1536. ProcessorHelper.setBackgroundImage(imgElement, "var(" + variableName + ")", backgroundStyle);
  1537. imgElement.removeAttribute(dataAttributeName);
  1538. return true;
  1539. }
  1540. }
  1541. }
  1542. }
  1543. function log(...args) {
  1544. console.log("S-File <core> ", ...args); // eslint-disable-line no-console
  1545. }
  1546. // -----
  1547. // Stats
  1548. // -----
  1549. const STATS_DEFAULT_VALUES = {
  1550. discarded: {
  1551. "HTML bytes": 0,
  1552. "hidden elements": 0,
  1553. "HTML imports": 0,
  1554. scripts: 0,
  1555. objects: 0,
  1556. "audio sources": 0,
  1557. "video sources": 0,
  1558. frames: 0,
  1559. "CSS rules": 0,
  1560. canvas: 0,
  1561. stylesheets: 0,
  1562. resources: 0,
  1563. medias: 0
  1564. },
  1565. processed: {
  1566. "HTML bytes": 0,
  1567. "hidden elements": 0,
  1568. "HTML imports": 0,
  1569. scripts: 0,
  1570. objects: 0,
  1571. "audio sources": 0,
  1572. "video sources": 0,
  1573. frames: 0,
  1574. "CSS rules": 0,
  1575. canvas: 0,
  1576. stylesheets: 0,
  1577. resources: 0,
  1578. medias: 0
  1579. }
  1580. };
  1581. class Stats {
  1582. constructor(options) {
  1583. this.options = options;
  1584. if (options.displayStats) {
  1585. this.data = JSON.parse(JSON.stringify(STATS_DEFAULT_VALUES));
  1586. }
  1587. }
  1588. set(type, subType, value) {
  1589. if (this.options.displayStats) {
  1590. this.data[type][subType] = value;
  1591. }
  1592. }
  1593. add(type, subType, value) {
  1594. if (this.options.displayStats) {
  1595. this.data[type][subType] += value;
  1596. }
  1597. }
  1598. addAll(pageData) {
  1599. if (this.options.displayStats) {
  1600. Object.keys(this.data.discarded).forEach(key => this.add("discarded", key, pageData.stats.discarded[key] || 0));
  1601. Object.keys(this.data.processed).forEach(key => this.add("processed", key, pageData.stats.processed[key] || 0));
  1602. }
  1603. }
  1604. }
  1605. return { getClass };
  1606. })();