|
|
@@ -101,8 +101,8 @@ async function runNextTask(tasks, options) {
|
|
|
task.status = "processing";
|
|
|
const pageData = await capturePage(options);
|
|
|
task.status = "processed";
|
|
|
- task.filename = pageData.filename;
|
|
|
if (pageData && options.crawlLinks && task.depth < options.crawlMaxDepth) {
|
|
|
+ task.filename = pageData.filename;
|
|
|
let newTasks = pageData.links
|
|
|
.map(urlLink => ({ url: rewriteURL(urlLink, options.urlRewriteRules), originalUrl: urlLink, depth: task.depth + 1 }))
|
|
|
.filter(task => task.url && VALID_URL_TEST.test(task.url) && !tasks.find(otherTask => otherTask.url == task.url));
|