-
Notifications
You must be signed in to change notification settings - Fork 0
/
index.js
145 lines (120 loc) · 3.16 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
/**
Please see README.md once before diving in :)
*/
const fs = require("fs");
const readline = require("readline");
/**
Some Resuable Utilities
*/
function isValidURL(someString) {
try {
return !!new URL(someString);
} catch (err) {
return false;
}
}
async function isUp(url) {
try {
return (await fetch(url, { method: "HEAD" })).status === 200
? "Online"
: "Offline";
} catch (err) {
return `Not Reachable`
}
}
async function sleep(timeInSec) { return new Promise(res => setTimeout(res, timeInSec * 1000)) }
/**
Advanced concurrency pattern
*/
async function* asyncMap(iterable, mapFn, numWorkers = 3, ...otherArgs) {
try {
const workerCount =
numWorkers > iterable.length ? iterable.length : numWorkers;
const iter = Array.from(iterable).values();
const workers = new Array(workerCount)
.fill(iter)
.map((iter, idx) => mapFn.call(null, iter, idx + 1, ...otherArgs));
// dispatch multiple workers at the same time
const results = (await Promise.all(workers)).flat();
// return results as iterable
for (let result of results) {
yield result;
}
return true;
} catch (err) {
throw err;
}
}
/**
Actual Heavylifting Parts
*/
const readUrlsFromFile = (filePath) => {
return new Promise((resolve, reject) => {
try {
const lineReader = readline.createInterface({
input: fs.createReadStream(filePath),
});
const links = [];
lineReader.on("line", (link) => {
isValidURL(link) ? links.push(link) : () => { };
});
lineReader.on("close", async () => {
resolve(links);
});
} catch (err) {
reject(err);
}
});
};
const prettyPrint = ({ url, status, at }) => `${url} | ${status} | ${at}`
const checkIfOnline = async (urls, workerId, oFile) => {
const results = [];
for (const url of urls) {
try {
await sleep(0.2);
const res = {
url,
status: await isUp(url),
at: new Date().toLocaleString(),
workerId,
};
str = (prettyPrint(res) + ` | ${workerId} \n`);
process.stdout.write(str);
oFile.write(str);
results.push(res);
} catch (err) {
const res = {
url,
status: false,
at: new Date().toLocaleString(),
error: true,
errorMsg: err.message || err,
workerId,
};
str = (prettyPrint(res) + ` | ${workerId}`);
process.stdout.write(str);
oFile.write(str);
results.push(res);
}
}
return results;
};
/**
The entrypoint
*/
const main = async () => {
console.time("time");
const links = await readUrlsFromFile("./links.txt");
const oFile = fs.createWriteStream("./links.status.txt")
const checkAllLinks = asyncMap(links, checkIfOnline, (numWorkers = Math.floor(links.length / 4)), oFile);
for await (let res of checkAllLinks) {
// Do Nothing - Just Consume - Wish there were a better way !
}
oFile.end();
console.log(`Total Links Scanned: ${links.length}`);
console.timeEnd("time");
return process.exit(0);
};
main()
// On force kill - remove all scheduled promises as well
process.on('SIGINT', () => process.exit(10))