-
Notifications
You must be signed in to change notification settings - Fork 1
/
filtering.py
48 lines (31 loc) · 1018 Bytes
/
filtering.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
import sys
import time
from ast import literal_eval
filename = sys.argv[1]
out_filename = filename.replace(".txt", "_filtered.txt")
out_f_p = open(out_filename, "wt")
start_time = time.time()
with open(filename, "rt") as f_p:
for line in f_p:
try:
data = literal_eval(line)
date_download = data["date_download"]
digest = data["digest"]
raw_content = data["raw_content"]
# Some filtering rules
if "<" in raw_content:
continue
if ">" in raw_content:
continue
if "http:" in raw_content:
continue
if "https:" in raw_content:
continue
if "�" in raw_content:
continue
out_f_p.write(raw_content + "\n")
except Exception as e:
print(e)
continue
print("Filtering for", filename, "took:", round(time.time() - start_time, 2), "seconds.")
out_f_p.close()