-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathInit.sh
114 lines (86 loc) · 3.46 KB
/
Init.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
#
# From a fresh machine
apt-get update # updates the package index cache
apt-get upgrade -y # updates packages
#
# installs system tools
apt-get install -y bzip2 gcc git # system tools
apt-get install -y htop screen vim wget # system tools
apt install make
apt install openjdk-14-jre-headless
apt-get upgrade -y bash # upgrades bash if necessary
apt-get clean # cleans up the package index cache
#
# Install docker and compose?
apt install docker.io
apt install docker-compose
#
# Install Miniconda
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O Miniconda.sh
bash Miniconda.sh -b # installs it
rm -rf Miniconda.sh # removes the installer
export PATH="/root/miniconda3/bin:$PATH" # prepends the new path
#
# In case it's needed... to remove Miniconda
#rm -rf ~/miniconda3
#rm -rf /root/miniconda3
#
# Alternative is to get the full Anaconda
wget https://repo.anaconda.com/archive/Anaconda3-2020.07-Linux-x86_64.sh -O Anaconda.sh
bash Anaconda.sh -b # installs it
rm -rf Anaconda.sh # removes the installer
export PATH="/root/anaconda3/bin:$PATH" # prepends the new path
#
# Virtual Python Env. with Faust & co on 3.8
conda create -n faust-poc python=3.8
conda activate faust-poc
pip install -r requirements.txt
#
# Download and unpak kafka for the scripts
wget http://mirror.cogentco.com/pub/apache/kafka/2.6.0/kafka_2.13-2.6.0.tgz
curl "http://mirror.cogentco.com/pub/apache/kafka/2.6.0/kafka_2.13-2.6.0.tgz" -o ~/downloads/kafka.tgz
mkdir ~/kafka && cd ~/kafka
tar -xvzf ~/downloads/kafka.tgz --strip 1
#
# Ref. for looking/generating Kafka traffic - test app to validate the infra
~/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic output --from-beginning
~/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic input --from-beginning
~/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic colors_topic --from-beginning
#
# Produce Kafka traffic - test app to validate the infra
~/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic input
#
# Weblogs streaming app topics
~/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic weblogs --from-beginning
~/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic weblogs_tokens --from-beginning
~/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic weblogs_stats --from-beginning
~/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic weblogs_persistence --from-beginning
#
# Run the kafka docker-compose to "power-up" the infra
make kafka-elk
#
# ... to take it down and cleanup
make kafka-elk-down
#
# Run the ElasticSearch app aka Weblogs streaming
make run-es-app
# In debug
make run-es-app-debug
#
# Standlalone app to make sure nothing is broken
make run-standalone-app:
make run-standalone-app-debug:
#
# Test elasticsearch is alive
curl -X GET "localhost:9200/_cat/nodes?v&pretty"
#
# client run
# python -m src.es_client_request -i <ip_address> -u "<user_agent>" -r "<request>" -l <lo_byte> -h <hi_byte>
python -m src.es_client_request -i 63.110.194.22 -u "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) Gecko/20100101 Firefox/78.0" -r "/6d8a9754-e8c7-4193-8491-58b2122c1c10" -l 0 -h 289
#
# Tricky setings with es? bootstrap checks
#
# max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]
vi /etc/sysctl.conf
# ++ > vm.max_map_count=262144
sudo sysctl -p