forked from BorderlessNomad/DataPlay
-
Notifications
You must be signed in to change notification settings - Fork 0
/
cassandra.sh
195 lines (148 loc) · 5.65 KB
/
cassandra.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
#!/bin/bash
# This is setup script for Cassandra Single-Node server.
set -ex
if [ "$(id -u)" != "0" ]; then
echo >&2 "Error: This script must be run as user 'root'";
exit 1
fi
IP=`ifconfig eth0 | grep "inet addr" | awk -F: '{print $2}' | awk '{print $1}'`
MAX_RETRIES="200"
TIMEOUT="5"
timestamp () {
date +"%F %T,%3N"
}
setuphost () {
HOSTNAME=$(hostname)
HOSTLOCAL="127.0.1.1"
echo "$HOSTLOCAL $HOSTNAME" >> /etc/hosts
}
install_java () {
echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \
apt-add-repository -y ppa:webupd8team/java && \
apt-get update && \
apt-get install -y axel oracle-java7-installer && \
apt-get autoclean
echo "export JAVA_HOME=/usr/lib/jvm/java-7-oracle" >> /etc/profile.d/dataplay.sh
. /etc/profile
}
check_cassandra() {
TRY="1"
if [[ $TRY -ge $MAX_RETRIES ]]; then
echo >&2 "Error: Unable Connect to Cassandra."; exit 1;
fi
until [[ $TRY -lt $MAX_RETRIES ]] && cqlsh $IP -e "exit" ; do
echo "Connect: attempt $TRY failed! trying again in $TIMEOUT seconds..."
TRY=$[$TRY+1]
sleep $TIMEOUT
done
}
restart_cassandra() {
/etc/init.d/cassandra restart
echo "Waiting for Cassandra restart..."
check_cassandra
echo "Cassandra is UP!"
}
install_cassandra () {
echo "deb http://debian.datastax.com/community 2.2 main" >> /etc/apt/sources.list.d/cassandra.sources.list && \
echo "deb-src http://debian.datastax.com/community 2.2 main" >> /etc/apt/sources.list.d/cassandra.sources.list && \
curl -L http://debian.datastax.com/debian/repo_key | sudo apt-key add - && \
apt-get update && \
apt-get install -y cassandra
echo "export CASSANDRA_CONFIG=/etc/cassandra" >> /etc/profile.d/dataplay.sh
. /etc/profile
}
configure_cassandra () {
# sed -i -e "s/num_tokens/\#num_tokens/" /etc/cassandra/cassandra.yaml # Disable virtual nodes
sed -i -e "s/^listen_address.*/listen_address: $IP/" /etc/cassandra/cassandra.yaml # Listen on IP of the container
sed -i -e "s/^rpc_address.*/rpc_address: $IP/" /etc/cassandra/cassandra.yaml # Enable Remote connections
# sed -i -e "s/^broadcast_rpc_address.*/broadcast_rpc_address: $IP/" /etc/cassandra/cassandra.yaml # Enable Remote connections
sed -i -e "s/- seeds: \"127.0.0.1\"/- seeds: \"$IP\"/" /etc/cassandra/cassandra.yaml # Be your own seed
# With virtual nodes disabled, we need to manually specify the token
# sed -i -e "s/# JVM_OPTS=\"$JVM_OPTS -Djava.rmi.server.hostname=<public name>\"/ JVM_OPTS=\"$JVM_OPTS -Djava.rmi.server.hostname=$IP\"/" /etc/cassandra/cassandra-env.sh
# echo "JVM_OPTS=\"\$JVM_OPTS -Dcassandra.initial_token=0\"" >> /etc/cassandra/cassandra-env.sh
# netstat -an | grep 9160.*LISTEN
restart_cassandra
}
install_opscenter () {
apt-get install -y opscenter && \
/etc/init.d/opscenterd start
# Connect using http://<IP>:8888
}
export_variables () {
. /etc/profile
su - ubuntu -c ". /etc/profile"
}
import_data () {
LASTDATE=$(date +%Y-%m-%d) # Today
BACKUP_HOST="109.231.121.72" # Flexiant
BACKUP_PORT="8080"
BACKUP_DIR="cassandra/$LASTDATE"
BACKUP_USER="playgen"
BACKUP_PASS="D@taP1aY"
KEYSPACE="dataplay"
BACKUP_SCHEMA_FILE="$KEYSPACE-schema.cql"
BACKUP_DATA_FILE="$KEYSPACE-data.tar.gz"
CASSANDRA_DIR="/var/lib/cassandra"
DATA_DIR="$CASSANDRA_DIR/data"
LOG_DIR="$CASSANDRA_DIR/commitlog"
SOURCE_DIR="/tmp/cassandra-data"
i="1"
if [[ $i -ge $MAX_RETRIES ]]; then
echo >&2 "Error: Unable to fetch '$BACKUP_SCHEMA_FILE' from backup server."; exit 1;
fi
until [[ $i -lt $MAX_RETRIES ]] && axel -a "http://$BACKUP_USER:$BACKUP_PASS@$BACKUP_HOST:$BACKUP_PORT/$BACKUP_DIR/$BACKUP_SCHEMA_FILE"; do
LASTDATE=$(date +%Y-%m-%d --date="$LASTDATE -1 days") # Decrement by 1 Day
BACKUP_DIR="cassandra/$LASTDATE"
echo "Latest $BACKUP_SCHEMA_FILE backup not available, trying $LASTDATE"
i=$[$i+1]
done
j="1"
if [[ $j -ge $MAX_RETRIES ]]; then
echo >&2 "Error: Unable to fetch '$BACKUP_DATA_FILE' from backup server."; exit 1;
fi
until [[ $j -lt $MAX_RETRIES ]] && axel -a "http://$BACKUP_USER:$BACKUP_PASS@$BACKUP_HOST:$BACKUP_PORT/$BACKUP_DIR/$BACKUP_DATA_FILE"; do
LASTDATE=$(date +%Y-%m-%d --date="$LASTDATE -1 days") # Decrement by 1 Day
BACKUP_DIR="cassandra/$LASTDATE"
echo "Latest $BACKUP_DATA_FILE backup not available, trying $LASTDATE"
j=$[$j+1]
done
check_cassandra
cqlsh $IP -f $BACKUP_SCHEMA_FILE
/etc/init.d/cassandra stop
mkdir -p $SOURCE_DIR
tar -xzvf $BACKUP_DATA_FILE -C $SOURCE_DIR
SOURCE_TABLES=`ls -l $SOURCE_DIR | egrep '^d' | awk '{print $9}'`
for table in $SOURCE_TABLES; do
table_name=$(echo $table | awk -F'-' '{print $1}')
mv $SOURCE_DIR/$table/* $DATA_DIR/$KEYSPACE/$table_name-*
done
chown -R cassandra:cassandra $DATA_DIR/$KEYSPACE # Fix permissions
rm -rf $LOG_DIR/*.log
restart_cassandra
# nodetool -h $IP repair $KEYSPACE
rm -rf $SOURCE_DIR
}
update_iptables () {
# iptables -A INPUT -p tcp --dport 7000 -j ACCEPT # Internode communication (not used if TLS enabled) Used internal by Cassandra
iptables -A INPUT -p tcp --dport 7199 -j ACCEPT # JMX
iptables -A INPUT -p tcp --dport 8888 -j ACCEPT # OpsCenter
iptables -A INPUT -p tcp --dport 9042 -j ACCEPT # CQL
iptables -A INPUT -p tcp --dport 9160 -j ACCEPT # Thift client API
iptables-save
}
echo "[$(timestamp)] ---- 1. Setup Host ----"
setuphost
echo "[$(timestamp)] ---- 2. Install Oracle Java 7 ----"
install_java
echo "[$(timestamp)] ---- 3. Install Cassandra ----"
install_cassandra
echo "[$(timestamp)] ---- 4. Configure Cassandra ----"
configure_cassandra
echo "[$(timestamp)] ---- 5. Export Variables ----"
export_variables
echo "[$(timestamp)] ---- 6. Import Data ----"
import_data
echo "[$(timestamp)] ---- 7. Update IPTables rules ----"
update_iptables
echo "[$(timestamp)] ---- Completed ----"
exit 0