-
Notifications
You must be signed in to change notification settings - Fork 1
/
sync_data.sh
executable file
·56 lines (47 loc) · 1.96 KB
/
sync_data.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
#!/bin/bash
# Command :
# - ./sync_stats.sh
# - ./sync_stats.sh 2022-08-10 (to execute the script with a different date)
# Note : DATABASE_URL is automatically created on scalingo machines.
# For testing :
# DATABASE_URL=${DATABASE_URL:-'postgresql://stats:stats@localhost:5432/stats'}
# echo $DATABASE_URL
echo "Starting job. Should display 'Done' when done, if there were no errors."
extract_date=$1 # date format ie. 2022-08-10
if [ -z "$extract_date" ]
then
extract_date=`date +'%Y-%m-%d'`
fi
echo $extract_date
echo "Fetch S3 subscriptions_aggregate"
time ./fetch_from_s3.sh subscriptions_aggregate $extract_date
#echo "Fetch S3 events_aggregate"
# time ./fetch_from_s3.sh events_aggregate $extract_date
echo "Fetch S3 events_roomv9_aggregate"
time ./fetch_from_s3.sh events_roomv9_aggregate $extract_date
echo "Fetch S3 user_daily_visits $extract_date"
time ./fetch_from_s3.sh user_daily_visits $extract_date
echo "Fetch S3 pushers $extract_date"
time ./fetch_from_s3.sh pushers $extract_date
echo "Fetch S3 account_data $extract_date"
time ./fetch_from_s3.sh account_data $extract_date
echo "Fetch S3 crisp_conversation_segments $extract_date"
time ./fetch_from_s3.sh crisp_conversation_segments $extract_date
## Set up DB
psql -d $DATABASE_URL -f scripts/tables.sql
#### Now insert into DB
echo "Insert Subscriptions"
time psql -d $DATABASE_URL -f scripts/insert_subscriptions_data.sql
# echo "Insert Events"
# time psql -d $DATABASE_URL -f scripts/insert_events_data.sql
echo "Insert Events roomv9"
time psql -d $DATABASE_URL -f scripts/insert_events_roomv9_data.sql
echo "Insert User Daily Visits"
time psql -d $DATABASE_URL -f scripts/insert_user_daily_visits_data.sql
echo "Insert Pushers"
time psql -d $DATABASE_URL -f scripts/insert_pushers_data.sql
echo "Insert Account Data"
time psql -d $DATABASE_URL -f scripts/insert_account_data_data.sql
echo "Insert crisp conversation segments"
time psql -d $DATABASE_URL -f scripts/insert_crisp_conversation_segments.sql
echo "Done !"