Update cloud deployment process

* Provide correct path to `run-local.sh` in `install.sh`
* Save logs to `dactionboard-vm` log
* Ensure that project files are copied into GCS
* Specify config file in and change default machine type in `settings.ini`
* Make message of ignoring output more prominent in `postcreate.sh`

Change-Id: I6390e3c4fbef4183294b149f04286a497bfe334f

gcp/cloud-run-button/prebuild.sh

@@ -30,4 +30,6 @@ gcloud config set project $GOOGLE_CLOUD_PROJECT
 
 ./gcp/install.sh
 
-echo -e "${CYAN}Please ignore all output below${WHITE}"
+echo -e "${CYAN}!!!!! Please ignore all output below !!!!!${WHITE}"
+echo
+echo

gcp/install.sh

@@ -19,7 +19,7 @@ pip install --require-hashes -r requirements.txt --no-deps
 echo -e "${CYAN}Generating dActionBoard configuration...${WHITE}"
 RUNNING_IN_GCE=true
 export RUNNING_IN_GCE   # signaling to run-local.sh that we're running inside GCE (there'll be less questions)
-./run-local.sh --generate-config-only --validate-google-ads-config
+./app/run-local.sh --generate-config-only --validate-google-ads-config
 
 # deploy solution
 echo -e "${CYAN}Deploying Cloud components...${WHITE}"

gcp/settings.ini

@@ -1,6 +1,7 @@
 ; Only kebab-style setting names are supported (no underscores)
 [config]
   name = dactionboard
+  config-file = dactionboard.yaml
 
 [repository]
   name = docker
@@ -22,7 +23,7 @@
 
 [compute]
   name = $NAME-vm
-  machine-type = n1-standard-1
-  region = europe-north1
-  zone = europe-north1-b
+  machine-type = e2-standard-2
+  region = europe-west1
+  zone = europe-west1-b
   no-public-ip = true

gcp/setup.sh

@@ -152,19 +152,29 @@ deploy_cf() {
 
 deploy_files() {
   echo 'Deploying files to GCS'
-  if ! gsutil ls gs://$PROJECT_ID; then
+  if ! gsutil ls gs://$PROJECT_ID > /dev/null 2> /dev/null; then
+    echo "Creating GCS bucket gs://$PROJECT_ID"
     gsutil mb -b on gs://$PROJECT_ID
   fi
 
   GCS_BASE_PATH=gs://$PROJECT_ID/$NAME
-  gsutil -h "Content-Type:text/plain" cp ./../dactionboard.yaml $GCS_BASE_PATH/dactionboard.yaml
-  gsutil -h "Content-Type:text/plain" cp ./../google-ads.yaml $GCS_BASE_PATH/google-ads.yaml
-
-  gsutil -m rm -r $GCS_BASE_PATH/google_ads_queries
-  gsutil -m cp -R ./../google_ads_queries $GCS_BASE_PATH/google_ads_queries
-  gsutil -m cp -R ./../bq_queries $GCS_BASE_PATH/bq_queries
-  gsutil -m rm -r $GCS_BASE_PATH/scripts
-  gsutil -m cp -R ./../scripts $GCS_BASE_PATH/scripts
+
+  # NOTE: DO NOT add -m flag for gsutil! When executed under cloudshell_open (via Cloud Run Button) it won't copy files
+  echo "Removing existing files at $GCS_BASE_PATH"
+  gsutil rm -r $GCS_BASE_PATH/
+
+  # NOTE: if an error "module 'sys' has no attribute 'maxint'" occures, run this: `pip3 install -U crcmod`
+  echo "Copying application files to $GCS_BASE_PATH"
+  gsutil rsync -r -x ".*/__pycache__/.*|[.].*" ./../app $GCS_BASE_PATH
+  echo "Copying configs to $GCS_BASE_PATH"
+  gsutil -h "Content-Type:text/plain" cp ./../app/*.yaml $GCS_BASE_PATH/
+  if [[ -f ./../google-ads.yaml ]]; then
+    gsutil -h "Content-Type:text/plain" cp ./../google-ads.yaml $GCS_BASE_PATH/google-ads.yaml
+  elif [[ -f $HOME/google-ads.yaml ]]; then
+    gsutil -h "Content-Type:text/plain" cp $HOME/google-ads.yaml $GCS_BASE_PATH/google-ads.yaml
+  else
+    echo "Please upload google-ads.yaml"
+  fi
 }
 
 deploy_public_index() {
@@ -291,10 +301,10 @@ enable_private_google_access() {
 deploy_all() {
   enable_apis
   set_iam_permissions
+  deploy_files
   create_registry
   build_docker_image
   deploy_cf
-  deploy_files
   schedule_run
 }
 

gcp/workload-vm/main.sh

@@ -21,10 +21,15 @@ if [[ -n $gcs_source_uri ]]; then
   mv "$folder_name/*" .
 fi
 
-# run dActionBoard
-./run-local.sh --quiet --config dactionboard.yaml --google-ads-config google-ads.yaml
+# run the application
+./start.sh 2>&1 | tee $LOG_NAME.log
 exitcode=$?
 
+while read -r line
+do
+  gcloud logging write $LOG_NAME "$line"
+done < $LOG_NAME.log
+
 if [ $exitcode -ne 0 ]; then
   gcloud logging write $LOG_NAME "[$(hostname)] dActionBoard application has finished execution with an error ($exitcode)" --severity ERROR
   # TODO: send the error somewhere

gcp/workload-vm/start.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+APP_CONFIG_FILE=$(git config -f "./settings.ini" config.config-file)
+
+chmod +x ./run-local.sh
+./run-local.sh --quiet --config $APP_CONFIG_FILE --google-ads-config google-ads.yaml
+