feat(deploy): add cp tensorflow_io.py

bytedance · Jan 30, 2024 · 7e61d49 · 7e61d49
1 parent 2cfdfb9
commit 7e61d49
Show file tree

Hide file tree

Showing 8 changed files with 30 additions and 9 deletions.
diff --git a/deploy/scripts/sgx/enclave_env.sh b/deploy/scripts/sgx/enclave_env.sh
@@ -25,8 +25,8 @@ function make_custom_env() {
     export GRPC_VERBOSITY=ERROR
     export GRPC_POLL_STRATEGY=epoll1
     export TF_CPP_MIN_LOG_LEVEL=1
-    export TF_GRPC_SGX_RA_TLS_ENABLE=on
-    export FL_GRPC_SGX_RA_TLS_ENABLE=on
+    export TF_GRPC_SGX_RA_TLS_ENABLE=off
+    export FL_GRPC_SGX_RA_TLS_ENABLE=off
     export TF_DISABLE_MKL=0
     export TF_ENABLE_MKL_NATIVE_FORMAT=1
     export parallel_num_threads=$1
@@ -55,4 +55,12 @@ function generate_token() {
     cd -
 }
 
+
+# 为站内不同临时设置，后续改为环境变量区分，参考文档‘隐私计算支持sgx’
+sed -i 's|PCCS_URL=https://[^ ]*|PCCS_URL=https://pccs_url:8081/sgx/certification/v3/|' /etc/sgx_default_qcnl.conf
+sed -i 's/USE_SECURE_CERT=TRUE/USE_SECURE_CERT=FALSE/' /etc/sgx_default_qcnl.conf
+echo >> /etc/hosts
+echo "10.137.29.200   pccs_url" | tee -a /etc/hosts
+mkdir -p /data
+
 generate_token
diff --git a/deploy/scripts/sgx/run_trainer_master_sgx.sh b/deploy/scripts/sgx/run_trainer_master_sgx.sh
@@ -17,6 +17,7 @@
 set -ex
 source ~/.env
 export CUDA_VISIBLE_DEVICES=
+cp /app/sgx/gramine/CI-Examples/tensorflow_io.py ./
 source /app/deploy/scripts/hdfs_common.sh || true
 source /app/deploy/scripts/pre_start_hook.sh || true
 source /app/deploy/scripts/env_to_args.sh
@@ -87,8 +88,8 @@ else
   pull_code ${CODE_TAR} $PWD
 fi
 cd ${ROLE}
-source /app/deploy/scripts/sgx/enclave_env.sh
 cp /app/sgx/gramine/CI-Examples/tensorflow_io.py ./
+source /app/deploy/scripts/sgx/enclave_env.sh
 cp /app/sgx/token/* ./
 unset HTTPS_PROXY https_proxy http_proxy ftp_proxy
 

diff --git a/deploy/scripts/sgx/run_trainer_ps_sgx.sh b/deploy/scripts/sgx/run_trainer_ps_sgx.sh
@@ -17,6 +17,7 @@
 set -ex
 source ~/.env
 export CUDA_VISIBLE_DEVICES=
+cp /app/sgx/gramine/CI-Examples/tensorflow_io.py ./
 source /app/deploy/scripts/hdfs_common.sh || true
 source /app/deploy/scripts/pre_start_hook.sh || true
 

diff --git a/deploy/scripts/sgx/run_trainer_worker_sgx.sh b/deploy/scripts/sgx/run_trainer_worker_sgx.sh
@@ -19,6 +19,7 @@ source ~/.env
 export CUDA_VISIBLE_DEVICES=
 export MODEL_NAME=${APPLICATION_ID}
 
+cp /app/sgx/gramine/CI-Examples/tensorflow_io.py ./
 source /app/deploy/scripts/hdfs_common.sh || true
 source /app/deploy/scripts/pre_start_hook.sh || true
 source /app/deploy/scripts/env_to_args.sh
@@ -32,8 +33,8 @@ else
 fi
 
 cd ${ROLE}
-source /app/deploy/scripts/sgx/enclave_env.sh
 cp /app/sgx/gramine/CI-Examples/tensorflow_io.py ./
+source /app/deploy/scripts/sgx/enclave_env.sh
 cp /app/sgx/token/* ./
 unset HTTPS_PROXY https_proxy http_proxy ftp_proxy
 

diff --git a/deploy/scripts/trainer/run_trainer_master.sh b/deploy/scripts/trainer/run_trainer_master.sh
@@ -17,6 +17,8 @@
 set -ex
 
 export CUDA_VISIBLE_DEVICES=
+
+cp /app/sgx/gramine/CI-Examples/tensorflow_io.py ./
 source /app/deploy/scripts/hdfs_common.sh || true
 source /app/deploy/scripts/pre_start_hook.sh || true
 source /app/deploy/scripts/env_to_args.sh
@@ -88,6 +90,7 @@ else
   pull_code ${CODE_TAR} $PWD
 fi
 cd ${ROLE}
+cp /app/sgx/gramine/CI-Examples/tensorflow_io.py ./
 
 LISTEN_PORT=50051
 if [[ -n "${PORT0}" ]]; then

diff --git a/deploy/scripts/trainer/run_trainer_ps.sh b/deploy/scripts/trainer/run_trainer_ps.sh
@@ -17,6 +17,8 @@
 set -ex
 
 export CUDA_VISIBLE_DEVICES=
+
+cp /app/sgx/gramine/CI-Examples/tensorflow_io.py ./
 source /app/deploy/scripts/hdfs_common.sh || true
 source /app/deploy/scripts/pre_start_hook.sh || true
 

diff --git a/deploy/scripts/trainer/run_trainer_worker.sh b/deploy/scripts/trainer/run_trainer_worker.sh
@@ -19,6 +19,7 @@ set -ex
 export CUDA_VISIBLE_DEVICES=
 export MODEL_NAME=${APPLICATION_ID}
 
+cp /app/sgx/gramine/CI-Examples/tensorflow_io.py ./
 source /app/deploy/scripts/hdfs_common.sh || true
 source /app/deploy/scripts/pre_start_hook.sh || true
 source /app/deploy/scripts/env_to_args.sh
@@ -32,6 +33,7 @@ else
 fi
 
 cd ${ROLE}
+cp /app/sgx/gramine/CI-Examples/tensorflow_io.py ./
 
 mode=$(normalize_env_to_args "--mode" "$MODE")
 sparse_estimator=$(normalize_env_to_args "--sparse-estimator" "$SPARSE_ESTIMATOR")

diff --git a/sgx/gramine/CI-Examples/generate-token/python.manifest.template b/sgx/gramine/CI-Examples/generate-token/python.manifest.template