repo-sync-2024-08-15T09:20:09+0800

anakinxc · Aug 15, 2024 · 0d6b9ec · 0d6b9ec
1 parent aedbe44
commit 0d6b9ec
Show file tree

Hide file tree

Showing 1,301 changed files with 72,312 additions and 89,908 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -13,7 +13,6 @@
 # limitations under the License.
 
 common --experimental_repo_remote_exec
-common --experimental_cc_shared_library
 
 # Required by OpenXLA
 build --nocheck_visibility
@@ -30,36 +29,54 @@ build:avx --host_copt=-mavx
 build:avx --copt=-DCHECK_AVX
 build:avx --host_copt=-DCHECK_AVX
 
-# default off CUDA build
-build --@rules_cuda//cuda:enable=false
-
-# Only on when asked
-build:gpu --@rules_cuda//cuda:archs=compute_80:compute_80
-build:gpu --@rules_cuda//cuda:enable=true
 
 # Binary safety flags
 build --copt=-fPIC
 build --copt=-fstack-protector-strong
 build:linux --copt=-Wl,-z,noexecstack
 build:macos --copt=-Wa,--noexecstack
 
+
 test --keep_going
 test --test_output=errors
 
 build:benchmark --copt -O3
 build:benchmark --copt -march=native
 
 # static link runtime libraries on Linux
-build:linux --action_env=BAZEL_LINKOPTS=-static-libstdc++:-static-libgcc
-build:linux --action_env=BAZEL_LINKLIBS=-l%:libstdc++.a:-l%:libgcc.a
+build:linux-release --action_env=BAZEL_LINKOPTS=-static-libstdc++:-static-libgcc
+build:linux-release --action_env=BAZEL_LINKLIBS=-l%:libstdc++.a:-l%:libgcc.a
 
 # platform specific config
 # Bazel will automatic pick platform config since we have enable_platform_specific_config set
 build:macos --copt="-Xpreprocessor -fopenmp"
 build:macos --copt=-Wno-unused-command-line-argument
 build:macos --features=-supports_dynamic_linker
-build:macos --macos_minimum_os=12.0
-build:macos --host_macos_minimum_os=12.0
+build:macos --cxxopt -Wno-deprecated-enum-enum-conversion
+build:macos --cxxopt -Wno-deprecated-anon-enum-enum-conversion
+build:macos --macos_minimum_os=11.0
+build:macos --host_macos_minimum_os=11.0
 
 build:linux --copt=-fopenmp
 build:linux --linkopt=-fopenmp
+
+build:asan --strip=never
+build:asan --copt -fno-sanitize-recover=all
+build:asan --copt -fsanitize=address
+build:asan --copt -Og
+build:asan --copt -g
+build:asan --copt -fno-omit-frame-pointer
+build:asan --linkopt -fsanitize=address
+build:asan --linkopt -static-libasan
+
+build:ubsan --strip=never
+build:ubsan --copt -fno-sanitize-recover=all
+build:ubsan --copt -fsanitize=undefined
+build:ubsan --copt -Og
+build:ubsan --copt -g
+build:ubsan --copt -fno-omit-frame-pointer
+build:ubsan --linkopt -fsanitize=undefined
+build:ubsan --linkopt -static-libubsan
+
+build:macos-asan --features=asan
+build:macos-ubsan --features=ubsan
diff --git a/.bazelversion b/.bazelversion
@@ -1 +1 @@
-6.5.0
+6.2.1
diff --git a/.circleci/asan-config.yml b/.circleci/asan-config.yml
diff --git a/.circleci/benchmark-config.yml b/.circleci/benchmark-config.yml
@@ -27,11 +27,30 @@ jobs:
   nn_benchmark:
     # Use machine executor for convenient data sharing between the host and docker-compose cluster
     machine:
-      image: ubuntu-2204:current
+      image: ubuntu-2004:2022.07.1
     resource_class: 2xlarge+
     shell: /bin/bash --login -eo pipefail
     steps:
       - checkout
+      - restore_cache:
+          name: "Restore build cache"
+          key: spu-build-linuxVM-
+      - run:
+          name: Build SPU
+          command: |
+            set +e
+            declare -i build_status
+            sed -i "s/tensorflow /tensorflow-cpu /g" ./requirements-dev.txt
+            docker run --name spu-build --mount type=bind,source="$(pwd)",target=/home/admin/dev/ secretflow/spu-ci:0.7 \
+                sh -c "cd /home/admin/dev && \
+                        python3 -m pip install -U pip && \
+                        python3 -m pip install -r requirements.txt && \
+                        python3 -m pip install -r requirements-dev.txt && \
+                        bazel build //examples/python/... -c opt --ui_event_filters=-info,-debug,-warning --disk_cache=/home/admin/dev/.cache/spu_build_cache" | tee build_results.log; build_status=${PIPESTATUS[0]}
+            exit ${build_status}
+      - run:
+          name: Save image
+          command: docker commit spu-build spu-build:v1
       - run:
           name: Launch docker-compose cluster
           command: docker-compose -f .circleci/benchmark.yml up -d
@@ -40,20 +59,18 @@ jobs:
           no_output_timeout: 30m
           command: |
             sleep 60
-            # Hide local spu path for import conflicts
-            mv spu .spu
             # TODO: run all stax_nn models with different optimizers
-            docker run --rm --mount type=bind,source="$(pwd)",target=/home/admin/dev/ --network nn-benchmark secretflow/ubuntu-base-ci:latest \
-                "python3 -m pip install -U pip && \
-                   python3 -m pip install spu && \
-                   cd /home/admin/dev/ && \
-                   python3 -m pip install -r requirements-dev.txt && \
-                   python3 -m pip install -r examples/python/ml/requirements.txt && \
-                   export PYTHONPATH="${PWD}:$PYTHONPATH" && \
-                   bash .circleci/run-nn.sh" | tee benchmark_results.log
+            docker run --rm --mount type=bind,source="$(pwd)",target=/home/admin/dev/ --network nn-benchmark spu-build:v1 \
+                sh -c "cd /home/admin/dev && bash .circleci/run-nn.sh" | tee benchmark_results.log
       - run:
           name: Shutdown docker-compose cluster
           command: docker-compose -f .circleci/benchmark.yml down
+      - save_cache:
+          key: spu-build-linuxVM-{{ .Environment.CIRCLE_BUILD_NUM }}
+          paths:
+            - .cache/spu_build_cache
+      - store_artifacts:
+          path: build_results.log
       - store_artifacts:
           path: benchmark_results.log
 # Invoke jobs via workflows

diff --git a/.circleci/benchmark.json b/.circleci/benchmark.json
@@ -1,11 +1,11 @@
 {
     "id": "outsourcing.3pc",
     "nodes": {
-        "node:0": "172.16.238.2:61920",
-        "node:1": "172.16.238.3:61921",
-        "node:2": "172.16.238.4:61922",
-        "node:3": "172.16.238.5:61923",
-        "node:4": "172.16.238.6:61924"
+        "node:0": "172.16.238.2:9920",
+        "node:1": "172.16.238.3:9921",
+        "node:2": "172.16.238.4:9922",
+        "node:3": "172.16.238.5:9923",
+        "node:4": "172.16.238.6:9924"
     },
     "devices": {
         "SPU": {
@@ -17,15 +17,21 @@
                     "node:2"
                 ],
                 "spu_internal_addrs": [
-                    "172.16.238.2:61930",
-                    "172.16.238.3:61931",
-                    "172.16.238.4:61932"
+                    "172.16.238.2:9930",
+                    "172.16.238.3:9931",
+                    "172.16.238.4:9932"
+                ],
+                "experimental_data_folder": [
+                    "/tmp/spu_data_0/",
+                    "/tmp/spu_data_1/",
+                    "/tmp/spu_data_2/"
                 ],
                 "runtime_config": {
                     "protocol": "ABY3",
                     "field": "FM64",
                     "enable_pphlo_profile": true,
-                    "enable_hal_profile": true
+                    "enable_hal_profile": true,
+                    "enable_pphlo_trace": false
                 }
             }
         },
@@ -42,4 +48,4 @@
             }
         }
     }
-}
+}
diff --git a/.circleci/benchmark.yml b/.circleci/benchmark.yml
@@ -22,44 +22,64 @@ networks:
 services:
   node_0:
     command:
-      - python3 -m pip install -U pip && python3 -m pip install spu && cd /home/admin/dev/examples/python/utils && python3 nodectl.py -c /home/admin/dev/.circleci/benchmark.json start --node_id node:0
-    image: secretflow/ubuntu-base-ci:latest
+      - /bin/sh
+      - -c
+      - |
+        cd /home/admin/dev
+        bazel-bin/examples/python/utils/nodectl -c .circleci/benchmark.json start --node_id node:0
+    image: spu-build:v1
     networks:
       nn-benchmark:
         ipv4_address: 172.16.238.2
     volumes:
       - ${PWD}:/home/admin/dev/
   node_1:
     command:
-      - python3 -m pip install -U pip && python3 -m pip install spu && cd /home/admin/dev/examples/python/utils && python3 nodectl.py -c /home/admin/dev/.circleci/benchmark.json start --node_id node:1
-    image: secretflow/ubuntu-base-ci:latest
+      - /bin/sh
+      - -c
+      - |
+        cd /home/admin/dev
+        bazel-bin/examples/python/utils/nodectl -c .circleci/benchmark.json start --node_id node:1
+    image: spu-build:v1
     networks:
       nn-benchmark:
         ipv4_address: 172.16.238.3
     volumes:
       - ${PWD}:/home/admin/dev/
   node_2:
     command:
-      - python3 -m pip install -U pip && python3 -m pip install spu && cd /home/admin/dev/examples/python/utils && python3 nodectl.py -c /home/admin/dev/.circleci/benchmark.json start --node_id node:2
-    image: secretflow/ubuntu-base-ci:latest
+      - /bin/sh
+      - -c
+      - |
+        cd /home/admin/dev
+        bazel-bin/examples/python/utils/nodectl -c .circleci/benchmark.json start --node_id node:2
+    image: spu-build:v1
     networks:
       nn-benchmark:
         ipv4_address: 172.16.238.4
     volumes:
       - ${PWD}:/home/admin/dev/
   node_3:
     command:
-      - python3 -m pip install -U pip && python3 -m pip install spu && cd /home/admin/dev/examples/python/utils && python3 nodectl.py -c /home/admin/dev/.circleci/benchmark.json start --node_id node:3
-    image: secretflow/ubuntu-base-ci:latest
+      - /bin/sh
+      - -c
+      - |
+        cd /home/admin/dev
+        bazel-bin/examples/python/utils/nodectl -c .circleci/benchmark.json start --node_id node:3
+    image: spu-build:v1
     networks:
       nn-benchmark:
         ipv4_address: 172.16.238.5
     volumes:
       - ${PWD}:/home/admin/dev/
   node_4:
     command:
-      - python3 -m pip install -U pip && python3 -m pip install spu && cd /home/admin/dev/examples/python/utils && python3 nodectl.py -c /home/admin/dev/.circleci/benchmark.json start --node_id node:4
-    image: secretflow/ubuntu-base-ci:latest
+      - /bin/sh
+      - -c
+      - |
+        cd /home/admin/dev
+        bazel-bin/examples/python/utils/nodectl -c .circleci/benchmark.json start --node_id node:4
+    image: spu-build:v1
     networks:
       nn-benchmark:
         ipv4_address: 172.16.238.6

diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -17,8 +17,8 @@ version: 2.1
 setup: true
 
 orbs:
-  path-filtering: circleci/path-filtering@1.1.0
-  continuation: circleci/continuation@1.0.0
+  path-filtering: circleci/path-filtering@0.1.4
+  continuation: circleci/continuation@0.3.1
 
 parameters:
   GHA_Actor:
@@ -36,9 +36,6 @@ parameters:
   run-schedule:
     type: boolean
     default: false
-  run-asan:
-    type: boolean
-    default: false
 
 # Define a job to be invoked later in a workflow.
 # See: https://circleci.com/docs/2.0/configuration-reference/#jobs
@@ -62,7 +59,6 @@ workflows:
   unittest-workflow:
     when:
       and:
-        - not: << pipeline.parameters.run-asan >>
         - not: << pipeline.parameters.run-schedule >>
         - not: << pipeline.parameters.GHA_Action >>
         - not: << pipeline.parameters.GHA_Meta >>
@@ -74,7 +70,6 @@ workflows:
             spu/.* build-and-run true
             libspu/.* build-and-run true
             exmaples/.* build-and-run true
-            bazel/.* build-and-run true
             .bazelrc build-and-run true
             requirements.txt build-and-run true
             requirements-dev.txt build-and-run true
@@ -85,11 +80,6 @@ workflows:
     jobs:
       - continuation/continue:
           configuration_path: .circleci/benchmark-config.yml
-  asan-workflow:
-    when: << pipeline.parameters.run-asan >>
-    jobs:
-      - continuation/continue:
-          configuration_path: .circleci/asan-config.yml
   publish-workflow:
     when:
       equal: [ "release", << pipeline.parameters.GHA_Event >> ]
@@ -99,4 +89,10 @@ workflows:
           # This is mandatory to trigger a pipeline when pushing a tag
           filters:
             tags:
-              only: /.*/
+              only: /.*(?<!dev\d)$/
+      - continuation/continue:
+          configuration_path: .circleci/dev-release-config.yml
+          # This is mandatory to trigger a pipeline when pushing a tag
+          filters:
+            tags:
+              only: /.*(?<=dev\d)$/