Fix Xcode 16 build issues (secretflow#861)

* Moved build env to Xcode 16.0.0 * Moved minimum macOS to 13.0
anakinxc · Sep 24, 2024 · d7cb0cf · d7cb0cf
1 parent 2e69196
commit d7cb0cf
Show file tree

Hide file tree

Showing 21 changed files with 69 additions and 46 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -43,7 +43,7 @@ build --copt=-fstack-protector-strong
 build:linux --copt=-Wl,-z,noexecstack
 build:macos --copt=-Wa,--noexecstack
 
-test --keep_going
+build --keep_going
 test --test_output=errors
 
 build:benchmark --copt -O3
@@ -55,11 +55,12 @@ build:linux --action_env=BAZEL_LINKLIBS=-l%:libstdc++.a:-l%:libgcc.a
 
 # platform specific config
 # Bazel will automatic pick platform config since we have enable_platform_specific_config set
-build:macos --copt="-Xpreprocessor -fopenmp"
+build:macos --copt=-Xclang=-fopenmp
 build:macos --copt=-Wno-unused-command-line-argument
 build:macos --features=-supports_dynamic_linker
-build:macos --macos_minimum_os=12.0
-build:macos --host_macos_minimum_os=12.0
+build:macos --macos_minimum_os=13.0
+build:macos --host_macos_minimum_os=13.0
+build:macos --action_env MACOSX_DEPLOYMENT_TARGET=13.0
 
 build:linux --copt=-fopenmp
 build:linux --linkopt=-fopenmp
diff --git a/.circleci/continue-config.yml b/.circleci/continue-config.yml
@@ -64,7 +64,7 @@ commands:
             ../devtools/bazel_cache_setup.py --in_file=../gcs.data --out_file=../gcs.json --min_download
       - run:
           name: "build"
-          command: bazel build <<parameters.targets>> -c opt --ui_event_filters=-info,-debug,-warning --jobs 20
+          command: bazel build <<parameters.targets>> -c opt --ui_event_filters=-info,-debug,-warning
       - run:
           name: "test"
           command: |
@@ -120,7 +120,7 @@ jobs:
           extra_bazel_test_args: --test_env LD_LIBRARY_PATH=/root/miniconda3/lib/
   macOS_ut:
     macos:
-      xcode: 15.4.0
+      xcode: 16.0.0
     resource_class: macos.m1.large.gen1
     steps:
       - checkout

diff --git a/.circleci/release-config.yml b/.circleci/release-config.yml
@@ -63,7 +63,7 @@ commands:
 jobs:
   macOS_publish:
     macos:
-      xcode: 15.4.0
+      xcode: 16.0.0
     resource_class: macos.m1.large.gen1
     parameters:
       python_ver:

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -72,7 +72,7 @@ python3 -m pip install -r requirements-dev.txt
 #### macOS
 
 ```sh
-# macOS >= 12.0, Xcode >= 14.0
+# macOS >= 13.0, Xcode >= 15.0
 
 # Install Xcode
 https://apps.apple.com/us/app/xcode/id497799835?mt=12

diff --git a/WORKSPACE b/WORKSPACE
@@ -53,6 +53,10 @@ rules_foreign_cc_dependencies(
     register_preinstalled_tools = True,
 )
 
+load("@bazel_features//:deps.bzl", "bazel_features_deps")
+
+bazel_features_deps()
+
 load("@rules_cuda//cuda:repositories.bzl", "register_detected_cuda_toolchains", "rules_cuda_dependencies")
 
 rules_cuda_dependencies()

diff --git a/bazel/eigen.BUILD b/bazel/eigen.BUILD
@@ -20,6 +20,7 @@
 #   matrices, and related algorithms.
 
 load("@rules_cc//cc:defs.bzl", "cc_library")
+load("@yacl//bazel:yacl.bzl", "OMP_DEPS")
 
 licenses([
     # Note: Eigen is an MPL2 library that includes GPL v3 and LGPL v2.1+ code.
@@ -64,6 +65,7 @@ cc_library(
     ],
     includes = ["."],
     visibility = ["//visibility:public"],
+    deps = OMP_DEPS,
 )
 
 filegroup(

diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl
@@ -39,21 +39,21 @@ def _yacl():
         http_archive,
         name = "yacl",
         urls = [
-            "https://github.com/secretflow/yacl/archive/refs/tags/0.4.5b4_nightly_20240731.tar.gz",
+            "https://github.com/secretflow/yacl/archive/refs/tags/0.4.5b5_nightly_20240919.tar.gz",
         ],
-        strip_prefix = "yacl-0.4.5b4_nightly_20240731",
-        sha256 = "952715bd56f6d9386984e9963426a1399bd2bd3702cf3efede9c82591cfab99b",
+        strip_prefix = "yacl-0.4.5b5_nightly_20240919",
+        sha256 = "0ef295f6878dce6160fd44e6af59fa369099f736fa8d4a10f9685dda66aefa71",
     )
 
 def _libpsi():
     maybe(
         http_archive,
         name = "psi",
         urls = [
-            "https://github.com/secretflow/psi/archive/refs/tags/v0.4.0.dev240814.tar.gz",
+            "https://github.com/secretflow/psi/archive/refs/tags/v0.4.3.dev240919.tar.gz",
         ],
-        strip_prefix = "psi-0.4.0.dev240814",
-        sha256 = "2a16a5751d1b7051f01edd11f1fcf01b67ff4d67ec136e7bc6d1d729d7f22634",
+        strip_prefix = "psi-0.4.3.dev240919",
+        sha256 = "1ee34fbbd9a8f36dea8f7c45588a858e8c31f3a38e60e1fc67cb428ea79334e3",
     )
 
 def _rules_proto_grpc():
@@ -229,7 +229,7 @@ def _com_github_eigenteam_eigen():
     EIGEN_SHA256 = "01fcd68409c038bbcfd16394274c2bf71e2bb6dda89a2319e23fc59a2da17210"
     maybe(
         http_archive,
-        name = "com_github_eigenteam_eigen",
+        name = "eigen_archive",
         sha256 = EIGEN_SHA256,
         build_file = "@spulib//bazel:eigen.BUILD",
         strip_prefix = "eigen-{commit}".format(commit = EIGEN_COMMIT),
@@ -241,7 +241,7 @@ def _com_github_eigenteam_eigen():
 def _com_github_nvidia_cutlass():
     maybe(
         http_archive,
-        name = "com_github_nvidia_cutlass",
+        name = "cutlass_archive",
         strip_prefix = "cutlass-3.5.1",
         urls = [
             "https://github.com/NVIDIA/cutlass/archive/refs/tags/v3.5.1.tar.gz",

diff --git a/experimental/squirrel/BUILD.bazel b/experimental/squirrel/BUILD.bazel
@@ -60,8 +60,8 @@ spu_cc_library(
         "//libspu/mpc/cheetah/rlwe:cheetah_rlwe",
         "//libspu/mpc/cheetah/rlwe:lwe",
         "//libspu/mpc/cheetah/rlwe:packlwes",
-        "@com_github_eigenteam_eigen//:eigen3",
         "@com_github_microsoft_seal//:seal",
+        "@eigen_archive//:eigen3",
         "@yacl//yacl/utils:elapsed_timer",
     ],
 )

diff --git a/libspu/compiler/front_end/BUILD.bazel b/libspu/compiler/front_end/BUILD.bazel
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+load("@yacl//bazel:yacl.bzl", "OMP_DEPS")
 load("//bazel:spu.bzl", "spu_cc_library")
 
 spu_cc_library(
@@ -60,7 +61,7 @@ spu_cc_library(
         "@xla//xla/service:zero_sized_hlo_elimination",
         "@xla//xla/service/gpu/transforms:dot_dimension_sorter",
         "@xla//xla/translate/hlo_to_mhlo:hlo_module_importer",
-    ],
+    ] + OMP_DEPS,
 )
 
 spu_cc_library(

diff --git a/libspu/compiler/front_end/fe.cc b/libspu/compiler/front_end/fe.cc
@@ -14,6 +14,7 @@
 
 #include "libspu/compiler/front_end/fe.h"
 
+#include "fmt/ranges.h"
 #include "mlir/Dialect/Func/Extensions/InlinerExtension.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/IR/BuiltinOps.h"

diff --git a/libspu/compiler/tools/spu-translate.cc b/libspu/compiler/tools/spu-translate.cc
@@ -41,9 +41,6 @@
 #include "libspu/compiler/core/core.h"
 #undef EXPOSE_PIPELINE_BUILDER
 
-template <typename T>
-struct fmt::formatter<xt::xarray<T>> : ostream_formatter {};
-
 llvm::cl::opt<uint32_t> ProtocolKind(
     "protocol_kind", llvm::cl::init(1),
     llvm::cl::desc("1 for REF2k, 2 for SEMI2k, 3 for ABY3, 4 for Cheetah"));
@@ -109,7 +106,7 @@ bool testOpHandler(::spu::SPUContext *sctx, mlir::Operation *op,
 
     auto error = xt::fabs(lhs - rhs);
 
-    for (auto v : error) {
+    for (double v : error) {
       if (v > tol) {
         llvm::report_fatal_error(
             fmt::format("Diff {} greater than tol {}", v, tol).c_str());

diff --git a/libspu/core/ndarray_ref.h b/libspu/core/ndarray_ref.h
@@ -409,7 +409,6 @@ struct SimdTrait<NdArrayRef> {
 NdArrayRef makeConstantArrayRef(const Type& eltype, const Shape& shape);
 
 std::ostream& operator<<(std::ostream& out, const NdArrayRef& v);
-inline auto format_as(const spu::NdArrayRef& f) { return fmt::streamed(f); }
 
 template <typename T>
 class NdArrayView {
@@ -491,3 +490,10 @@ struct std::hash<spu::NdArrayRef> {
     return std::hash<const void*>{}(r.data());
   }
 };
+
+namespace fmt {
+
+template <>
+struct formatter<spu::NdArrayRef> : ostream_formatter {};
+
+}  // namespace fmt
diff --git a/libspu/core/shape.h b/libspu/core/shape.h
@@ -22,6 +22,7 @@
 #include <vector>
 
 #include "absl/types/span.h"
+#include "fmt/ranges.h"
 #include "llvm/ADT/ArrayRef.h"
 
 #include "libspu/core/prelude.h"
@@ -73,8 +74,6 @@ class Shape : public std::vector<int64_t> {
   bool empty() const { return Base::empty(); }
 };
 
-inline auto format_as(const Shape &s) { return fmt::streamed(s); }
-
 class Index : public std::vector<int64_t> {
  private:
   using Base = std::vector<int64_t>;
@@ -95,8 +94,6 @@ class Index : public std::vector<int64_t> {
   }
 };
 
-inline auto format_as(const Index &idx) { return fmt::streamed(idx); }
-
 using Stride = int64_t;
 
 class Strides : public std::vector<Stride> {
@@ -117,8 +114,6 @@ class Strides : public std::vector<Stride> {
   }
 };
 
-inline auto format_as(const Strides &s) { return fmt::streamed(s); }
-
 class Sizes : public std::vector<int64_t> {
  private:
   using Base = std::vector<int64_t>;
@@ -135,8 +130,6 @@ class Sizes : public std::vector<int64_t> {
   }
 };
 
-inline auto format_as(const Sizes &s) { return fmt::streamed(s); }
-
 class Axes : public std::vector<int64_t> {
  private:
   using Base = std::vector<int64_t>;
@@ -153,8 +146,6 @@ class Axes : public std::vector<int64_t> {
   }
 };
 
-inline auto format_as(const Axes &axes) { return fmt::streamed(axes); }
-
 Strides makeCompactStrides(const Shape &shape);
 
 int64_t flattenIndex(const Index &index, const Shape &shape);
@@ -191,3 +182,22 @@ inline size_t calcFlattenOffset(const Index &indices, const Shape &shape,
 }
 
 }  // namespace spu
+
+namespace fmt {
+
+template <>
+struct formatter<spu::Shape> : ostream_formatter {};
+
+template <>
+struct formatter<spu::Axes> : ostream_formatter {};
+
+template <>
+struct formatter<spu::Index> : ostream_formatter {};
+
+template <>
+struct formatter<spu::Strides> : ostream_formatter {};
+
+template <>
+struct formatter<spu::Sizes> : ostream_formatter {};
+
+}  // namespace fmt
diff --git a/libspu/core/trace.h b/libspu/core/trace.h
@@ -23,6 +23,7 @@
 
 #include "absl/types/span.h"
 #include "fmt/format.h"
+#include "fmt/ranges.h"
 #include "spdlog/spdlog.h"
 #include "yacl/link/context.h"
 

diff --git a/libspu/dialect/pphlo/IR/ops.cc b/libspu/dialect/pphlo/IR/ops.cc
@@ -15,6 +15,7 @@
 #include "libspu/dialect/pphlo/IR/ops.h"
 
 #include "fmt/format.h"
+#include "fmt/ranges.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/TypeUtilities.h"

diff --git a/libspu/mpc/semi2k/beaver/beaver_impl/BUILD.bazel b/libspu/mpc/semi2k/beaver/beaver_impl/BUILD.bazel
@@ -56,7 +56,7 @@ spu_cc_library(
         "//libspu/mpc/semi2k/beaver/beaver_impl/ttp_server:beaver_stream",
         "//libspu/mpc/semi2k/beaver/beaver_impl/ttp_server:service_cc_proto",
         "//libspu/mpc/utils:ring_ops",
-        "@yacl//yacl/crypto/pke:asymmetric_sm2_crypto",
+        "@yacl//yacl/crypto/pke:sm2_enc",
         "@yacl//yacl/link",
         "@yacl//yacl/link/algorithm:barrier",
         "@yacl//yacl/utils:parallel",

diff --git a/libspu/mpc/semi2k/beaver/beaver_impl/beaver_ttp.cc b/libspu/mpc/semi2k/beaver/beaver_impl/beaver_ttp.cc
@@ -18,7 +18,7 @@
 #include <utility>
 #include <vector>
 
-#include "yacl/crypto/pke/asymmetric_sm2_crypto.h"
+#include "yacl/crypto/pke/sm2_enc.h"
 #include "yacl/crypto/rand/rand.h"
 #include "yacl/link/algorithm/allgather.h"
 
@@ -303,7 +303,7 @@ BeaverTtp::BeaverTtp(std::shared_ptr<yacl::link::Context> lctx, Options ops)
 
   yacl::Buffer encrypted_seed;
   {
-    std::unique_ptr<yacl::crypto::AsymmetricEncryptor> encryptor;
+    std::unique_ptr<yacl::crypto::PkeEncryptor> encryptor;
     auto lower_schema = absl::AsciiStrToLower(options_.asym_crypto_schema);
     if (lower_schema == "sm2") {
       encryptor = std::make_unique<yacl::crypto::Sm2Encryptor>(

diff --git a/libspu/mpc/semi2k/beaver/beaver_impl/ttp_server/BUILD.bazel b/libspu/mpc/semi2k/beaver/beaver_impl/ttp_server/BUILD.bazel
@@ -42,7 +42,7 @@ spu_cc_library(
         ":service_cc_proto",
         "//libspu/mpc/semi2k/beaver/beaver_impl/trusted_party",
         "@com_github_brpc_brpc//:brpc",
-        "@yacl//yacl/crypto/pke:asymmetric_sm2_crypto",
+        "@yacl//yacl/crypto/pke:sm2_enc",
     ],
 )
 

diff --git a/libspu/mpc/semi2k/beaver/beaver_impl/ttp_server/beaver_server.cc b/libspu/mpc/semi2k/beaver/beaver_impl/ttp_server/beaver_server.cc
@@ -22,7 +22,7 @@
 #include "spdlog/spdlog.h"
 #include "yacl/base/byte_container_view.h"
 #include "yacl/base/exception.h"
-#include "yacl/crypto/pke/asymmetric_sm2_crypto.h"
+#include "yacl/crypto/pke/sm2_enc.h"
 
 #include "libspu/core/ndarray_ref.h"
 #include "libspu/mpc/common/prg_tensor.h"
@@ -51,9 +51,8 @@ class DecryptError : public yacl::Exception {
 template <class AdjustRequest>
 std::tuple<std::vector<TrustedParty::Operand>,
            std::vector<std::vector<PrgSeed>>, size_t>
-BuildOperand(
-    const AdjustRequest& req, uint32_t field_size,
-    const std::unique_ptr<yacl::crypto::AsymmetricDecryptor>& decryptor) {
+BuildOperand(const AdjustRequest& req, uint32_t field_size,
+             const std::unique_ptr<yacl::crypto::PkeDecryptor>& decryptor) {
   std::vector<TrustedParty::Operand> ops;
   std::vector<std::vector<PrgSeed>> seeds;
   size_t pad_length = 0;
@@ -261,7 +260,7 @@ class StreamReader : public brpc::StreamInputHandler {
 template <class AdjustRequest>
 std::vector<yacl::Buffer> AdjustImpl(
     const AdjustRequest& req, StreamReader& stream_reader,
-    const std::unique_ptr<yacl::crypto::AsymmetricDecryptor>& decryptor) {
+    const std::unique_ptr<yacl::crypto::PkeDecryptor>& decryptor) {
   std::vector<NdArrayRef> ret;
   size_t field_size;
   if constexpr (std::is_same_v<AdjustRequest, AdjustAndRequest>) {
@@ -319,7 +318,7 @@ std::vector<yacl::Buffer> AdjustImpl(
 
 class ServiceImpl final : public BeaverService {
  private:
-  std::unique_ptr<yacl::crypto::AsymmetricDecryptor> decryptor_;
+  std::unique_ptr<yacl::crypto::PkeDecryptor> decryptor_;
 
  public:
   ServiceImpl(const std::string& asym_crypto_schema,

diff --git a/libspu/mpc/utils/BUILD.bazel b/libspu/mpc/utils/BUILD.bazel
@@ -106,7 +106,7 @@ spu_cc_library(
     linkopts = OMP_LINKFLAGS,
     deps = [
         "//libspu/core:parallel_utils",
-        "@com_github_eigenteam_eigen//:eigen3",
+        "@eigen_archive//:eigen3",
     ] + OMP_DEPS,
 )
 

diff --git a/setup.py b/setup.py
@@ -256,9 +256,9 @@ def has_ext_modules(self):
 if sys.platform == "darwin":
     # Due to a bug in conda x64 python, platform tag has to be 10_16 for X64 wheel
     if platform.machine() == "x86_64":
-        plat_name = "macosx_12_0_x86_64"
+        plat_name = "macosx_13_0_x86_64"
     else:
-        plat_name = "macosx_12_0_arm64"
+        plat_name = "macosx_13_0_arm64"
 elif platform.machine() == "aarch64":
     # Linux aarch64
     plat_name = "manylinux_2_28_aarch64"