repo-sync-2024-03-20T10:26:53+0800

secretflow · Mar 20, 2024 · 4c4a683 · 4c4a683
1 parent c33e02d
commit 4c4a683
Show file tree

Hide file tree

Showing 285 changed files with 1,593 additions and 881 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -23,9 +23,6 @@ build --cxxopt=-std=c++17
 build --host_cxxopt=-std=c++17
 build --linkopt -lm
 
-# HACK
-build:linux --cxxopt  -Wno-error=mismatched-new-delete
-
 # Binary safety flags
 build --host_copt=-fPIE
 build --host_copt=-fstack-protector-strong

diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml
@@ -81,6 +81,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@05963f47d870e2cb19a537396c1f668a348c7d8f # v3.24.8
+        uses: github/codeql-action/upload-sarif@3ab4101902695724f9365a384f86c1074d94e18c # v3.24.7
         with:
           sarif_file: results.sarif
diff --git a/README.md b/README.md
@@ -15,11 +15,8 @@ Supported platforms:
 ## Repo Layout
 
 - [base](yacl/base/): some basic types and utils in yacl.
-- [crypto](yacl/crypto/): a crypto library desigend for secure computation and so on.
-  - [base](yacl/crypto/base): **basic/standarized crypto**, i.e. AES, hashing.
-  - [primitives](yacl/crypto/primitives/): **crypto primitives**, i.e. OT, DPF.
-  - [tools](yacl/crypto/tools/): **theoretical crypto tools**, i.e. Random Oracle (RO), PRG.
-  - [utils](yacl/crypto/utils/): easy-to-use **crypto utilities**.
+- [crypto](yacl/crypto/): **crypto algorithms** without [link](yacl/link/).
+- [kernels](yacl/kernels/): **crypto kernels** that includes [link](yacl/link/) with multi-thread support, i.e. OT, DPF.
 - [io](yacl/io/): a simple streaming-based io library.
 - [link](yacl/link/): a simple rpc-based MPI framework, providing the [SPMD](https://en.wikipedia.org/wiki/SPMD) parallel programming capability.
 

diff --git a/bazel/patches/brpc.patch b/bazel/patches/brpc.patch
@@ -157,5 +157,5 @@ index 0bdaa4d3..43597f63 100644
 +    "@bazel_tools//src/conditions:linux_aarch64": ["-O1"],
 +    "//conditions:default": [""],
  })
- 
+
  LINKOPTS = [
diff --git a/bazel/patches/msgpack.patch b/bazel/patches/msgpack.patch
@@ -0,0 +1,17 @@
+diff --git a/include/msgpack/v1/detail/cpp11_zone.hpp b/include/msgpack/v1/detail/cpp11_zone.hpp
+index 2586f275..e325ee3b 100644
+--- a/include/msgpack/v1/detail/cpp11_zone.hpp
++++ b/include/msgpack/v1/detail/cpp11_zone.hpp
+@@ -20,6 +20,12 @@
+ #include <memory>
+ #include <vector>
+
++#if (defined __arm__ || defined __aarch64__) &&                                \
++    (defined(linux) || defined(__linux) || defined(__linux__)) &&              \
++    (defined(__GNUC__) || defined(__GNUG__))
++// This is a bug of arm linux gcc, suppressing false positives.
++#pragma GCC diagnostic ignored "-Wmismatched-new-delete"
++#endif
+ namespace msgpack {
+
+ /// @cond
diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl
@@ -60,10 +60,10 @@ def _org_interconnection():
         http_archive,
         name = "org_interconnection",
         urls = [
-            "https://github.com/secretflow/interconnection/archive/70672ae363614fddbdece4db719e0957b64fcae3.tar.gz",
+            "https://github.com/secretflow/interconnection/archive/32ec6f51aeb662f84c4d06fd8a0c3675a1e801a6.tar.gz",
         ],
-        strip_prefix = "interconnection-70672ae363614fddbdece4db719e0957b64fcae3",
-        sha256 = "82ad4a5c8d40e2ebbdc35db86b594bebe1fa89b0e06a7f5f4cb9a9fba91003da",
+        strip_prefix = "interconnection-32ec6f51aeb662f84c4d06fd8a0c3675a1e801a6",
+        sha256 = "a6b65900c36f1f9bc9c5bfa4fbf1b66fb1045f09eca8beb3a09c4be4c80b85f3",
     )
 
     # Add homebrew openmp for macOS, somehow..homebrew installs to different location on Apple Silcon/Intel macs.. so we need two rules here
@@ -339,6 +339,10 @@ def _com_github_msgpack_msgpack():
         type = "tar.gz",
         strip_prefix = "msgpack-c-cpp-6.1.0",
         sha256 = "5e63e4d9b12ab528fccf197f7e6908031039b1fc89cd8da0e97fbcbf5a6c6d3a",
+        patches = [
+            "@yacl//bazel:patches/msgpack.patch",
+        ],
+        patch_args = ["-p1"],
         urls = [
             "https://github.com/msgpack/msgpack-c/archive/refs/tags/cpp-6.1.0.tar.gz",
         ],

diff --git a/yacl/crypto/primitives/psu/BUILD.bazel → examples/psu/BUILD.bazel b/yacl/crypto/primitives/psu/BUILD.bazel → examples/psu/BUILD.bazel
@@ -28,11 +28,11 @@ yacl_cc_library(
     deps = [
         "//yacl/base:exception",
         "//yacl/base:int128",
-        "//yacl/crypto/base/hash:hash_utils",
-        "//yacl/crypto/primitives/ot:base_ot",
-        "//yacl/crypto/primitives/ot:iknp_ote",
-        "//yacl/crypto/primitives/ot:kkrt_ote",
-        "//yacl/crypto/utils:rand",
+        "//yacl/crypto/hash:hash_utils",
+        "//yacl/crypto/rand",
+        "//yacl/kernels/algorithms:base_ot",
+        "//yacl/kernels/algorithms:iknp_ote",
+        "//yacl/kernels/algorithms:kkrt_ote",
         "//yacl/link",
         "//yacl/math/f2k",
         "@com_google_absl//absl/types:span",

diff --git a/yacl/crypto/primitives/psu/krtw19_psu.cc → examples/psu/krtw19_psu.cc b/yacl/crypto/primitives/psu/krtw19_psu.cc → examples/psu/krtw19_psu.cc
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "yacl/crypto/primitives/psu/krtw19_psu.h"
+#include "examples/psu/krtw19_psu.h"
 
 #include <algorithm>
 #include <array>
@@ -22,7 +22,7 @@
 
 #include "yacl/utils/serialize.h"
 
-namespace yacl::crypto {
+namespace examples::psu {
 
 namespace {
 
@@ -32,12 +32,11 @@ constexpr size_t kBinSize{64ul};  // m+1
 constexpr uint128_t kBot{};
 constexpr size_t kNumBinsPerBatch{16ul};
 constexpr size_t kBatchSize{kNumBinsPerBatch * kBinSize};
-
 constexpr size_t kNumBaseOT{128ul};
 constexpr size_t kNumInkpOT{512ul};
 
 static auto HashToSizeT = [](const uint128_t& x) {
-  auto hash = Blake3_128({&x, sizeof x});
+  auto hash = yacl::crypto::Blake3_128({&x, sizeof x});
   size_t ret;
   std::memcpy(&ret, &hash, sizeof ret);
   return ret;
@@ -56,7 +55,7 @@ auto HashInputs(const std::vector<uint128_t>& elem_hashes, size_t count) {
 uint64_t Evaluate(const std::vector<uint64_t>& coeffs, uint64_t x) {
   uint64_t y{coeffs.back()};
   for (auto it = std::next(coeffs.rbegin()); it != coeffs.rend(); ++it) {
-    y = GfMul64(y, x) ^ *it;
+    y = yacl::GfMul64(y, x) ^ *it;
   }
   return y;
 }
@@ -72,15 +71,16 @@ auto Interpolate(const std::vector<uint64_t>& xs,
     uint64_t prod{1};
     for (size_t j{}; j != size; ++j) {
       if (xs[i] != xs[j]) {
-        prod = GfMul64(prod, xs[i] ^ xs[j]);
+        prod = yacl::GfMul64(prod, xs[i] ^ xs[j]);
         uint64_t sum{};
         for (size_t k{}; k != size; ++k) {
-          sum = std::exchange(Li_coeffs[k], GfMul64(Li_coeffs[k], xs[j]) ^ sum);
+          sum = std::exchange(Li_coeffs[k],
+                              yacl::GfMul64(Li_coeffs[k], xs[j]) ^ sum);
         }
       }
     }
     for (size_t k{}; k != size; ++k) {
-      L_coeffs[k] ^= GfMul64(Li_coeffs[k], Inv64(prod));
+      L_coeffs[k] ^= yacl::GfMul64(Li_coeffs[k], yacl::Inv64(prod));
     }
   }
   return L_coeffs;
@@ -90,7 +90,7 @@ auto Interpolate(const std::vector<uint64_t>& xs,
 
 void KrtwPsuSend(const std::shared_ptr<yacl::link::Context>& ctx,
                  const std::vector<uint128_t>& elem_hashes) {
-  ctx->SendAsync(ctx->NextRank(), SerializeUint128(elem_hashes.size()),
+  ctx->SendAsync(ctx->NextRank(), yacl::SerializeUint128(elem_hashes.size()),
                  "Send set size");
   size_t peer_count =
       DeserializeUint128(ctx->Recv(ctx->PrevRank(), "Receive set size"));
@@ -102,11 +102,11 @@ void KrtwPsuSend(const std::shared_ptr<yacl::link::Context>& ctx,
   auto hashing = HashInputs(elem_hashes, count);
 
   // Step 2. Prepares OPRF
-  KkrtOtExtReceiver receiver;
+  yacl::crypto::KkrtOtExtReceiver receiver;
   size_t num_ot{hashing.size() * kBinSize};
-  auto choice = RandBits(kNumBaseOT);
-  auto base_ot = BaseOtRecv(ctx, choice, kNumBaseOT);
-  auto store = IknpOtExtSend(ctx, base_ot, kNumInkpOT);
+  auto choice = yacl::crypto::RandBits(kNumBaseOT);
+  auto base_ot = yacl::crypto::BaseOtRecv(ctx, choice, kNumBaseOT);
+  auto store = yacl::crypto::IknpOtExtSend(ctx, base_ot, kNumInkpOT);
   receiver.Init(ctx, store, num_ot);
   receiver.SetBatchSize(kBatchSize);
 
@@ -130,15 +130,16 @@ void KrtwPsuSend(const std::shared_ptr<yacl::link::Context>& ctx,
       auto buf = ctx->Recv(ctx->PrevRank(), "Receive coefficients");
       std::memcpy(coeffs.data(), buf.data(), buf.size());
       auto y = Evaluate(coeffs, HashToSizeT(elem)) ^ eval;
-      ctx->SendAsync(ctx->NextRank(), SerializeUint128(y), "Send evaluation");
+      ctx->SendAsync(ctx->NextRank(), yacl::SerializeUint128(y),
+                     "Send evaluation");
     }
   }
 
   // Step 4. Sends new elements through OT
   std::vector<std::array<uint128_t, 2>> keys(num_ot);
-  choice = SecureRandBits(kNumBaseOT);
-  base_ot = BaseOtRecv(ctx, choice, kNumBaseOT);
-  IknpOtExtSend(ctx, base_ot, absl::MakeSpan(keys));
+  choice = yacl::crypto::SecureRandBits(kNumBaseOT);
+  base_ot = yacl::crypto::BaseOtRecv(ctx, choice, kNumBaseOT);
+  yacl::crypto::IknpOtExtSend(ctx, base_ot, absl::MakeSpan(keys));
   std::vector<uint128_t> ciphers(num_ot);
   for (size_t i{}; i != num_ot; ++i) {
     ciphers[i] = elems[i] ^ keys[i][0];
@@ -154,7 +155,7 @@ std::vector<uint128_t> KrtwPsuRecv(
     const std::vector<uint128_t>& elem_hashes) {
   size_t peer_count =
       DeserializeUint128(ctx->Recv(ctx->PrevRank(), "Receive set size"));
-  ctx->SendAsync(ctx->NextRank(), SerializeUint128(elem_hashes.size()),
+  ctx->SendAsync(ctx->NextRank(), yacl::SerializeUint128(elem_hashes.size()),
                  "Send set size");
   auto count = std::max(elem_hashes.size(), peer_count);
   if (count == 0) {
@@ -164,11 +165,11 @@ std::vector<uint128_t> KrtwPsuRecv(
   auto hashing = HashInputs(elem_hashes, count);
 
   // Step 2. Prepares OPRF
-  KkrtOtExtSender sender;
+  yacl::crypto::KkrtOtExtSender sender;
   size_t num_ot{hashing.size() * kBinSize};
-  auto base_ot = BaseOtSend(ctx, kNumBaseOT);
-  auto choice = RandBits(kNumInkpOT);
-  auto store = IknpOtExtRecv(ctx, base_ot, choice, kNumInkpOT);
+  auto base_ot = yacl::crypto::BaseOtSend(ctx, kNumBaseOT);
+  auto choice = yacl::crypto::RandBits(kNumInkpOT);
+  auto store = yacl::crypto::IknpOtExtRecv(ctx, base_ot, choice, kNumInkpOT);
   sender.Init(ctx, store, num_ot);
   sender.SetBatchSize(kBatchSize);
   auto oprf = sender.GetOprf();
@@ -183,27 +184,27 @@ std::vector<uint128_t> KrtwPsuRecv(
     }
     auto bin_size = hashing[bin_idx].size();
     for (size_t elem_idx{}; elem_idx != kBinSize; ++elem_idx, ++oprf_idx) {
-      auto seed = FastRandU64();
+      auto seed = yacl::crypto::FastRandU64();
       std::vector<uint64_t> xs(kBinSize), ys(kBinSize);
       for (size_t i{}; i != kBinSize; ++i) {
         xs[i] = (i < bin_size   ? HashToSizeT(hashing[bin_idx][i])
-                 : i > bin_size ? FastRandU64()
+                 : i > bin_size ? yacl::crypto::FastRandU64()
                                 : kBot);
         ys[i] = oprf->Eval(oprf_idx, xs[i]) ^ seed;
       }
       auto coeffs = Interpolate(xs, ys);
       yacl::Buffer buf(coeffs.data(), coeffs.size() * sizeof(uint64_t));
       ctx->SendAsync(ctx->NextRank(), buf, "Send coefficients");
-      auto eval =
-          DeserializeUint128(ctx->Recv(ctx->PrevRank(), "Receive evaluation"));
+      auto eval = yacl::DeserializeUint128(
+          ctx->Recv(ctx->PrevRank(), "Receive evaluation"));
       ot_choice[oprf_idx] = eval == seed;
     }
   }
 
   // Step 4. Receives new elements through OT
   std::vector<uint128_t> keys(num_ot);
-  base_ot = BaseOtSend(ctx, kNumBaseOT);
-  IknpOtExtRecv(ctx, base_ot, ot_choice, absl::MakeSpan(keys));
+  base_ot = yacl::crypto::BaseOtSend(ctx, kNumBaseOT);
+  yacl::crypto::IknpOtExtRecv(ctx, base_ot, ot_choice, absl::MakeSpan(keys));
   std::vector<uint128_t> ciphers(num_ot);
   auto buf = ctx->Recv(ctx->PrevRank(), "Receive ciphertexts");
   std::memcpy(ciphers.data(), buf.data(), buf.size());
@@ -219,4 +220,4 @@ std::vector<uint128_t> KrtwPsuRecv(
   return std::vector(set_union.begin(), set_union.end());
 }
 
-}  // namespace yacl::crypto
+}  // namespace examples::psu
diff --git a/yacl/crypto/primitives/psu/krtw19_psu.h → examples/psu/krtw19_psu.h b/yacl/crypto/primitives/psu/krtw19_psu.h → examples/psu/krtw19_psu.h
@@ -18,21 +18,21 @@
 #include <vector>
 
 #include "yacl/base/int128.h"
-#include "yacl/crypto/utils/secparam.h"
 #include "yacl/link/link.h"
 #include "yacl/math/f2k/f2k.h"
+#include "yacl/secparam.h"
 
 /* submodules */
-#include "yacl/crypto/base/hash/hash_utils.h"
-#include "yacl/crypto/primitives/ot/base_ot.h"
-#include "yacl/crypto/primitives/ot/iknp_ote.h"
-#include "yacl/crypto/primitives/ot/kkrt_ote.h"
-#include "yacl/crypto/utils/rand.h"
+#include "yacl/crypto/hash/hash_utils.h"
+#include "yacl/crypto/rand/rand.h"
+#include "yacl/kernels/algorithms/base_ot.h"
+#include "yacl/kernels/algorithms/iknp_ote.h"
+#include "yacl/kernels/algorithms/kkrt_ote.h"
 
 /* security parameter declaration */
 YACL_MODULE_DECLARE("krtw_psu", SecParam::C::k128, SecParam::S::k40);
 
-namespace yacl::crypto {
+namespace examples::psu {
 
 // Scalable Private Set Union from Symmetric-Key Techniques
 // https://eprint.iacr.org/2019/776.pdf (Figure 10)
@@ -43,4 +43,4 @@ void KrtwPsuSend(const std::shared_ptr<yacl::link::Context>&,
 std::vector<uint128_t> KrtwPsuRecv(const std::shared_ptr<yacl::link::Context>&,
                                    const std::vector<uint128_t>&);
 
-}  // namespace yacl::crypto
+}  // namespace examples::psu
diff --git a/.../crypto/primitives/psu/krtw19_psu_test.cc → examples/psu/krtw19_psu_test.cc b/.../crypto/primitives/psu/krtw19_psu_test.cc → examples/psu/krtw19_psu_test.cc
@@ -12,8 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "yacl/crypto/primitives/psu/krtw19_psu.h"
+#include "examples/psu/krtw19_psu.h"
 
+#include <algorithm>
 #include <future>
 #include <iostream>
 #include <set>
@@ -22,16 +23,16 @@
 #include "gtest/gtest.h"
 
 #include "yacl/base/exception.h"
-#include "yacl/crypto/base/hash/hash_utils.h"
-#include "yacl/crypto/utils/secparam.h"
+#include "yacl/crypto/hash/hash_utils.h"
 #include "yacl/link/test_util.h"
+#include "yacl/secparam.h"
 
 struct TestParams {
   std::vector<uint128_t> items_a;
   std::vector<uint128_t> items_b;
 };
 
-namespace yacl::crypto {
+namespace examples::psu {
 
 class KrtwPsuTest : public testing::TestWithParam<TestParams> {};
 
@@ -60,17 +61,17 @@ TEST_P(KrtwPsuTest, Works) {
 std::vector<uint128_t> CreateRangeItems(size_t begin, size_t size) {
   std::vector<uint128_t> ret;
   for (size_t i = 0; i < size; i++) {
-    ret.push_back(Blake3_128(std::to_string(begin + i)));
+    ret.push_back(yacl::crypto::Blake3_128(std::to_string(begin + i)));
   }
   return ret;
 }
 
 INSTANTIATE_TEST_SUITE_P(
     Works_Instances, KrtwPsuTest,
     testing::Values(
-        TestParams{{}, {}},                 //
-        TestParams{{}, {Blake3_128("a")}},  //
-        TestParams{{Blake3_128("a")}, {}},  //
+        TestParams{{}, {}},                               //
+        TestParams{{}, {yacl::crypto::Blake3_128("a")}},  //
+        TestParams{{yacl::crypto::Blake3_128("a")}, {}},  //
         // No overlap
         TestParams{CreateRangeItems(0, 1024), CreateRangeItems(1024, 1024)},  //
         // Partial overlap
@@ -79,4 +80,4 @@ INSTANTIATE_TEST_SUITE_P(
         TestParams{CreateRangeItems(0, 1024), CreateRangeItems(0, 1024)}  //
         ));
 
-}  // namespace yacl::crypto
+}  // namespace examples::psu