Build protobuf from cmake (#214)

* build protobuf from source

* rework protobuf cmake

* bugfixing

* generate protobuf files first

* cmake formatter

* include paths

* revert changes for windows

* debug

* update gtest workflow

.github/workflows/scripts/build_windows.bat

@@ -6,7 +6,7 @@ set PATH=%PATH%;%programfiles%/protobuf/bin/;%programfiles(x86)%/protobuf/bin/
 :: Protubf library enforces the MT flag on the static .lib build. If we want to link it, we need to use MT everywhere, otherwise the build will fail.
 :: The general recommendation is to use MD, but for that we need to build and use protobufs as DLL.
 set _CL_=/MT
- 
+
 protoc --proto_path=tenseal\proto --cpp_out=tenseal\proto tenseal\proto\tensealcontext.proto
 protoc --proto_path=tenseal\proto --cpp_out=tenseal\proto tenseal\proto\tensors.proto
 

.github/workflows/scripts/install_req_centos.sh

@@ -6,15 +6,6 @@ set -e
 curl https://github.com/Kitware/CMake/releases/download/v3.17.1/cmake-3.17.1-Linux-x86_64.tar.gz -L | tar xz -C /opt/
 echo "/opt/cmake-3.17.1-Linux-x86_64/bin" >> $GITHUB_PATH
 
-# install latest protobuf release
-curl https://github.com/protocolbuffers/protobuf/releases/download/v3.14.0/protobuf-cpp-3.14.0.tar.gz -L | tar xz -C /opt/
-pushd /opt/protobuf-3.14.0
-./configure CXXFLAGS=-fPIC
-make
-make install
-ldconfig
-popd
-
 # install python dependencies
 python -m pip install --upgrade pip
 pip install setuptools wheel twine auditwheel

.github/workflows/scripts/install_req_docker.sh

@@ -5,18 +5,6 @@ set -e
 apt update -y
 apt install curl git build-essential cmake automake libtool libtool-bin -y
 
-proto_version="3.14.0"
-curl https://github.com/protocolbuffers/protobuf/releases/download/v${proto_version}/protobuf-cpp-${proto_version}.tar.gz -L | tar xz -C /opt/ && \
-    cd /opt/protobuf-${proto_version} && \
-    autoreconf --install && \
-    ./configure CXXFLAGS=-fPIC && \
-    make && make install && ldconfig && cd -
-
-if [ $? -ne 0 ]
-then
-    exit 1
-fi
-
 python -m pip install --upgrade pip
 pip install -r requirements_dev.txt
 pip install setuptools wheel twine auditwheel

.github/workflows/scripts/install_req_macos.sh

@@ -2,8 +2,6 @@
 
 set -e
 
-brew install protobuf
-
 python -m pip install --upgrade pip
 pip install -r requirements_dev.txt
 

.github/workflows/scripts/install_req_ubuntu.sh

@@ -5,18 +5,6 @@ set -e
 sudo apt update -y
 sudo apt install curl git build-essential cmake automake libtool libtool-bin -y
 
-proto_version="3.14.0"
-curl https://github.com/protocolbuffers/protobuf/releases/download/v${proto_version}/protobuf-cpp-${proto_version}.tar.gz -L | tar xz -C /opt/ && \
-    cd /opt/protobuf-${proto_version} && \
-    autoreconf --install && \
-    ./configure CXXFLAGS=-fPIC && \
-    make && sudo make install && sudo ldconfig && cd -
-
-if [ $? -ne 0 ]
-then
-    exit 1
-fi
-
 python -m pip install --upgrade pip
 pip install -r requirements_dev.txt
 pip install setuptools wheel twine auditwheel

.github/workflows/scripts/run_gtest_nix.sh

@@ -1,5 +1,3 @@
 #!/bin/sh
 
-cd tenseal/proto && cmake . && make && cd ../../
-cmake . -D BUILD_TEST=TRUE
-make && CTEST_OUTPUT_ON_FAILURE=1 make test
+cmake . -D BUILD_TEST=TRUE && make -j && CTEST_OUTPUT_ON_FAILURE=1 make test

.gitmodules

@@ -16,3 +16,6 @@
 [submodule "third_party/xsimd"]
 	path = third_party/xsimd
 	url = https://github.com/xtensor-stack/xsimd
+[submodule "third_party/protobuf"]
+	path = third_party/protobuf
+	url = https://github.com/protocolbuffers/protobuf

cmake/protobuf.cmake

@@ -4,15 +4,58 @@ if(WIN32)
   link_libraries("$ENV{PROGRAMFILES\(x86\)}/protobuf/lib/libprotobuf.lib")
   link_libraries("$ENV{PROGRAMFILES\(x86\)}/protobuf/lib/libprotoc.lib")
 else()
-  # building protobuf using cmake
-  add_subdirectory(tenseal/proto)
-
   set(Protobuf_USE_STATIC_LIBS ON)
   set(Protobuf_DEBUG ON)
+  set(Protobuf_MSVC_STATIC_RUNTIME OFF)
 
-  find_package(Protobuf REQUIRED)
-  include(FindProtobuf)
-  include_directories(${Protobuf_INCLUDE_DIRS})
-endif()
+  set(Protobuf_ROOT ${CMAKE_SOURCE_DIR}/third_party/protobuf/cmake)
+  set(Protobuf_DIR
+      ${CMAKE_SOURCE_DIR}/third_party/protobuf/cmake/lib/cmake/protobuf/)
+
+  message(STATUS "Setting up protobuf ...")
+  execute_process(
+    COMMAND
+      ${CMAKE_COMMAND} -D protobuf_BUILD_TESTS=OFF -D
+      protobuf_MSVC_STATIC_RUNTIME=OFF -D protobuf_BUILD_LIBPROTOC=ON -D
+      protobuf_BUILD_PROTOC_BINARIES=ON -D CMAKE_POSITION_INDEPENDENT_CODE=ON -G
+      "${CMAKE_GENERATOR}" .
+    OUTPUT_QUIET
+    RESULT_VARIABLE result
+    WORKING_DIRECTORY ${Protobuf_ROOT})
+  if(result)
+    message(FATAL_ERROR "Failed to download protobuf (${result})!")
+  endif()
+
+  message(STATUS "Building protobuf ...")
+  execute_process(
+    COMMAND ${CMAKE_COMMAND} --build .
+    OUTPUT_QUIET
+    RESULT_VARIABLE result
+    WORKING_DIRECTORY ${Protobuf_ROOT})
+  if(result)
+    message(FATAL_ERROR "Failed to build protobuf (${result})!")
+  endif()
 
+  find_package(Protobuf REQUIRED HINTS ${Protobuf_ROOT}/lib/cmake/protobuf)
 
+  include(${Protobuf_ROOT}/lib/cmake/protobuf/protobuf-config.cmake)
+  include(${Protobuf_ROOT}/lib/cmake/protobuf/protobuf-module.cmake)
+  include(${Protobuf_ROOT}/lib/cmake/protobuf/protobuf-options.cmake)
+  include(${Protobuf_ROOT}/lib/cmake/protobuf/protobuf-targets.cmake)
+
+  if(Protobuf_FOUND)
+    message(STATUS "Protobuf version : ${Protobuf_VERSION}")
+    message(STATUS "Protobuf include path : ${Protobuf_INCLUDE_DIRS}")
+    message(STATUS "Protobuf libraries : ${Protobuf_LIBRARIES}")
+    message(STATUS "Protobuf compiler libraries : ${Protobuf_PROTOC_LIBRARIES}")
+    message(STATUS "Protobuf lite libraries : ${Protobuf_LITE_LIBRARIES}")
+  else()
+    message(
+      WARNING
+        "Protobuf package not found -> specify search path via Protobuf_ROOT variable"
+    )
+  endif()
+
+  include_directories(${Protobuf_INCLUDE_DIRS})
+  add_subdirectory(tenseal/proto)
+endif()

cmake/tenseal.cmake

@@ -37,5 +37,3 @@ if(NOT WIN32)
   target_link_libraries(tenseal PRIVATE tenseal_proto)
   target_link_libraries(_tenseal_cpp PRIVATE tenseal_proto)
 endif()
-
-

cmake/xtensor.cmake

@@ -3,12 +3,18 @@ set(XTENSOR_USE_XSIMD ON)
 
 include_directories(third_party/xtl/include)
 add_subdirectory(third_party/xtl)
-set(xtl_DIR "${CMAKE_CURRENT_BINARY_DIR}/third_party/xtl" CACHE STRING "" FORCE)
+set(xtl_DIR
+    "${CMAKE_CURRENT_BINARY_DIR}/third_party/xtl"
+    CACHE STRING "" FORCE)
 
 include_directories(third_party/xsimd/include)
 add_subdirectory(third_party/xsimd)
-set(xsimd_DIR "${CMAKE_CURRENT_BINARY_DIR}/third_party/xsimd" CACHE STRING "" FORCE)
+set(xsimd_DIR
+    "${CMAKE_CURRENT_BINARY_DIR}/third_party/xsimd"
+    CACHE STRING "" FORCE)
 
 include_directories(third_party/xtensor/include)
 add_subdirectory(third_party/xtensor)
-set(xtensor_DIR "${CMAKE_CURRENT_BINARY_DIR}/third_party/xtensor" CACHE STRING "" FORCE)
+set(xtensor_DIR
+    "${CMAKE_CURRENT_BINARY_DIR}/third_party/xtensor"
+    CACHE STRING "" FORCE)

tenseal/proto/CMakeLists.txt

@@ -2,9 +2,16 @@ set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
 
 set(Protobuf_USE_STATIC_LIBS ON)
+set(Protobuf_ROOT ${CMAKE_SOURCE_DIR}/third_party/protobuf/cmake)
+set(Protobuf_DIR
+    ${CMAKE_SOURCE_DIR}/third_party/protobuf/cmake/lib/cmake/protobuf/)
 
-include(FindProtobuf)
-find_package(Protobuf REQUIRED)
+if(WIN32)
+  include(FindProtobuf)
+  find_package(Protobuf REQUIRED)
+else()
+  find_package(Protobuf REQUIRED HINTS ${Protobuf_DIR})
+endif()
 
 include_directories(${Protobuf_INCLUDE_DIRS})