From ab23f22a71bc5cf619aedb92fcbc3ade176a2e8d Mon Sep 17 00:00:00 2001 From: Bogdan Cebere Date: Tue, 19 Jan 2021 13:00:19 +0200 Subject: [PATCH] Build protobuf from cmake (#214) * build protobuf from source * rework protobuf cmake * bugfixing * generate protobuf files first * cmake formatter * include paths * revert changes for windows * debug * update gtest workflow --- .github/workflows/scripts/build_windows.bat | 2 +- .../workflows/scripts/install_req_centos.sh | 9 --- .../workflows/scripts/install_req_docker.sh | 12 ---- .../workflows/scripts/install_req_macos.sh | 2 - .../workflows/scripts/install_req_ubuntu.sh | 12 ---- .github/workflows/scripts/run_gtest_nix.sh | 4 +- .gitmodules | 3 + cmake/protobuf.cmake | 57 ++++++++++++++++--- cmake/tenseal.cmake | 2 - cmake/xtensor.cmake | 12 +++- tenseal/proto/CMakeLists.txt | 11 +++- third_party/protobuf | 1 + 12 files changed, 74 insertions(+), 53 deletions(-) create mode 160000 third_party/protobuf diff --git a/.github/workflows/scripts/build_windows.bat b/.github/workflows/scripts/build_windows.bat index 048638c1..7ca9697f 100755 --- a/.github/workflows/scripts/build_windows.bat +++ b/.github/workflows/scripts/build_windows.bat @@ -6,7 +6,7 @@ set PATH=%PATH%;%programfiles%/protobuf/bin/;%programfiles(x86)%/protobuf/bin/ :: Protubf library enforces the MT flag on the static .lib build. If we want to link it, we need to use MT everywhere, otherwise the build will fail. :: The general recommendation is to use MD, but for that we need to build and use protobufs as DLL. set _CL_=/MT - + protoc --proto_path=tenseal\proto --cpp_out=tenseal\proto tenseal\proto\tensealcontext.proto protoc --proto_path=tenseal\proto --cpp_out=tenseal\proto tenseal\proto\tensors.proto diff --git a/.github/workflows/scripts/install_req_centos.sh b/.github/workflows/scripts/install_req_centos.sh index 48bd232a..cb69dfb9 100755 --- a/.github/workflows/scripts/install_req_centos.sh +++ b/.github/workflows/scripts/install_req_centos.sh @@ -6,15 +6,6 @@ set -e curl https://github.com/Kitware/CMake/releases/download/v3.17.1/cmake-3.17.1-Linux-x86_64.tar.gz -L | tar xz -C /opt/ echo "/opt/cmake-3.17.1-Linux-x86_64/bin" >> $GITHUB_PATH -# install latest protobuf release -curl https://github.com/protocolbuffers/protobuf/releases/download/v3.14.0/protobuf-cpp-3.14.0.tar.gz -L | tar xz -C /opt/ -pushd /opt/protobuf-3.14.0 -./configure CXXFLAGS=-fPIC -make -make install -ldconfig -popd - # install python dependencies python -m pip install --upgrade pip pip install setuptools wheel twine auditwheel diff --git a/.github/workflows/scripts/install_req_docker.sh b/.github/workflows/scripts/install_req_docker.sh index 21e397ed..62e5268e 100755 --- a/.github/workflows/scripts/install_req_docker.sh +++ b/.github/workflows/scripts/install_req_docker.sh @@ -5,18 +5,6 @@ set -e apt update -y apt install curl git build-essential cmake automake libtool libtool-bin -y -proto_version="3.14.0" -curl https://github.com/protocolbuffers/protobuf/releases/download/v${proto_version}/protobuf-cpp-${proto_version}.tar.gz -L | tar xz -C /opt/ && \ - cd /opt/protobuf-${proto_version} && \ - autoreconf --install && \ - ./configure CXXFLAGS=-fPIC && \ - make && make install && ldconfig && cd - - -if [ $? -ne 0 ] -then - exit 1 -fi - python -m pip install --upgrade pip pip install -r requirements_dev.txt pip install setuptools wheel twine auditwheel diff --git a/.github/workflows/scripts/install_req_macos.sh b/.github/workflows/scripts/install_req_macos.sh index 440cef31..1f668b3f 100755 --- a/.github/workflows/scripts/install_req_macos.sh +++ b/.github/workflows/scripts/install_req_macos.sh @@ -2,8 +2,6 @@ set -e -brew install protobuf - python -m pip install --upgrade pip pip install -r requirements_dev.txt diff --git a/.github/workflows/scripts/install_req_ubuntu.sh b/.github/workflows/scripts/install_req_ubuntu.sh index 4ea417d1..dcae95fe 100755 --- a/.github/workflows/scripts/install_req_ubuntu.sh +++ b/.github/workflows/scripts/install_req_ubuntu.sh @@ -5,18 +5,6 @@ set -e sudo apt update -y sudo apt install curl git build-essential cmake automake libtool libtool-bin -y -proto_version="3.14.0" -curl https://github.com/protocolbuffers/protobuf/releases/download/v${proto_version}/protobuf-cpp-${proto_version}.tar.gz -L | tar xz -C /opt/ && \ - cd /opt/protobuf-${proto_version} && \ - autoreconf --install && \ - ./configure CXXFLAGS=-fPIC && \ - make && sudo make install && sudo ldconfig && cd - - -if [ $? -ne 0 ] -then - exit 1 -fi - python -m pip install --upgrade pip pip install -r requirements_dev.txt pip install setuptools wheel twine auditwheel diff --git a/.github/workflows/scripts/run_gtest_nix.sh b/.github/workflows/scripts/run_gtest_nix.sh index 77a98a6c..2a2eb07d 100755 --- a/.github/workflows/scripts/run_gtest_nix.sh +++ b/.github/workflows/scripts/run_gtest_nix.sh @@ -1,5 +1,3 @@ #!/bin/sh -cd tenseal/proto && cmake . && make && cd ../../ -cmake . -D BUILD_TEST=TRUE -make && CTEST_OUTPUT_ON_FAILURE=1 make test +cmake . -D BUILD_TEST=TRUE && make -j && CTEST_OUTPUT_ON_FAILURE=1 make test diff --git a/.gitmodules b/.gitmodules index 422407d7..480040b0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,6 @@ [submodule "third_party/xsimd"] path = third_party/xsimd url = https://github.com/xtensor-stack/xsimd +[submodule "third_party/protobuf"] + path = third_party/protobuf + url = https://github.com/protocolbuffers/protobuf diff --git a/cmake/protobuf.cmake b/cmake/protobuf.cmake index ebe6491e..63c92529 100644 --- a/cmake/protobuf.cmake +++ b/cmake/protobuf.cmake @@ -4,15 +4,58 @@ if(WIN32) link_libraries("$ENV{PROGRAMFILES\(x86\)}/protobuf/lib/libprotobuf.lib") link_libraries("$ENV{PROGRAMFILES\(x86\)}/protobuf/lib/libprotoc.lib") else() - # building protobuf using cmake - add_subdirectory(tenseal/proto) - set(Protobuf_USE_STATIC_LIBS ON) set(Protobuf_DEBUG ON) + set(Protobuf_MSVC_STATIC_RUNTIME OFF) - find_package(Protobuf REQUIRED) - include(FindProtobuf) - include_directories(${Protobuf_INCLUDE_DIRS}) -endif() + set(Protobuf_ROOT ${CMAKE_SOURCE_DIR}/third_party/protobuf/cmake) + set(Protobuf_DIR + ${CMAKE_SOURCE_DIR}/third_party/protobuf/cmake/lib/cmake/protobuf/) + + message(STATUS "Setting up protobuf ...") + execute_process( + COMMAND + ${CMAKE_COMMAND} -D protobuf_BUILD_TESTS=OFF -D + protobuf_MSVC_STATIC_RUNTIME=OFF -D protobuf_BUILD_LIBPROTOC=ON -D + protobuf_BUILD_PROTOC_BINARIES=ON -D CMAKE_POSITION_INDEPENDENT_CODE=ON -G + "${CMAKE_GENERATOR}" . + OUTPUT_QUIET + RESULT_VARIABLE result + WORKING_DIRECTORY ${Protobuf_ROOT}) + if(result) + message(FATAL_ERROR "Failed to download protobuf (${result})!") + endif() + + message(STATUS "Building protobuf ...") + execute_process( + COMMAND ${CMAKE_COMMAND} --build . + OUTPUT_QUIET + RESULT_VARIABLE result + WORKING_DIRECTORY ${Protobuf_ROOT}) + if(result) + message(FATAL_ERROR "Failed to build protobuf (${result})!") + endif() + find_package(Protobuf REQUIRED HINTS ${Protobuf_ROOT}/lib/cmake/protobuf) + include(${Protobuf_ROOT}/lib/cmake/protobuf/protobuf-config.cmake) + include(${Protobuf_ROOT}/lib/cmake/protobuf/protobuf-module.cmake) + include(${Protobuf_ROOT}/lib/cmake/protobuf/protobuf-options.cmake) + include(${Protobuf_ROOT}/lib/cmake/protobuf/protobuf-targets.cmake) + + if(Protobuf_FOUND) + message(STATUS "Protobuf version : ${Protobuf_VERSION}") + message(STATUS "Protobuf include path : ${Protobuf_INCLUDE_DIRS}") + message(STATUS "Protobuf libraries : ${Protobuf_LIBRARIES}") + message(STATUS "Protobuf compiler libraries : ${Protobuf_PROTOC_LIBRARIES}") + message(STATUS "Protobuf lite libraries : ${Protobuf_LITE_LIBRARIES}") + else() + message( + WARNING + "Protobuf package not found -> specify search path via Protobuf_ROOT variable" + ) + endif() + + include_directories(${Protobuf_INCLUDE_DIRS}) + add_subdirectory(tenseal/proto) +endif() diff --git a/cmake/tenseal.cmake b/cmake/tenseal.cmake index 22ca7ab4..0789a543 100644 --- a/cmake/tenseal.cmake +++ b/cmake/tenseal.cmake @@ -37,5 +37,3 @@ if(NOT WIN32) target_link_libraries(tenseal PRIVATE tenseal_proto) target_link_libraries(_tenseal_cpp PRIVATE tenseal_proto) endif() - - diff --git a/cmake/xtensor.cmake b/cmake/xtensor.cmake index 89b5093d..6f55d86a 100644 --- a/cmake/xtensor.cmake +++ b/cmake/xtensor.cmake @@ -3,12 +3,18 @@ set(XTENSOR_USE_XSIMD ON) include_directories(third_party/xtl/include) add_subdirectory(third_party/xtl) -set(xtl_DIR "${CMAKE_CURRENT_BINARY_DIR}/third_party/xtl" CACHE STRING "" FORCE) +set(xtl_DIR + "${CMAKE_CURRENT_BINARY_DIR}/third_party/xtl" + CACHE STRING "" FORCE) include_directories(third_party/xsimd/include) add_subdirectory(third_party/xsimd) -set(xsimd_DIR "${CMAKE_CURRENT_BINARY_DIR}/third_party/xsimd" CACHE STRING "" FORCE) +set(xsimd_DIR + "${CMAKE_CURRENT_BINARY_DIR}/third_party/xsimd" + CACHE STRING "" FORCE) include_directories(third_party/xtensor/include) add_subdirectory(third_party/xtensor) -set(xtensor_DIR "${CMAKE_CURRENT_BINARY_DIR}/third_party/xtensor" CACHE STRING "" FORCE) +set(xtensor_DIR + "${CMAKE_CURRENT_BINARY_DIR}/third_party/xtensor" + CACHE STRING "" FORCE) diff --git a/tenseal/proto/CMakeLists.txt b/tenseal/proto/CMakeLists.txt index e5de1664..002d49c1 100644 --- a/tenseal/proto/CMakeLists.txt +++ b/tenseal/proto/CMakeLists.txt @@ -2,9 +2,16 @@ set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED True) set(Protobuf_USE_STATIC_LIBS ON) +set(Protobuf_ROOT ${CMAKE_SOURCE_DIR}/third_party/protobuf/cmake) +set(Protobuf_DIR + ${CMAKE_SOURCE_DIR}/third_party/protobuf/cmake/lib/cmake/protobuf/) -include(FindProtobuf) -find_package(Protobuf REQUIRED) +if(WIN32) + include(FindProtobuf) + find_package(Protobuf REQUIRED) +else() + find_package(Protobuf REQUIRED HINTS ${Protobuf_DIR}) +endif() include_directories(${Protobuf_INCLUDE_DIRS}) diff --git a/third_party/protobuf b/third_party/protobuf new file mode 160000 index 00000000..c17e3c13 --- /dev/null +++ b/third_party/protobuf @@ -0,0 +1 @@ +Subproject commit c17e3c134c16f9c39424d2342f723344fb0596dd