From 012ee1fb770b11d104b23eb5ea4104b90b48c200 Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Sun, 27 Feb 2022 01:21:50 -0500 Subject: [PATCH 01/18] Added Bazel platforms dependency --- WORKSPACE | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/WORKSPACE b/WORKSPACE index 65370cfbcc7..73467abd961 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -173,6 +173,16 @@ load( ) jar_jar_repositories() +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +http_archive( + name = "platforms", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.5/platforms-0.0.5.tar.gz", + "https://github.com/bazelbuild/platforms/releases/download/0.0.5/platforms-0.0.5.tar.gz", + ], + sha256 = "379113459b0feaf6bfbb584a91874c065078aa673222846ac765f86661c27407", +) + http_archive( name = "rules_python", sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0", From 288bcea4df0c602b6ea5458282e6d60945b7b3c1 Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Sun, 27 Feb 2022 01:33:51 -0500 Subject: [PATCH 02/18] Updated BUILD files to use platform info --- heron/common/src/cpp/basics/BUILD | 2 +- heron/common/src/cpp/metrics/BUILD | 2 +- third_party/gperftools/gperftools.BUILD | 20 +--------- third_party/helm/BUILD | 2 +- third_party/zookeeper/BUILD | 10 +---- tools/bazel.rc | 50 +++---------------------- tools/platform/BUILD | 36 +++++++++++++----- tools/rules/pex/BUILD | 2 +- 8 files changed, 38 insertions(+), 86 deletions(-) diff --git a/heron/common/src/cpp/basics/BUILD b/heron/common/src/cpp/basics/BUILD index 22eb43e67e3..e8d3646f41b 100644 --- a/heron/common/src/cpp/basics/BUILD +++ b/heron/common/src/cpp/basics/BUILD @@ -54,7 +54,7 @@ cc_library( "@com_github_corvusoft_kashmir_cpp//:kashmir-cxx", "@com_google_protobuf//:protobuf", ] + select({ - "//tools/platform:darwin": [], + "@platforms//os:osx": [], "//conditions:default": ["@org_nongnu_libunwind//:libunwind"], }), ) diff --git a/heron/common/src/cpp/metrics/BUILD b/heron/common/src/cpp/metrics/BUILD index 05bf2cd54ca..dfabd418913 100644 --- a/heron/common/src/cpp/metrics/BUILD +++ b/heron/common/src/cpp/metrics/BUILD @@ -43,7 +43,7 @@ cc_library( "@com_github_google_glog//:glog", "@com_google_protobuf//:protobuf", ] + select({ - "//tools/platform:darwin": [], + "@platforms//os:osx": [], "//conditions:default": ["@org_nongnu_libunwind//:libunwind"], }), ) diff --git a/third_party/gperftools/gperftools.BUILD b/third_party/gperftools/gperftools.BUILD index 6f8d2b7a4f8..2f294ed31a1 100644 --- a/third_party/gperftools/gperftools.BUILD +++ b/third_party/gperftools/gperftools.BUILD @@ -2,22 +2,6 @@ licenses(["notice"]) package(default_visibility = ["//visibility:public"]) -config_setting( - name = "darwin", - values = { - "cpu": "darwin", - }, - visibility = ["//visibility:public"], -) - -config_setting( - name = "k8", - values = { - "cpu": "k8", - }, - visibility = ["//visibility:public"], -) - common_script = [ "export UNWIND_DIR=$$(pwd)/$(GENDIR)/external/org_nongnu_libunwind", "echo $$UNWIND_DIR", @@ -43,7 +27,7 @@ linux_script = "\n".join(common_script + [ genrule( name = "gperftools-srcs", srcs = select({ - ":darwin": [], + "@platforms//os:osx": [], "//conditions:default": ["@org_apache_heron//third_party/libunwind:libunwind-files"] }), outs = [ @@ -77,7 +61,7 @@ genrule( "lib/libtcmalloc_minimal_debug.a", ], cmd = select({ - ":darwin": mac_script, + "@platforms//os:osx": mac_script, "//conditions:default": linux_script, }), ) diff --git a/third_party/helm/BUILD b/third_party/helm/BUILD index 9e4a24b003d..c3581761ce9 100644 --- a/third_party/helm/BUILD +++ b/third_party/helm/BUILD @@ -5,7 +5,7 @@ package(default_visibility = ["//visibility:public"]) filegroup( name = "helm", srcs = select({ - "//tools/platform:darwin": ["@helm_mac//:helm-exec"], + "@platforms//os:osx": ["@helm_mac//:helm-exec"], "//conditions:default": ["@helm_linux//:helm-exec"], }), ) diff --git a/third_party/zookeeper/BUILD b/third_party/zookeeper/BUILD index 456f080b4f4..b53078496f1 100644 --- a/third_party/zookeeper/BUILD +++ b/third_party/zookeeper/BUILD @@ -20,7 +20,7 @@ genrule( "lib/libzookeeper_st.a", ], cmd = select({ - ":darwin": "\n".join([ + "@platforms//os:osx": "\n".join([ "export INSTALL_DIR=$$(pwd)/$(@D)", "export TMP_DIR=$$(mktemp -d -t zookeeper.XXXXX)", "export CXXFLAGS=-Wno-error", @@ -86,11 +86,3 @@ cc_library( ], linkstatic = 1, ) - -config_setting( - name = "darwin", - values = { - "cpu": "darwin", - }, - visibility = ["//visibility:public"], -) diff --git a/tools/bazel.rc b/tools/bazel.rc index bdb85374b07..b91713fab53 100644 --- a/tools/bazel.rc +++ b/tools/bazel.rc @@ -22,48 +22,8 @@ build --ignore_unsupported_sandboxing build --spawn_strategy=standalone build --workspace_status_command scripts/release/status.sh -# For Linux -# To use it: bazel build --config=linux -build:linux --experimental_action_listener=tools/cpp:compile_cpp -build:linux --experimental_action_listener=tools/java:compile_java -build:linux --experimental_action_listener=tools/python:compile_python -build:linux --genrule_strategy=standalone -build:linux --ignore_unsupported_sandboxing -build:linux --linkopt -lm -build:linux --linkopt -lpthread -build:linux --linkopt -lrt -build:linux --spawn_strategy=standalone -build:linux --workspace_status_command scripts/release/status.sh -build:linux --copt=-O3 - -# For Mac -# To use it: bazel build --config=darwin -build:darwin --experimental_action_listener=tools/cpp:compile_cpp -build:darwin --experimental_action_listener=tools/java:compile_java -build:darwin --experimental_action_listener=tools/python:compile_python -build:darwin --genrule_strategy=standalone -build:darwin --ignore_unsupported_sandboxing -build:darwin --spawn_strategy=standalone -build:darwin --workspace_status_command scripts/release/status.sh -build:darwin --copt=-O3 - -### Disabled checkstyle - -# For Linux -# To use it: bazel build --config=linux_nostyle -build:linux_nostyle --genrule_strategy=standalone -build:linux_nostyle --ignore_unsupported_sandboxing -build:linux_nostyle --linkopt -lm -build:linux_nostyle --linkopt -lpthread -build:linux_nostyle --linkopt -lrt -build:linux_nostyle --spawn_strategy=standalone -build:linux_nostyle --workspace_status_command scripts/release/status.sh -build:linux_nostyle --copt=-O3 - -# For Mac -# To use it: bazel build --config=darwin_nostyle -build:darwin_nostyle --genrule_strategy=standalone -build:darwin_nostyle --ignore_unsupported_sandboxing -build:darwin_nostyle --spawn_strategy=standalone -build:darwin_nostyle --workspace_status_command scripts/release/status.sh -build:darwin_nostyle --copt=-O3 +# To enable stylecheck +# To use it: bazel build --config=stylecheck +build:stylecheck --experimental_action_listener=tools/cpp:compile_cpp +build:stylecheck --experimental_action_listener=tools/java:compile_java +build:stylecheck --experimental_action_listener=tools/python:compile_python diff --git a/tools/platform/BUILD b/tools/platform/BUILD index d1767700508..1fea88399db 100644 --- a/tools/platform/BUILD +++ b/tools/platform/BUILD @@ -1,17 +1,33 @@ package(default_visibility = ["//visibility:public"]) -config_setting( - name = "darwin", - values = { - "cpu": "darwin", - }, - visibility = ["//visibility:public"], -) +# config_setting( +# name = "osx_x86_64", +# constraint_values = { +# "@platforms//arch:x86_64", +# "@platforms//os:osx", +# }, +# visibility = ["//visibility:public"], +# ) + +# config_setting( +# name = "osx_arm", +# constraint_values = { +# "@platforms//arch:armv6-m1", +# "@platforms//os:osx", +# }, +# visibility = ["//visibility:public"], +# ) config_setting( - name = "k8", - values = { - "cpu": "k8", + name = "linux_x86_64", + constraint_values = { + "@platforms//arch:x86_64", + "@platforms//os:linux", }, + values = { + "linkopt": "-lm", + "linkopt": "-lpthread", + "linkopt": "-lrt", + } visibility = ["//visibility:public"], ) diff --git a/tools/rules/pex/BUILD b/tools/rules/pex/BUILD index b2aa361a7f5..f3736e4e041 100644 --- a/tools/rules/pex/BUILD +++ b/tools/rules/pex/BUILD @@ -71,7 +71,7 @@ genrule( ], outs = ["pex_wrapper.pex"], cmd = select({ - "//tools/platform:darwin": "\n".join(PRE_EXECUTE + DARWIN_EXECUTE + POST_EXECUTE), + "@platforms//os:osx": "\n".join(PRE_EXECUTE + DARWIN_EXECUTE + POST_EXECUTE), "//conditions:default": "\n".join(PRE_EXECUTE + LINUX_EXECUTE + POST_EXECUTE), }), executable = True, From 887a72a1fbc265abfbc03dfd6d0d45765189e960 Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Sun, 27 Feb 2022 02:11:00 -0500 Subject: [PATCH 03/18] Updates to remove platform setting in the scripts --- docker/compile/Dockerfile.debian10 | 3 --- docker/compile/Dockerfile.rocky8 | 3 --- docker/compile/Dockerfile.ubuntu18.04 | 3 --- docker/compile/Dockerfile.ubuntu20.04 | 3 --- docker/compile/Dockerfile.ubuntu22.04 | 3 --- docker/scripts/compile-platform.sh | 21 +++------------------ docker/scripts/test-platform.sh | 18 +----------------- docker/test/Dockerfile.centos7 | 6 ++---- docker/test/Dockerfile.ubuntu18.04 | 4 ++-- scripts/applatix/build.sh | 14 +++++--------- scripts/applatix/test.sh | 4 ---- scripts/applatix/testutils.sh | 4 ---- scripts/get_all_heron_paths.sh | 4 ++-- scripts/run_integration_test.sh | 4 ++-- scripts/run_integration_topology_test.sh | 4 ++-- scripts/shutils/common.sh | 15 --------------- scripts/travis/build.sh | 16 ++++++---------- scripts/travis/test.sh | 10 +++------- vagrant/init.sh | 2 +- vagrant/local-ci.sh | 4 +--- website2/docs/compiling-docker.md | 4 ++-- website2/docs/compiling-linux.md | 12 ++++++------ website2/website/scripts/python-doc-gen.sh | 2 +- 23 files changed, 39 insertions(+), 124 deletions(-) diff --git a/docker/compile/Dockerfile.debian10 b/docker/compile/Dockerfile.debian10 index d676cd5bbfc..e5086591f6a 100644 --- a/docker/compile/Dockerfile.debian10 +++ b/docker/compile/Dockerfile.debian10 @@ -17,9 +17,6 @@ FROM openjdk:11.0.6-jdk-buster -# This is passed to the heron build command via the --config flag -ENV TARGET_PLATFORM linux - ENV LC_ALL C.UTF-8 ENV LANG C.UTF-8 diff --git a/docker/compile/Dockerfile.rocky8 b/docker/compile/Dockerfile.rocky8 index f22c9921736..a6724e91f34 100644 --- a/docker/compile/Dockerfile.rocky8 +++ b/docker/compile/Dockerfile.rocky8 @@ -17,9 +17,6 @@ FROM rockylinux:8.5 -# This is passed to the heron build command via the --config flag -ENV TARGET_PLATFORM linux - RUN yum -y upgrade RUN yum -y install \ ant \ diff --git a/docker/compile/Dockerfile.ubuntu18.04 b/docker/compile/Dockerfile.ubuntu18.04 index 32a3431dc9a..b92cbc37d45 100644 --- a/docker/compile/Dockerfile.ubuntu18.04 +++ b/docker/compile/Dockerfile.ubuntu18.04 @@ -17,9 +17,6 @@ FROM ubuntu:18.04 -# This is passed to the heron build command via the --config flag -ENV TARGET_PLATFORM linux - ARG DEBIAN_FRONTEND=noninteractive ENV LC_ALL C.UTF-8 diff --git a/docker/compile/Dockerfile.ubuntu20.04 b/docker/compile/Dockerfile.ubuntu20.04 index ac9c79dbe02..238ffa8b230 100644 --- a/docker/compile/Dockerfile.ubuntu20.04 +++ b/docker/compile/Dockerfile.ubuntu20.04 @@ -17,9 +17,6 @@ FROM ubuntu:20.04 -# This is passed to the heron build command via the --config flag -ENV TARGET_PLATFORM linux - ARG DEBIAN_FRONTEND=noninteractive ENV LC_ALL C.UTF-8 diff --git a/docker/compile/Dockerfile.ubuntu22.04 b/docker/compile/Dockerfile.ubuntu22.04 index e411921e636..195c96ec603 100644 --- a/docker/compile/Dockerfile.ubuntu22.04 +++ b/docker/compile/Dockerfile.ubuntu22.04 @@ -17,9 +17,6 @@ FROM ubuntu:22.04 -# This is passed to the heron build command via the --config flag -ENV TARGET_PLATFORM ubuntu - ARG DEBIAN_FRONTEND=noninteractive ENV LC_ALL C.UTF-8 diff --git a/docker/scripts/compile-platform.sh b/docker/scripts/compile-platform.sh index a96bc15f480..d275e71e8e8 100755 --- a/docker/scripts/compile-platform.sh +++ b/docker/scripts/compile-platform.sh @@ -36,29 +36,14 @@ cd $SCRATCH_DIR echo "Extracting source" tar -C . -xzf $SOURCE_TARBALL -if [[ "$TARGET_PLATFORM" =~ "ubuntu" ]]; then - CONFIG_PLATFORM=linux_nostyle -elif [[ "$TARGET_PLATFORM" =~ "centos" ]]; then - CONFIG_PLATFORM=linux_nostyle -elif [[ "$TARGET_PLATFORM" =~ "rocky" ]]; then - CONFIG_PLATFORM=linux_nostyle -elif [[ "$TARGET_PLATFORM" =~ "debian" ]]; then - CONFIG_PLATFORM=linux_nostyle -elif [[ "$TARGET_PLATFORM" =~ "darwin" ]]; then - CONFIG_PLATFORM=darwin_nostyle -else - echo "Unknown platform: $TARGET_PLATFORM" - exit 1 -fi - bazel version ./bazel_configure.py bazel clean echo "Creating packages" -bazel build -c opt --jobs 25 --config=$CONFIG_PLATFORM scripts/packages:tarpkgs -bazel build -c opt --jobs 25 --config=$CONFIG_PLATFORM scripts/packages:binpkgs -bazel build -c opt --jobs 25 --config=$CONFIG_PLATFORM scripts/images:heron.tar +bazel build -c opt --copt=-O3 scripts/packages:tarpkgs +bazel build -c opt --copt=-O3 scripts/packages:binpkgs +bazel build -c opt --copt=-O3 scripts/images:heron.tar echo "Moving packages to /$OUTPUT_DIRECTORY" for file in ./bazel-bin/scripts/packages/*.tar.gz; do diff --git a/docker/scripts/test-platform.sh b/docker/scripts/test-platform.sh index 07125f27e19..2b146ec3984 100755 --- a/docker/scripts/test-platform.sh +++ b/docker/scripts/test-platform.sh @@ -36,28 +36,12 @@ cd $SCRATCH_DIR echo "Extracting source" tar -C . -xzf $SOURCE_TARBALL -if [[ "$TARGET_PLATFORM" =~ "ubuntu" ]]; then - CONFIG_PLATFORM=linux_nostyle -elif [[ "$TARGET_PLATFORM" =~ "centos" ]]; then - CONFIG_PLATFORM=linux_nostyle -elif [[ "$TARGET_PLATFORM" =~ "rocky" ]]; then - CONFIG_PLATFORM=linux_nostyle -elif [[ "$TARGET_PLATFORM" =~ "darwin" ]]; then - CONFIG_PLATFORM=darwin -elif [[ "$TARGET_PLATFORM" =~ "debian" ]]; then - CONFIG_PLATFORM=linux_nostyle -else - echo "Unknown platform: $TARGET_PLATFORM" - exit 1 -fi - bazel version ./bazel_configure.py bazel clean echo "UnitTest" -bazel test -c opt --jobs 25 \ - --config=$CONFIG_PLATFORM \ +bazel test -c opt \ --test_output=all \ --test_summary=detailed \ heron/... \ diff --git a/docker/test/Dockerfile.centos7 b/docker/test/Dockerfile.centos7 index 5925597d12f..f9085b61ef8 100644 --- a/docker/test/Dockerfile.centos7 +++ b/docker/test/Dockerfile.centos7 @@ -17,10 +17,6 @@ FROM centos:centos7 -# This is passed to the heron build command via the --config flag -ENV TARGET_PLATFORM linux -ENV LC_ALL en_US.utf8 - RUN yum -y upgrade && yum -y install centos-release-scl-rh RUN yum -y install \ automake \ @@ -31,6 +27,7 @@ RUN yum -y install \ devtoolset-8-gcc \ devtoolset-8-gcc-c++ \ git \ + glibc-langpack-en \ kernel-devel \ libtool \ make \ @@ -51,6 +48,7 @@ RUN /usr/bin/ln -sfT /opt/rh/rh-python38/root/usr/bin/python3 /usr/bin/python3 \ && /usr/bin/ln -sfT /opt/rh/devtoolset-8/root/bin/g++ /usr/bin/g++ \ && /usr/bin/ln -sfT /opt/rh/devtoolset-8/root/bin/cpp /usr/bin/cpp +ENV LC_ALL en_US.utf8 ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk ENV bazelVersion 4.2.2 diff --git a/docker/test/Dockerfile.ubuntu18.04 b/docker/test/Dockerfile.ubuntu18.04 index 874b8673bbc..98053f1fa50 100644 --- a/docker/test/Dockerfile.ubuntu18.04 +++ b/docker/test/Dockerfile.ubuntu18.04 @@ -17,8 +17,8 @@ FROM ubuntu:18.04 -# This is passed to the heron build command via the --config flag -ENV TARGET_PLATFORM linux +ENV LC_ALL C.UTF-8 +ENV LANG C.UTF-8 RUN apt-get update && apt-get -y install \ g++ \ diff --git a/scripts/applatix/build.sh b/scripts/applatix/build.sh index f39bb3ea3ff..d02fcc36279 100755 --- a/scripts/applatix/build.sh +++ b/scripts/applatix/build.sh @@ -63,10 +63,6 @@ fi set +x -# Autodiscover the platform -PLATFORM=$(discover_platform) -echo "Using $PLATFORM platform" - # Run this manually, since if it fails when run # as -workspace_status_command we don't get good output ./scripts/release/status.sh @@ -80,7 +76,7 @@ echo "Using $PLATFORM platform" T="heron build" start_timer "$T" ${UTILS}/save-logs.py "heron_build.txt" bazel\ - --bazelrc=tools/applatix/bazel.rc build --config=$PLATFORM heron/... + --bazelrc=tools/applatix/bazel.rc build heron/... end_timer "$T" # run heron unit tests @@ -89,7 +85,7 @@ start_timer "$T" ${UTILS}/save-logs.py "heron_test_non_flaky.txt" bazel\ --bazelrc=tools/applatix/bazel.rc test\ --test_summary=detailed --test_output=errors\ - --config=$PLATFORM --test_tag_filters=-flaky heron/... + --test_tag_filters=-flaky heron/... end_timer "$T" # flaky tests are often due to test port race conditions, @@ -99,21 +95,21 @@ start_timer "$T" ${UTILS}/save-logs.py "heron_test_flaky.txt" bazel\ --bazelrc=tools/applatix/bazel.rc test\ --test_summary=detailed --test_output=errors\ - --config=$PLATFORM --test_tag_filters=flaky --jobs=1 heron/... + --test_tag_filters=flaky --jobs=1 heron/... end_timer "$T" T="heron build binpkgs" start_timer "$T" ${UTILS}/save-logs.py "heron_build_binpkgs.txt" bazel\ --bazelrc=tools/applatix/bazel.rc build\ - --config=$PLATFORM scripts/packages:binpkgs + scripts/packages:binpkgs end_timer "$T" T="heron build testpkgs" start_timer "$T" ${UTILS}/save-logs.py "heron_build_binpkgs.txt" bazel\ --bazelrc=tools/applatix/bazel.rc build\ - --config=$PLATFORM scripts/packages:testpkgs + scripts/packages:testpkgs end_timer "$T" T="heron clear tar and zip files" diff --git a/scripts/applatix/test.sh b/scripts/applatix/test.sh index d0619b9fdf1..a1a3313593f 100755 --- a/scripts/applatix/test.sh +++ b/scripts/applatix/test.sh @@ -24,10 +24,6 @@ DIR=`dirname $0` UTILS=${DIR}/../shutils source ${UTILS}/common.sh -# Autodiscover the platform -PLATFORM=$(discover_platform) -echo "Using $PLATFORM platform" - # include HOME directory bin in PATH for heron cli, tools and tests export PATH=${HOME}/bin:$PATH diff --git a/scripts/applatix/testutils.sh b/scripts/applatix/testutils.sh index 988e9841999..aa425575410 100755 --- a/scripts/applatix/testutils.sh +++ b/scripts/applatix/testutils.sh @@ -24,10 +24,6 @@ DIR=`dirname $0` UTILS=${DIR}/../shutils source ${UTILS}/common.sh -# Autodiscover the platform -PLATFORM=$(discover_platform) -echo "Using $PLATFORM platform" - # include HOME directory bin in PATH for heron cli, tools and tests export PATH=${HOME}/bin:$PATH diff --git a/scripts/get_all_heron_paths.sh b/scripts/get_all_heron_paths.sh index 1be59f57acb..7dbf65775c0 100755 --- a/scripts/get_all_heron_paths.sh +++ b/scripts/get_all_heron_paths.sh @@ -21,8 +21,8 @@ set -eu set +e # Build everything DIR=`dirname $0` -source ${DIR}/detect_os_type.sh -bazel build --config=`platform`_nostyle {heron,integration_test,tools/java,examples,heronpy,storm-compatibility/v0.10.2,storm-compatibility-examples,eco,eco-storm-examples,eco-heron-examples}/... + +bazel build {heron,integration_test,tools/java,examples,heronpy,storm-compatibility/v0.10.2,storm-compatibility-examples,eco,eco-storm-examples,eco-heron-examples}/... result=$? if [ "${result}" -eq "0" ] ; then echo "Bazel build successful!!" diff --git a/scripts/run_integration_test.sh b/scripts/run_integration_test.sh index 20f224eeeca..5557b849904 100755 --- a/scripts/run_integration_test.sh +++ b/scripts/run_integration_test.sh @@ -76,8 +76,8 @@ set -e # building tar packages DIR=`dirname $0` source ${DIR}/detect_os_type.sh -bazel run --config=`platform` -- scripts/packages:heron-install.sh --user -bazel build --config=`platform` {heron/...,scripts/packages:tarpkgs,integration_test/src/...} +bazel run -- scripts/packages:heron-install.sh --user +bazel build {heron/...,scripts/packages:tarpkgs,integration_test/src/...} # run the simple http server ${HTTP_SERVER} 8080 & diff --git a/scripts/run_integration_topology_test.sh b/scripts/run_integration_topology_test.sh index 04861e9a633..318e1ccde9c 100755 --- a/scripts/run_integration_topology_test.sh +++ b/scripts/run_integration_topology_test.sh @@ -38,8 +38,8 @@ set -e # building tar packages DIR=`dirname $0` source ${DIR}/detect_os_type.sh -bazel run --config=`platform` -- scripts/packages:heron-install.sh --user -bazel build --config=`platform` {heron/...,scripts/packages:tarpkgs,integration_test/src/...} +bazel run -- scripts/packages:heron-install.sh --user +bazel build {heron/...,scripts/packages:tarpkgs,integration_test/src/...} # run the simple http server ${HTTP_SERVER} 8080 & diff --git a/scripts/shutils/common.sh b/scripts/shutils/common.sh index 236ab73a895..3888bc3433e 100755 --- a/scripts/shutils/common.sh +++ b/scripts/shutils/common.sh @@ -90,21 +90,6 @@ function print_timer_summary { done } -# Discover the platform that we are running on -function discover_platform { - platform='unknown' - unamestr=$(uname) - if [[ "$unamestr" == 'Linux' ]]; then - echo "linux" - elif [[ "$unamestr" == 'Darwin' ]]; then - echo "darwin" - else - mysterious=`echo $unamestr | awk -F- '{print $6}'` - echo "$unamestr platform not supported" - exit 1 - fi -} - # Check the ci environment is valid function ci_environ { environ=$1 diff --git a/scripts/travis/build.sh b/scripts/travis/build.sh index cf04f988f60..b26bc1e07b6 100755 --- a/scripts/travis/build.sh +++ b/scripts/travis/build.sh @@ -60,10 +60,6 @@ fi set +x -# Autodiscover the platform -PLATFORM=$(discover_platform) -echo "Using $PLATFORM platform" - # Run this manually, since if it fails when run # as -workspace_status_command we don't get good output ./scripts/release/status.sh @@ -77,7 +73,7 @@ echo "Using $PLATFORM platform" T="heron build" start_timer "$T" ${UTILS}/save-logs.py "heron_build.txt" bazel\ - --bazelrc=tools/travis/bazel.rc build --config=$PLATFORM heron/... \ + --bazelrc=tools/travis/bazel.rc build --config=stylecheck heron/... \ heronpy/... examples/... storm-compatibility-examples/v0.10.2/... \ eco-storm-examples/... eco-heron-examples/... contrib/... end_timer "$T" @@ -88,7 +84,7 @@ start_timer "$T" ${UTILS}/save-logs.py "heron_test_non_flaky.txt" bazel\ --bazelrc=tools/travis/bazel.rc test\ --test_summary=detailed --test_output=errors\ - --config=$PLATFORM --test_tag_filters=-flaky heron/... \ + --config=stylecheck --test_tag_filters=-flaky heron/... \ heronpy/... examples/... storm-compatibility-examples/v0.10.2/... \ eco-storm-examples/... eco-heron-examples/... contrib/... end_timer "$T" @@ -100,7 +96,7 @@ start_timer "$T" ${UTILS}/save-logs.py "heron_test_flaky.txt" bazel\ --bazelrc=tools/travis/bazel.rc test\ --test_summary=detailed --test_output=errors\ - --config=$PLATFORM --test_tag_filters=flaky --jobs=1 heron/... \ + --config=stylecheck --test_tag_filters=flaky --jobs=1 heron/... \ heronpy/... examples/... storm-compatibility-examples/v0.10.2/... \ eco-storm-examples/... eco-heron-examples/... end_timer "$T" @@ -110,21 +106,21 @@ T="heron build tarpkgs" start_timer "$T" ${UTILS}/save-logs.py "heron_build_tarpkgs.txt" bazel\ --bazelrc=tools/travis/bazel.rc build\ - --config=$PLATFORM scripts/packages:tarpkgs + --config=stylecheck scripts/packages:tarpkgs end_timer "$T" T="heron build binpkgs" start_timer "$T" ${UTILS}/save-logs.py "heron_build_binpkgs.txt" bazel\ --bazelrc=tools/travis/bazel.rc build\ - --config=$PLATFORM scripts/packages:binpkgs + --config=stylecheck scripts/packages:binpkgs end_timer "$T" T="heron build docker images" start_timer "$T" ${UTILS}/save-logs.py "heron_build_binpkgs.txt" bazel\ --bazelrc=tools/travis/bazel.rc build\ - --config=$PLATFORM scripts/images:heron.tar + --config=stylecheck scripts/images:heron.tar end_timer "$T" diff --git a/scripts/travis/test.sh b/scripts/travis/test.sh index e1915fe3d5a..ca0648e2fed 100755 --- a/scripts/travis/test.sh +++ b/scripts/travis/test.sh @@ -24,10 +24,6 @@ DIR=`dirname $0` UTILS=${DIR}/../shutils source ${UTILS}/common.sh -# Autodiscover the platform -PLATFORM=$(discover_platform) -echo "Using $PLATFORM platform" - # integration test binaries have to be specified as absolute path JAVA_INTEGRATION_TESTS_BIN="${HOME}/.herontests/lib/integration-tests.jar" JAVA_INTEGRATION_TOPOLOGY_TESTS_BIN="${HOME}/.herontests/lib/integration-topology-tests.jar" @@ -37,19 +33,19 @@ SCALA_INTEGRATION_TESTS_BIN="${HOME}/.herontests/lib/scala-integration-tests.jar # build test related jar T="heron build integration_test" start_timer "$T" -${UTILS}/save-logs.py "heron_build_integration_test.txt" bazel --bazelrc=tools/travis/bazel.rc build --config=$PLATFORM integration_test/src/... +${UTILS}/save-logs.py "heron_build_integration_test.txt" bazel --bazelrc=tools/travis/bazel.rc build integration_test/src/... end_timer "$T" # install heron T="heron install" start_timer "$T" -${UTILS}/save-logs.py "heron_install.txt" bazel --bazelrc=tools/travis/bazel.rc run --config=$PLATFORM -- scripts/packages:heron-install.sh --user +${UTILS}/save-logs.py "heron_install.txt" bazel --bazelrc=tools/travis/bazel.rc run -- scripts/packages:heron-install.sh --user end_timer "$T" # install tests T="heron tests install" start_timer "$T" -${UTILS}/save-logs.py "heron_tests_install.txt" bazel --bazelrc=tools/travis/bazel.rc run --config=$PLATFORM -- scripts/packages:heron-tests-install.sh --user +${UTILS}/save-logs.py "heron_tests_install.txt" bazel --bazelrc=tools/travis/bazel.rc run -- scripts/packages:heron-tests-install.sh --user end_timer "$T" pathadd ${HOME}/bin/ diff --git a/vagrant/init.sh b/vagrant/init.sh index 963ac255627..5753c0ef8d4 100644 --- a/vagrant/init.sh +++ b/vagrant/init.sh @@ -59,7 +59,7 @@ build_heron() { pushd /vagrant bazel clean ./bazel_configure.py - bazel --bazelrc=tools/travis/bazel.rc build --config=linux heron/... + bazel --bazelrc=tools/travis/bazel.rc build heron/... popd } diff --git a/vagrant/local-ci.sh b/vagrant/local-ci.sh index 087894c6664..256991ad1d6 100755 --- a/vagrant/local-ci.sh +++ b/vagrant/local-ci.sh @@ -24,8 +24,6 @@ To only run integration tests: To run the full ci pipeline: ./local-ci.sh ci -The VM does not report the platform in python as expected, so PLATFORM=Ubuntu is needed to work around that for the CI script's platform discovery. - DOC set -o errexit -o nounset -o pipefail @@ -41,7 +39,7 @@ fi # allows you to do `$0 test` to run only integration tests script="${1-ci}" -env="PLATFORM=Ubuntu JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/" +env="JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/" # run the CI, if it fails drop into a shell vagrant ssh primary --command "cd /vagrant && $env ./scripts/travis/$script.sh" \ || vagrant ssh primary --command "cd /vagrant && $env exec bash" diff --git a/website2/docs/compiling-docker.md b/website2/docs/compiling-docker.md index 04c8b80cd1f..1abfb28a04c 100644 --- a/website2/docs/compiling-docker.md +++ b/website2/docs/compiling-docker.md @@ -147,8 +147,8 @@ After the commands, a new docker container is started with all the libraries and installed. The operation system is Ubuntu 18.04 by default. Now you can build Heron like: ```bash -\# bazel build --config=linux scripts/packages:binpkgs -\# bazel build --config=linux scripts/packages:tarpkgs +\# bazel build scripts/packages:binpkgs +\# bazel build scripts/packages:tarpkgs ``` The current folder is mapped to the '/heron' directory in the container and any changes diff --git a/website2/docs/compiling-linux.md b/website2/docs/compiling-linux.md index 84e94cebc31..24fbf3bf488 100644 --- a/website2/docs/compiling-linux.md +++ b/website2/docs/compiling-linux.md @@ -96,14 +96,14 @@ $ ./bazel_configure.py ### Step 10 --- Build the project ```bash -$ bazel build --config=linux heron/... +$ bazel build heron/... ``` ### Step 11 --- Build the packages ```bash -$ bazel build --config=linux scripts/packages:binpkgs -$ bazel build --config=linux scripts/packages:tarpkgs +$ bazel build scripts/packages:binpkgs +$ bazel build scripts/packages:tarpkgs ``` This will install Heron packages in the `bazel-bin/scripts/packages/` directory. @@ -198,14 +198,14 @@ bazelVersion %}}). ```bash $ git clone https://github.com/apache/incubator-heron.git && cd heron $ ./bazel_configure.py -$ bazel build --config=linux heron/... +$ bazel build heron/... ``` ### Step 7 --- Build the binary packages ```bash -$ bazel build --config=linux scripts/packages:binpkgs -$ bazel build --config=linux scripts/packages:tarpkgs +$ bazel build scripts/packages:binpkgs +$ bazel build scripts/packages:tarpkgs ``` This will install Heron packages in the `bazel-bin/scripts/packages/` directory. diff --git a/website2/website/scripts/python-doc-gen.sh b/website2/website/scripts/python-doc-gen.sh index 506abf370a7..87aec134e53 100755 --- a/website2/website/scripts/python-doc-gen.sh +++ b/website2/website/scripts/python-doc-gen.sh @@ -25,7 +25,7 @@ cd ${HERON_ROOT_DIR} ./bazel_configure.py # Generate python whl packages, packages will be generated in ${HERON_ROOT_DIR}/bazel-bin/scripts/packages/ -bazel build --config=linux_nostyle scripts/packages:pypkgs +bazel build scripts/packages:pypkgs cd website2/website/ mkdir -p ./tmp/ From 5ef6e294a24f5e4bda34707a825951032af06328 Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Sun, 27 Feb 2022 02:42:47 -0500 Subject: [PATCH 04/18] Text cleanup --- docker/scripts/dev-env-create.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/scripts/dev-env-create.sh b/docker/scripts/dev-env-create.sh index f8fd3623745..e26b5d5d170 100755 --- a/docker/scripts/dev-env-create.sh +++ b/docker/scripts/dev-env-create.sh @@ -28,8 +28,8 @@ # After the container is started, you can build Heron with bazel # (ubuntu config is used in the example): # ./bazel_configure.py -# bazel build --config=linux heron/... -# bazel build --config=linux scripts/packages:binpkgs +# bazel build heron/... +# bazel build scripts/packages:binpkgs set -o nounset set -o errexit From e4875bac8d283f45d56d14dcfebb2345adf80c25 Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Sun, 27 Feb 2022 02:44:52 -0500 Subject: [PATCH 05/18] Removing TravisCI special config. --- scripts/travis/build.sh | 12 +- scripts/travis/test.sh | 6 +- tools/travis/bazel.rc | 60 ------- tools/travis/toolchain/BUILD | 60 ------- tools/travis/toolchain/CROSSTOOL | 147 ------------------ .../travis/toolchain/cc_toolchain_config.bzl | 18 --- 6 files changed, 9 insertions(+), 294 deletions(-) delete mode 100644 tools/travis/bazel.rc delete mode 100644 tools/travis/toolchain/BUILD delete mode 100644 tools/travis/toolchain/CROSSTOOL delete mode 100644 tools/travis/toolchain/cc_toolchain_config.bzl diff --git a/scripts/travis/build.sh b/scripts/travis/build.sh index b26bc1e07b6..1cb0c9dc3fe 100755 --- a/scripts/travis/build.sh +++ b/scripts/travis/build.sh @@ -73,7 +73,7 @@ set +x T="heron build" start_timer "$T" ${UTILS}/save-logs.py "heron_build.txt" bazel\ - --bazelrc=tools/travis/bazel.rc build --config=stylecheck heron/... \ + build --config=stylecheck heron/... \ heronpy/... examples/... storm-compatibility-examples/v0.10.2/... \ eco-storm-examples/... eco-heron-examples/... contrib/... end_timer "$T" @@ -82,7 +82,7 @@ end_timer "$T" T="heron test non-flaky" start_timer "$T" ${UTILS}/save-logs.py "heron_test_non_flaky.txt" bazel\ - --bazelrc=tools/travis/bazel.rc test\ + test\ --test_summary=detailed --test_output=errors\ --config=stylecheck --test_tag_filters=-flaky heron/... \ heronpy/... examples/... storm-compatibility-examples/v0.10.2/... \ @@ -94,7 +94,7 @@ end_timer "$T" T="heron test flaky" start_timer "$T" ${UTILS}/save-logs.py "heron_test_flaky.txt" bazel\ - --bazelrc=tools/travis/bazel.rc test\ + test\ --test_summary=detailed --test_output=errors\ --config=stylecheck --test_tag_filters=flaky --jobs=1 heron/... \ heronpy/... examples/... storm-compatibility-examples/v0.10.2/... \ @@ -105,21 +105,21 @@ end_timer "$T" T="heron build tarpkgs" start_timer "$T" ${UTILS}/save-logs.py "heron_build_tarpkgs.txt" bazel\ - --bazelrc=tools/travis/bazel.rc build\ + build\ --config=stylecheck scripts/packages:tarpkgs end_timer "$T" T="heron build binpkgs" start_timer "$T" ${UTILS}/save-logs.py "heron_build_binpkgs.txt" bazel\ - --bazelrc=tools/travis/bazel.rc build\ + build\ --config=stylecheck scripts/packages:binpkgs end_timer "$T" T="heron build docker images" start_timer "$T" ${UTILS}/save-logs.py "heron_build_binpkgs.txt" bazel\ - --bazelrc=tools/travis/bazel.rc build\ + build\ --config=stylecheck scripts/images:heron.tar end_timer "$T" diff --git a/scripts/travis/test.sh b/scripts/travis/test.sh index ca0648e2fed..a9a2dca76b9 100755 --- a/scripts/travis/test.sh +++ b/scripts/travis/test.sh @@ -33,19 +33,19 @@ SCALA_INTEGRATION_TESTS_BIN="${HOME}/.herontests/lib/scala-integration-tests.jar # build test related jar T="heron build integration_test" start_timer "$T" -${UTILS}/save-logs.py "heron_build_integration_test.txt" bazel --bazelrc=tools/travis/bazel.rc build integration_test/src/... +${UTILS}/save-logs.py "heron_build_integration_test.txt" bazel build integration_test/src/... end_timer "$T" # install heron T="heron install" start_timer "$T" -${UTILS}/save-logs.py "heron_install.txt" bazel --bazelrc=tools/travis/bazel.rc run -- scripts/packages:heron-install.sh --user +${UTILS}/save-logs.py "heron_install.txt" bazel run -- scripts/packages:heron-install.sh --user end_timer "$T" # install tests T="heron tests install" start_timer "$T" -${UTILS}/save-logs.py "heron_tests_install.txt" bazel --bazelrc=tools/travis/bazel.rc run -- scripts/packages:heron-tests-install.sh --user +${UTILS}/save-logs.py "heron_tests_install.txt" bazel run -- scripts/packages:heron-tests-install.sh --user end_timer "$T" pathadd ${HOME}/bin/ diff --git a/tools/travis/bazel.rc b/tools/travis/bazel.rc deleted file mode 100644 index 518f2d1d0bf..00000000000 --- a/tools/travis/bazel.rc +++ /dev/null @@ -1,60 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -build --host_force_python=PY3 - -# This is from Bazel's former travis setup, to avoid blowing up the RAM usage. -startup --host_jvm_args=-Xmx2500m -startup --host_jvm_args=-Xms2500m -startup --batch -test --local_ram_resources=HOST_RAM*0.10 -test --test_output=errors - -# This is so we understand failures better -build --verbose_failures - -build --show_timestamps -test --show_timestamps - -# Limits the jobs to 25 since the resources are limited -build --jobs 25 - -# Link with the appropriate libs -build --linkopt -lm -build --linkopt -lpthread -build --linkopt -lrt -build --experimental_action_listener=tools/cpp:compile_cpp -build --experimental_action_listener=tools/java:compile_java -build --experimental_action_listener=tools/python:compile_python -build --workspace_status_command scripts/release/status.sh - -# This is so we use a recent enough GCC when building. -# build --crosstool_top //tools/travis/toolchain:CROSSTOOL - -# This is so we don't use sandboxed execution. Sandboxed execution -# runs stuff in a container, and since Travis already runs its script -# in a container (unless you require sudo in your .travis.yml) this -# fails to run tests. -build --spawn_strategy=standalone --genrule_strategy=standalone -test --test_strategy=standalone - -# Ignore unsupported warning for sandboxing -build --ignore_unsupported_sandboxing - -# Below this line, .travis.yml will cat the default bazelrc. -# This is needed so Bazel starts with the base workspace in its -# package path. diff --git a/tools/travis/toolchain/BUILD b/tools/travis/toolchain/BUILD deleted file mode 100644 index 157a1d8b28f..00000000000 --- a/tools/travis/toolchain/BUILD +++ /dev/null @@ -1,60 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_toolchain") -load(":cc_toolchain_config.bzl", "cc_toolchain_config") - -licenses(["restricted"]) - -package(default_visibility = ["//visibility:public"]) - -filegroup( - name = "empty", - srcs = [], -) - -# This is the entry point for --crosstool_top. Toolchains are found -# by lopping off the name of --crosstool_top and searching for -# "cc-compiler-${CPU}" in this BUILD file, where CPU is the target CPU -# specified in --cpu. -# -# This file group should include -# * all cc_toolchain targets supported -# * all file groups that said cc_toolchain might refer to, -# including the default_grte_top setting in the CROSSTOOL -# protobuf. -filegroup( - name = "toolchain", - srcs = [ - ":cc-compiler-local", - ":empty", - ], -) - -cc_toolchain_config(name = "k8_toolchain_config") - -cc_toolchain( - name = "cc-compiler-local", - all_files = ":empty", - compiler_files = ":empty", - dwp_files = ":empty", - linker_files = ":empty", - objcopy_files = ":empty", - strip_files = ":empty", - supports_param_files = 0, - toolchain_config = ":k8_toolchain_config", -) - -cc_toolchain( - name = "cc-compiler-k8", - all_files = ":empty", - compiler_files = ":empty", - dwp_files = ":empty", - linker_files = ":empty", - objcopy_files = ":empty", - strip_files = ":empty", - supports_param_files = 0, - toolchain_config = ":k8_toolchain_config", -) - -filegroup( - name = "srcs", - srcs = glob(["**"]), -) diff --git a/tools/travis/toolchain/CROSSTOOL b/tools/travis/toolchain/CROSSTOOL deleted file mode 100644 index e70b1881fbf..00000000000 --- a/tools/travis/toolchain/CROSSTOOL +++ /dev/null @@ -1,147 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -major_version: "local" -minor_version: "" -default_target_cpu: "same_as_host" - -default_toolchain { - cpu: "k8" - toolchain_identifier: "local_linux" -} - -toolchain { - abi_version: "local" - abi_libc_version: "local" - builtin_sysroot: "" - compiler: "compiler" - host_system_name: "local" - needsPic: true - supports_gold_linker: false - supports_incremental_linker: false - supports_fission: false - supports_interface_shared_objects: false - supports_normalizing_ar: false - supports_start_end_lib: false - supports_thin_archives: false - target_libc: "local" - target_cpu: "local" - target_system_name: "local" - toolchain_identifier: "local_linux" - - tool_path { name: "ar" path: "/usr/bin/ar" } - tool_path { name: "compat-ld" path: "/usr/bin/ld" } - tool_path { name: "cpp" path: "/usr/bin/cpp" } - tool_path { name: "dwp" path: "/usr/bin/dwp" } - tool_path { name: "gcc" path: "/usr/bin/gcc-4.8" } - cxx_flag: "-std=c++0x" - linker_flag: "-lstdc++" - linker_flag: "-B/usr/bin/" - - # TODO(bazel-team): In theory, the path here ought to exactly match the path - # used by gcc. That works because bazel currently doesn't track files at - # absolute locations and has no remote execution, yet. However, this will need - # to be fixed, maybe with auto-detection? - cxx_builtin_include_directory: "/usr/lib/gcc/" - cxx_builtin_include_directory: "/usr/local/include" - cxx_builtin_include_directory: "/usr/include" - tool_path { name: "gcov" path: "/usr/bin/gcov" } - - # C(++) compiles invoke the compiler (as that is the one knowing where - # to find libraries), but we provide LD so other rules can invoke the linker. - tool_path { name: "ld" path: "/usr/bin/ld" } - - tool_path { name: "nm" path: "/usr/bin/nm" } - tool_path { name: "objcopy" path: "/usr/bin/objcopy" } - objcopy_embed_flag: "-I" - objcopy_embed_flag: "binary" - tool_path { name: "objdump" path: "/usr/bin/objdump" } - tool_path { name: "strip" path: "/usr/bin/strip" } - - # Anticipated future default. - unfiltered_cxx_flag: "-no-canonical-prefixes" - - # Make C++ compilation deterministic. Use linkstamping instead of these - # compiler symbols. - unfiltered_cxx_flag: "-Wno-builtin-macro-redefined" - unfiltered_cxx_flag: "-D__DATE__=\"redacted\"" - unfiltered_cxx_flag: "-D__TIMESTAMP__=\"redacted\"" - unfiltered_cxx_flag: "-D__TIME__=\"redacted\"" - - # Security hardening on by default. - # Conservative choice; -D_FORTIFY_SOURCE=2 may be unsafe in some cases. - # We need to undef it before redefining it as some distributions now have - # it enabled by default. - compiler_flag: "-U_FORTIFY_SOURCE" - compiler_flag: "-D_FORTIFY_SOURCE=1" - compiler_flag: "-fstack-protector" - linker_flag: "-Wl,-z,relro,-z,now" - - # Enable coloring even if there's no attached terminal. Bazel removes the - # escape sequences if --nocolor is specified. This isn't supported by gcc - # on Ubuntu 14.04. - # compiler_flag: "-fcolor-diagnostics" - - # All warnings are enabled. Maybe enable -Werror as well? - compiler_flag: "-Wall" - # Enable a few more warnings that aren't part of -Wall. - compiler_flag: "-Wunused-but-set-parameter" - # But disable some that are problematic. - compiler_flag: "-Wno-free-nonheap-object" # has false positives - - # Keep stack frames for debugging, even in opt mode. - compiler_flag: "-fno-omit-frame-pointer" - - # Anticipated future default. - linker_flag: "-no-canonical-prefixes" - # Have gcc return the exit code from ld. - linker_flag: "-pass-exit-codes" - # Stamp the binary with a unique identifier. - linker_flag: "-Wl,--build-id=md5" - linker_flag: "-Wl,--hash-style=gnu" - # Gold linker only? Can we enable this by default? - # linker_flag: "-Wl,--warn-execstack" - # linker_flag: "-Wl,--detect-odr-violations" - - compilation_mode_flags { - mode: DBG - # Enable debug symbols. - compiler_flag: "-g" - } - compilation_mode_flags { - mode: OPT - - # No debug symbols. - # Maybe we should enable https://gcc.gnu.org/wiki/DebugFission for opt or - # even generally? However, that can't happen here, as it requires special - # handling in Bazel. - compiler_flag: "-g0" - - # Conservative choice for -O - # -O3 can increase binary size and even slow down the resulting binaries. - # Profile first and / or use FDO if you need better performance than this. - compiler_flag: "-O3" - - # Disable assertions - compiler_flag: "-DNDEBUG" - - # Removal of unused code and data at link time (can this increase binary size in some cases?). - compiler_flag: "-ffunction-sections" - compiler_flag: "-fdata-sections" - linker_flag: "-Wl,--gc-sections" - } -} diff --git a/tools/travis/toolchain/cc_toolchain_config.bzl b/tools/travis/toolchain/cc_toolchain_config.bzl deleted file mode 100644 index d8f0a3d93a5..00000000000 --- a/tools/travis/toolchain/cc_toolchain_config.bzl +++ /dev/null @@ -1,18 +0,0 @@ -def _impl(ctx): - return cc_common.create_cc_toolchain_config_info( - ctx = ctx, - toolchain_identifier = "k8-toolchain", - host_system_name = "local", - target_system_name = "local", - target_cpu = "k8", - target_libc = "unknown", - compiler = "clang", - abi_version = "unknown", - abi_libc_version = "unknown", - ) - -cc_toolchain_config = rule( - implementation = _impl, - attrs = {}, - provides = [CcToolchainConfigInfo], -) \ No newline at end of file From 457023f853d4531277bd86495de36888b8753828 Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Sun, 27 Feb 2022 15:32:06 -0500 Subject: [PATCH 06/18] Remove outdated Applatix CI scripts --- scripts/applatix/build.sh | 125 -------------------------------- scripts/applatix/ci.sh | 30 -------- scripts/applatix/javatests.sh | 58 --------------- scripts/applatix/prepare.sh | 20 ----- scripts/applatix/pythontests.sh | 43 ----------- scripts/applatix/test.sh | 85 ---------------------- scripts/applatix/testutils.sh | 42 ----------- 7 files changed, 403 deletions(-) delete mode 100755 scripts/applatix/build.sh delete mode 100755 scripts/applatix/ci.sh delete mode 100755 scripts/applatix/javatests.sh delete mode 100755 scripts/applatix/prepare.sh delete mode 100755 scripts/applatix/pythontests.sh delete mode 100755 scripts/applatix/test.sh delete mode 100755 scripts/applatix/testutils.sh diff --git a/scripts/applatix/build.sh b/scripts/applatix/build.sh deleted file mode 100755 index d02fcc36279..00000000000 --- a/scripts/applatix/build.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Script to kick off the travis CI build. We want the build to fail-fast if any -# of the below commands fail so we need to chain them in this script. -# - -set -e - -DIR=`dirname $0` -UTILS=${DIR}/../shutils -source ${UTILS}/common.sh - -# verify that jars have not been added to the repo -JARS=`find . -name "*.jar"` -if [ "$JARS" ]; then - echo "ERROR: The following jars were found in the repo, "\ - "which is not permitted. Instead add the jar to WORKSPACE as a maven_jar." - echo $JARS - exit 1 -fi - -# verify that eggs have not been added to the repo -# ./third_party/pex/wheel-0.23.0-py2.7.egg should be the only one -set +e -EGGS=`find . -name "*.egg" | grep -v "third_party/pex/wheel"` -set -e -if [ "$EGGS" ]; then - echo 'ERROR: The following eggs were found in the repo, '\ - 'which is not permitted. Python dependencies should be '\ - 'added using the "reqs" attribute:' - echo $EGGS - exit 1 -fi - -# verify that wheels have not been added to the repo -# ./third_party/pex/setuptools-18.0.1-py2.py3-none-any.whl should be the only one -set +e -WHEELS=`find . -name "*.whl" | grep -v "third_party/pex/setuptools"` -set -e -if [ "$WHEELS" ]; then - echo 'ERROR: The following wheels were found in the repo, '\ - 'which is not permitted. Python dependencies should be added using '\ - 'the "reqs" attribute:' - echo $WHEELS - exit 1 -fi - -set +x - -# Run this manually, since if it fails when run -# as -workspace_status_command we don't get good output -./scripts/release/status.sh - -# append the bazel default bazelrc to applatix/bazel.rc -# for using rules provided by bazel -# cat ~/.bazelrc >> tools/applatix/bazel.rc -./bazel_configure.py - -# build heron -T="heron build" -start_timer "$T" -${UTILS}/save-logs.py "heron_build.txt" bazel\ - --bazelrc=tools/applatix/bazel.rc build heron/... -end_timer "$T" - -# run heron unit tests -T="heron test non-flaky" -start_timer "$T" -${UTILS}/save-logs.py "heron_test_non_flaky.txt" bazel\ - --bazelrc=tools/applatix/bazel.rc test\ - --test_summary=detailed --test_output=errors\ - --test_tag_filters=-flaky heron/... -end_timer "$T" - -# flaky tests are often due to test port race conditions, -# which should be fixed. For now, run them serially -T="heron test flaky" -start_timer "$T" -${UTILS}/save-logs.py "heron_test_flaky.txt" bazel\ - --bazelrc=tools/applatix/bazel.rc test\ - --test_summary=detailed --test_output=errors\ - --test_tag_filters=flaky --jobs=1 heron/... -end_timer "$T" - -T="heron build binpkgs" -start_timer "$T" -${UTILS}/save-logs.py "heron_build_binpkgs.txt" bazel\ - --bazelrc=tools/applatix/bazel.rc build\ - scripts/packages:binpkgs -end_timer "$T" - -T="heron build testpkgs" -start_timer "$T" -${UTILS}/save-logs.py "heron_build_binpkgs.txt" bazel\ - --bazelrc=tools/applatix/bazel.rc build\ - scripts/packages:testpkgs -end_timer "$T" - -T="heron clear tar and zip files" -start_timer "$T" -rm -rf ./bazel-bin/scripts/packages/*.tar -rm -rf ./bazel-bin/scripts/packages/heron-api.tar.gz -rm -rf ./bazel-bin/scripts/packages/heron-tools.tar.gz -rm -rf ./bazel-bin/scripts/packages/heron-client.tar.gz -rm -rf ./bazel-bin/scripts/packages/*.args -rm -rf ./bazel-bin/scripts/packages/*.zip -end_timer "$T" - -print_timer_summary diff --git a/scripts/applatix/ci.sh b/scripts/applatix/ci.sh deleted file mode 100755 index 7412ba6d906..00000000000 --- a/scripts/applatix/ci.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -export PATH=$PATH:/root/bin - -which gcc -gcc --version - -which g++ -g++ --version - -which python -python --version - -cd /heron && USER=abc scripts/travis/ci.sh diff --git a/scripts/applatix/javatests.sh b/scripts/applatix/javatests.sh deleted file mode 100755 index 308aef9a4fa..00000000000 --- a/scripts/applatix/javatests.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Script to kick off the travis CI integration test. Fail-fast if any of tthe below commands fail. -# -set -e - -DIR=`dirname $0` -source ${DIR}/testutils.sh - -# integration test binaries have to be specified as absolute path -JAVA_INTEGRATION_TESTS_BIN="${HOME}/.herontests/lib/integration-tests.jar" -SCALA_INTEGRATION_TESTS_BIN="${HOME}/.herontests/lib/scala-integration-tests.jar" - -# initialize http-server for integration tests -T="heron integration_test http-server initialization" -start_timer "$T" -${HOME}/bin/http-server 8080 & -http_server_id=$! -trap "kill -9 $http_server_id" SIGINT SIGTERM EXIT -end_timer "$T" - -# run the scala integration test -T="heron integration_test scala" -start_timer "$T" -${HOME}/bin/test-runner \ - -hc heron -tb ${SCALA_INTEGRATION_TESTS_BIN} \ - -rh localhost -rp 8080\ - -tp ${HOME}/.herontests/data/scala \ - -cl local -rl heron-staging -ev devel -end_timer "$T" - -# run the java integration test -T="heron integration_test java" -start_timer "$T" -${HOME}/bin/test-runner \ - -hc heron -tb ${JAVA_INTEGRATION_TESTS_BIN} \ - -rh localhost -rp 8080\ - -tp ${HOME}/.herontests/data/java \ - -cl local -rl heron-staging -ev devel -end_timer "$T" - -print_timer_summary diff --git a/scripts/applatix/prepare.sh b/scripts/applatix/prepare.sh deleted file mode 100755 index a1d2f1e5f6f..00000000000 --- a/scripts/applatix/prepare.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ln -s /usr/bin/nodejs /usr/bin/node -cd /heron/website && npm install && npm install gulp diff --git a/scripts/applatix/pythontests.sh b/scripts/applatix/pythontests.sh deleted file mode 100755 index 1a94b3521c5..00000000000 --- a/scripts/applatix/pythontests.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Script to kick off the travis CI integration test. Fail-fast if any of tthe below commands fail. -# -set -e - -DIR=`dirname $0` -source ${DIR}/testutils.sh - -# integration test binaries have to be specified as absolute path -PYTHON_INTEGRATION_TESTS_BIN="${HOME}/.herontests/lib/heron_integ_topology.pex" - -# run the python integration test -T="heron integration_test python" -start_timer "$T" -${HOME}/bin/http-server 8080 & -http_server_id=$! -trap "kill -9 $http_server_id" SIGINT SIGTERM EXIT - -${HOME}/bin/test-runner \ - -hc heron -tb ${PYTHON_INTEGRATION_TESTS_BIN} \ - -rh localhost -rp 8080\ - -tp ${HOME}/.herontests/data/python \ - -cl local -rl heron-staging -ev devel -end_timer "$T" - -print_timer_summary diff --git a/scripts/applatix/test.sh b/scripts/applatix/test.sh deleted file mode 100755 index a1a3313593f..00000000000 --- a/scripts/applatix/test.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Script to kick off the travis CI integration test. Fail-fast if any of tthe below commands fail. -# -set -e - -DIR=`dirname $0` -UTILS=${DIR}/../shutils -source ${UTILS}/common.sh - -# include HOME directory bin in PATH for heron cli, tools and tests -export PATH=${HOME}/bin:$PATH - -# integration test binaries have to be specified as absolute path -JAVA_INTEGRATION_TESTS_BIN="${HOME}/.herontests/lib/integration-tests.jar" -PYTHON_INTEGRATION_TESTS_BIN="${HOME}/.herontests/lib/heron_integ_topology.pex" -SCALA_INTEGRATION_TESTS_BIN="${HOME}/.herontests/lib/scala-integration-tests.jar" - -# install clients and tools -T="heron clients/tools install" -start_timer "$T" -${UTILS}/save-logs.py "heron_install.txt" ./heron-install.sh --user -end_timer "$T" - -# install tests -T="heron tests install" -start_timer "$T" -${UTILS}/save-logs.py "heron_tests_install.txt" ./heron-tests-install.sh --user -end_timer "$T" - -# initialize http-server for integration tests -T="heron integration_test http-server initialization" -start_timer "$T" -${HOME}/bin/http-server 8080 & -http_server_id=$! -trap "kill -9 $http_server_id" SIGINT SIGTERM EXIT -end_timer "$T" - -# run the scala integration test -T="heron integration_test scala" -start_timer "$T" -${HOME}/bin/test-runner \ - -hc heron -tb ${SCALA_INTEGRATION_TESTS_BIN} \ - -rh localhost -rp 8080\ - -tp ${HOME}/.herontests/data/scala \ - -cl local -rl heron-staging -ev devel -end_timer "$T" - -# run the java integration test -T="heron integration_test java" -start_timer "$T" -${HOME}/bin/test-runner \ - -hc heron -tb ${JAVA_INTEGRATION_TESTS_BIN} \ - -rh localhost -rp 8080\ - -tp ${HOME}/.herontests/data/java \ - -cl local -rl heron-staging -ev devel -end_timer "$T" - -# run the python integration test -T="heron integration_test python" -start_timer "$T" -${HOME}/bin/test-runner \ - -hc heron -tb ${PYTHON_INTEGRATION_TESTS_BIN} \ - -rh localhost -rp 8080\ - -tp ${HOME}/.herontests/data/python \ - -cl local -rl heron-staging -ev devel -end_timer "$T" - -print_timer_summary diff --git a/scripts/applatix/testutils.sh b/scripts/applatix/testutils.sh deleted file mode 100755 index aa425575410..00000000000 --- a/scripts/applatix/testutils.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Script to kick off the travis CI integration test. Fail-fast if any of tthe below commands fail. -# -set -e - -DIR=`dirname $0` -UTILS=${DIR}/../shutils -source ${UTILS}/common.sh - -# include HOME directory bin in PATH for heron cli, tools and tests -export PATH=${HOME}/bin:$PATH - -# install clients and tools -T="heron clients/tools install" -start_timer "$T" -${UTILS}/save-logs.py "heron_install.txt" ./heron-install.sh --user -end_timer "$T" - -# install tests -T="heron tests install" -start_timer "$T" -${UTILS}/save-logs.py "heron_tests_install.txt" ./heron-tests-install.sh --user -end_timer "$T" - -print_timer_summary From 6fd5e2f75945b37c2d054c376def1bb330cce242 Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Sun, 27 Feb 2022 15:32:25 -0500 Subject: [PATCH 07/18] Remove function not used anywhere --- scripts/shutils/common.sh | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/scripts/shutils/common.sh b/scripts/shutils/common.sh index 3888bc3433e..1b55aee279c 100755 --- a/scripts/shutils/common.sh +++ b/scripts/shutils/common.sh @@ -90,19 +90,6 @@ function print_timer_summary { done } -# Check the ci environment is valid -function ci_environ { - environ=$1 - if [[ $environ =~ travis ]]; then - echo "travis" - elif [[ $environ =~ applatix ]]; then - echo "applatix" - else - echo "$environ ci not supported" - exit 1 - fi -} - function pathadd { if [ -d "$1" ] && [[ ":$PATH:" != *":$1:"* ]]; then PATH="${PATH:+"$PATH:"}$1" From 626c777c43a83d52630e609181fa36624d7ab194 Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Sun, 27 Feb 2022 15:33:06 -0500 Subject: [PATCH 08/18] Another travis script cleanup --- scripts/travis/build.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/travis/build.sh b/scripts/travis/build.sh index 1cb0c9dc3fe..90ab1d6dfa6 100755 --- a/scripts/travis/build.sh +++ b/scripts/travis/build.sh @@ -64,9 +64,6 @@ set +x # as -workspace_status_command we don't get good output ./scripts/release/status.sh -# append the bazel default bazelrc to travis/bazel.rc -# for using rules provided by bazel -# cat ~/.bazelrc >> tools/travis/bazel.rc ./bazel_configure.py # build heron From 27db9498a940ab94cb2f462a7b63a1f6453697ef Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Sun, 27 Feb 2022 15:34:26 -0500 Subject: [PATCH 09/18] Remove outdated Docker bazel.rc --- docker/scripts/compile-docker.sh | 5 --- docker/scripts/dev-env-create.sh | 1 - docker/scripts/test-docker.sh | 5 --- tools/docker/bazel.rc | 36 ---------------------- website2/website/scripts/compile-docker.sh | 5 --- website2/website/scripts/javadocs.sh | 7 +---- 6 files changed, 1 insertion(+), 58 deletions(-) delete mode 100644 tools/docker/bazel.rc diff --git a/docker/scripts/compile-docker.sh b/docker/scripts/compile-docker.sh index 82e18a646b7..e31a89c21ca 100755 --- a/docker/scripts/compile-docker.sh +++ b/docker/scripts/compile-docker.sh @@ -36,13 +36,8 @@ dockerfile_path_for_platform() { echo "$SCRATCH_DIR/compile/Dockerfile.$1" } -copy_bazel_rc_to() { - cp $PROJECT_DIR/tools/docker/bazel.rc $1 -} - DOCKER_FILE=$(dockerfile_path_for_platform $TARGET_PLATFORM) verify_dockerfile_exists $DOCKER_FILE -copy_bazel_rc_to $SCRATCH_DIR/bazelrc echo "Building heron-compiler container" docker buildx build -t heron-compiler:$TARGET_PLATFORM -f $DOCKER_FILE $SCRATCH_DIR diff --git a/docker/scripts/dev-env-create.sh b/docker/scripts/dev-env-create.sh index e26b5d5d170..e288043420b 100755 --- a/docker/scripts/dev-env-create.sh +++ b/docker/scripts/dev-env-create.sh @@ -69,7 +69,6 @@ dockerfile_path_for_platform() { copy_extra_files() { mkdir -p $SCRATCH_DIR/scripts - cp $PROJECT_DIR/tools/docker/bazel.rc $SCRATCH_DIR/bazelrc cp $DOCKER_DIR/scripts/compile-docker.sh $SCRATCH_DIR/scripts/compile-platform.sh } diff --git a/docker/scripts/test-docker.sh b/docker/scripts/test-docker.sh index f816e2f2257..73e472cb7ce 100755 --- a/docker/scripts/test-docker.sh +++ b/docker/scripts/test-docker.sh @@ -36,13 +36,8 @@ dockerfile_path_for_platform() { echo "$SCRATCH_DIR/test/Dockerfile.$1" } -copy_bazel_rc_to() { - cp $PROJECT_DIR/tools/docker/bazel.rc $1 -} - DOCKER_FILE=$(dockerfile_path_for_platform $TARGET_PLATFORM) verify_dockerfile_exists $DOCKER_FILE -copy_bazel_rc_to $SCRATCH_DIR/bazelrc echo "Building heron-compiler container" docker buildx build -t heron-compiler:$TARGET_PLATFORM -f $DOCKER_FILE $SCRATCH_DIR diff --git a/tools/docker/bazel.rc b/tools/docker/bazel.rc deleted file mode 100644 index ef31f17d34e..00000000000 --- a/tools/docker/bazel.rc +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# This is so we understand failures better -build --verbose_failures -build --host_force_python=PY3 - -# This is so we don't use sandboxed execution. Sandboxed execution -# runs stuff in a container, and since Travis already runs its script -# in a container (unless you require sudo in your .travis.yml) this -# fails to run tests. -build --spawn_strategy=standalone --genrule_strategy=standalone -test --test_strategy=standalone - -# Workaround https://github.com/bazelbuild/bazel/issues/3645 -# Bazel doesn't calculate the memory ceiling correctly when running under Docker. -# Limit Bazel to consuming 4G ram and 2 cores. -build --local_ram_resources=4096 -# build --local_cpu_resources=2 - -# Echo all the configuration settings and their source - build --announce_rc diff --git a/website2/website/scripts/compile-docker.sh b/website2/website/scripts/compile-docker.sh index b4ba2e3f007..80a008f1f4b 100755 --- a/website2/website/scripts/compile-docker.sh +++ b/website2/website/scripts/compile-docker.sh @@ -38,14 +38,9 @@ dockerfile_path_for_platform() { echo "$PROJECT_DIR/website/scripts/Dockerfile.$1" } -copy_bazel_rc_to() { - cp $PROJECT_DIR/../tools/docker/bazel.rc $1 -} - TARGET_PLATFORM="ubuntu18.04" DOCKER_FILE=$(dockerfile_path_for_platform $TARGET_PLATFORM) verify_dockerfile_exists $DOCKER_FILE -copy_bazel_rc_to $PROJECT_DIR/website/scripts/bazelrc echo "docker file" echo $DOCKER_FILE diff --git a/website2/website/scripts/javadocs.sh b/website2/website/scripts/javadocs.sh index d9c24817725..3910a92160b 100755 --- a/website2/website/scripts/javadocs.sh +++ b/website2/website/scripts/javadocs.sh @@ -27,12 +27,7 @@ GEN_PROTO_DIR=$HERON_ROOT_DIR/bazel-bin/heron/proto/_javac # The path of the customized landing page for the Javadocs OVERVIEW_HTML_FILE=$HERON_ROOT_DIR/website2/website/scripts/javadocs-overview.html -# Check if this script is run with Travis flag -if [ $# -eq 1 ] && [ $1 == "--travis" ]; then - BAZEL_CMD="bazel --bazelrc=$HERON_ROOT_DIR/tools/travis/bazel.rc build" -else - BAZEL_CMD="bazel build" -fi +BAZEL_CMD="bazel build" (cd $HERON_ROOT_DIR && $BAZEL_CMD \ `bazel query 'kind("java_library", "heron/...")'` \ From e733f439a683b8469284961675de582d637c06bc Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Sun, 27 Feb 2022 15:35:52 -0500 Subject: [PATCH 10/18] Removed Mesos/Marathon from Vagrantfile init.sh They were outdatted and commented out anyway. --- vagrant/init.sh | 62 ------------------------------------------------- 1 file changed, 62 deletions(-) diff --git a/vagrant/init.sh b/vagrant/init.sh index 5753c0ef8d4..f9a44dc9ae0 100644 --- a/vagrant/init.sh +++ b/vagrant/init.sh @@ -16,34 +16,6 @@ set -o errexit -o nounset -o pipefail # See the License for the specific language governing permissions and # limitations under the License. -# NB: Apache Mesos requires the use of "master"/"slave" -install_mesos() { - mode=$1 # master | slave - apt-get -qy install mesos=0.25.0* - - echo "zk://primary:2181/mesos" > /etc/mesos/zk - echo '5mins' > /etc/mesos-slave/executor_registration_timeout - - ip=$(cat /etc/hosts | grep `hostname` | grep -E -o "([0-9]{1,3}[\.]){3}[0-9]{1,3}") - echo $ip > "/etc/mesos-$mode/ip" - - if [ "$mode" == "master" ]; then - ln -s /lib/init/upstart-job /etc/init.d/mesos-master - service mesos-master start - else - apt-get -qy remove zookeeper - fi - - ln -s /lib/init/upstart-job /etc/init.d/mesos-slave - echo 'docker,mesos' > /etc/mesos-slave/containerizers - service mesos-slave start -} - -install_marathon() { - apt-get install -qy marathon=0.10.0* - service marathon start -} - bazelVersion=4.2.2 bazel_install() { apt-get install -y automake cmake gcc g++ zlib1g-dev zip pkg-config wget libssl-dev libunwind-dev @@ -55,14 +27,6 @@ bazel_install() { popd } -build_heron() { - pushd /vagrant - bazel clean - ./bazel_configure.py - bazel --bazelrc=tools/travis/bazel.rc build heron/... - popd -} - if [[ "$1" != "master" && $1 != "slave" ]]; then echo "Usage: $0 master|slave" exit 1 @@ -74,32 +38,10 @@ cd /vagrant/vagrant # name resolution cp .vagrant/hosts /etc/hosts -# XXX: not needed? -# ssh key -key=".vagrant/ssh_key.pub" -if [ -f "$key" ]; then - cat $key >> /home/vagrant/.ssh/authorized_keys -fi - # disable ipv6 echo -e "\nnet.ipv6.conf.all.disable_ipv6 = 1\n" >> /etc/sysctl.conf sysctl -p -# use apt-proxy if present -if [ -f ".vagrant/apt-proxy" ]; then - apt_proxy=$(cat ".vagrant/apt-proxy") - echo "Using apt-proxy: $apt_proxy"; - echo "Acquire::http::Proxy \"$apt_proxy\";" > /etc/apt/apt.conf.d/90-apt-proxy.conf -fi - -:<<'REMOVED' -# add mesosphere repo -apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E56151BF -DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') -CODENAME=$(lsb_release -cs) -echo "deb http://repos.mesosphere.io/${DISTRO} cosmic main" | tee /etc/apt/sources.list.d/mesosphere.list -REMOVED - # install docker repo apt-get install -qy ca-certificates curl gnupg lsb-release curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg @@ -117,10 +59,6 @@ apt-get install -qy ant vim zip mc curl wget openjdk-11-jdk scala git python3-se apt-get install -qy docker-ce docker-ce-cli containerd.io usermod -aG docker vagrant -# install_mesos $mode if [ $mode == "master" ]; then - # install_marathon bazel_install - # switch to non-root so bazel cache can be reused when SSHing in - # su --login vagrant /vagrant/scripts/travis/ci.sh fi From 0c8aea233e1377ddcd058b14f88d94fb761bb475 Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Sun, 27 Feb 2022 16:22:27 -0500 Subject: [PATCH 11/18] More cleanup and documentation update --- examples/src/scala/compile.sh | 2 +- integration_test/README.md | 6 +-- .../src/python/local_test_runner/README | 4 +- scripts/release_check/build.sh | 4 +- .../bazel/checkstyle/CppCheckstyle.java | 2 +- .../bazel/checkstyle/JavaCheckstyle.java | 2 +- .../bazel/checkstyle/PythonCheckstyle.java | 2 +- website2/docs/compiling-linux.md | 40 ++++++++++++++----- website2/docs/compiling-osx.md | 27 +++++++++---- website2/docs/compiling-overview.md | 31 +++++--------- website2/docs/compiling-running-tests.md | 6 +-- 11 files changed, 76 insertions(+), 50 deletions(-) diff --git a/examples/src/scala/compile.sh b/examples/src/scala/compile.sh index cf220f34443..d55ef4802c3 100755 --- a/examples/src/scala/compile.sh +++ b/examples/src/scala/compile.sh @@ -32,7 +32,7 @@ function die { which scalac || die "scalac must be installed to run this script. Exiting." rm -f heron-storm.jar -(cd $root && bazel build --config=darwin scripts/packages:tarpkgs) +(cd $root && bazel build scripts/packages:tarpkgs) # Verify storm and heron bolts compile with heron-storm.jar scalac -cp bazel-bin/./storm-compatibility/v0.10.2/src/java/heron-storm.jar \ diff --git a/integration_test/README.md b/integration_test/README.md index ae235300956..4b397e1f171 100644 --- a/integration_test/README.md +++ b/integration_test/README.md @@ -46,9 +46,9 @@ To run integration tests on your mac first run the following to build the test p the heron client: ```Bash - bazel build --config=darwin integration_test/src/... - bazel run --config=darwin -- scripts/packages:heron-client-install.sh --user - bazel run --config=darwin -- scripts/packages:heron-tools-install.sh --user + bazel build integration_test/src/... + bazel run -- scripts/packages:heron-client-install.sh --user + bazel run -- scripts/packages:heron-tools-install.sh --user ``` To run the local integration tests on your mac run the following from the heron repo's top dir: diff --git a/integration_test/src/python/local_test_runner/README b/integration_test/src/python/local_test_runner/README index dbf122872dc..d918f7fa1f0 100644 --- a/integration_test/src/python/local_test_runner/README +++ b/integration_test/src/python/local_test_runner/README @@ -4,6 +4,6 @@ tuple to an output file and the two files are compared. To run the tests: - bazel run --config=darwin -- scripts/packages:heron-install.sh --user - bazel build --config=darwin integration_test/src/... + bazel run -- scripts/packages:heron-install.sh --user + bazel build integration_test/src/... ./bazel-bin/integration_test/src/python/local_test_runner/local-test-runner diff --git a/scripts/release_check/build.sh b/scripts/release_check/build.sh index fbb73192a48..e1924850c93 100644 --- a/scripts/release_check/build.sh +++ b/scripts/release_check/build.sh @@ -18,5 +18,5 @@ set -o errexit -bazel build --compilation_mode=dbg --config=darwin heron/... --verbose_failures -bazel build --compilation_mode=dbg --config=darwin scripts/packages:binpkgs --verbose_failures +bazel build -c dbg heron/... --verbose_failures +bazel build -c dbg scripts/packages:binpkgs --verbose_failures diff --git a/tools/java/src/org/apache/bazel/checkstyle/CppCheckstyle.java b/tools/java/src/org/apache/bazel/checkstyle/CppCheckstyle.java index 2166f2469b1..885c898b919 100644 --- a/tools/java/src/org/apache/bazel/checkstyle/CppCheckstyle.java +++ b/tools/java/src/org/apache/bazel/checkstyle/CppCheckstyle.java @@ -43,7 +43,7 @@ * Usage: java org.apache.bazel.checkstyle.CppCheckstyle -f <extra_action_file> -c <cpplint_file> *

* To test: - * $ bazel build --config=darwin --experimental_action_listener=tools/cpp:compile_cpp heron/stmgr/src/cpp:grouping-cxx + * $ bazel build --experimental_action_listener=tools/cpp:compile_cpp heron/stmgr/src/cpp:grouping-cxx */ public final class CppCheckstyle { public static final Logger LOG = Logger.getLogger(CppCheckstyle.class.getName()); diff --git a/tools/java/src/org/apache/bazel/checkstyle/JavaCheckstyle.java b/tools/java/src/org/apache/bazel/checkstyle/JavaCheckstyle.java index 6abcf9b519a..d55e8934960 100644 --- a/tools/java/src/org/apache/bazel/checkstyle/JavaCheckstyle.java +++ b/tools/java/src/org/apache/bazel/checkstyle/JavaCheckstyle.java @@ -44,7 +44,7 @@ * Usage: java org.apache.bazel.checkstyle.JavaCheckstyle -f <extra_action_file> -c <checkstyle_config> *

* To test: - * $ bazel build --config=darwin heron/spi/src/java:heron-spi --experimental_action_listener=tools/java:compile_java + * $ bazel build heron/spi/src/java:heron-spi --experimental_action_listener=tools/java:compile_java */ public final class JavaCheckstyle { public static final Logger LOG = Logger.getLogger(JavaCheckstyle.class.getName()); diff --git a/tools/java/src/org/apache/bazel/checkstyle/PythonCheckstyle.java b/tools/java/src/org/apache/bazel/checkstyle/PythonCheckstyle.java index d85609faa98..f98b9e31c2f 100644 --- a/tools/java/src/org/apache/bazel/checkstyle/PythonCheckstyle.java +++ b/tools/java/src/org/apache/bazel/checkstyle/PythonCheckstyle.java @@ -43,7 +43,7 @@ * Usage: java org.apache.bazel.checkstyle.PythonCheckstyle -f <extra_action_file> -p <pylint_file> *

* To test: - * $ bazel build --config=darwin --experimental_action_listener=tools/python:compile_python heron/cli/src/python/... + * $ bazel build --experimental_action_listener=tools/python:compile_python heron/cli/src/python/... */ public final class PythonCheckstyle { public static final Logger LOG = Logger.getLogger(PythonCheckstyle.class.getName()); diff --git a/website2/docs/compiling-linux.md b/website2/docs/compiling-linux.md index 24fbf3bf488..b2af4b964b1 100644 --- a/website2/docs/compiling-linux.md +++ b/website2/docs/compiling-linux.md @@ -182,26 +182,48 @@ $ sudo yum install java-11-openjdk java-11-openjdk-devel $ export JAVA_HOME=/usr/lib/jvm/java-11-openjdk ``` -#### Step 5 - Install Bazel {{% bazelVersion %}} +#### Step 5 - Install Bazelisk + +Bazelisk helps automate the management of Bazel versions ```bash -wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/0.26.0/bazel-0.26.0-installer-linux-x86_64.sh -chmod +x /tmp/bazel.sh -/tmp/bazel.sh --user +wget -O /tmp/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-darwin-amd64 +chmod +x /tmp/bazelisk +sudo mv /tmp/bazelisk /usr/local/bin/bazel +``` + +### Step 6 --- Fetch the latest version of Heron's source code + +```bash +$ git clone https://github.com/apache/incubator-heron.git && cd incubator-heron ``` -Make sure to download the appropriate version of Bazel (currently {{% -bazelVersion %}}). -### Step 6 --- Download Heron and compile it +### Step 7 --- Configure Heron for building with Bazel ```bash -$ git clone https://github.com/apache/incubator-heron.git && cd heron $ ./bazel_configure.py +``` + +### Step 8 --- Build the project + +```bash $ bazel build heron/... ``` -### Step 7 --- Build the binary packages +This will build in the Bazel default `fastbuild` mode. Production release packages include additional performance optimizations not enabled by default. To enable production optimizations, include the `opt` flag. This defaults to optimization level `-O2`. The second option overrides the setting to bump it to `-CO3`. + +```bash +$ bazel build -c opt heron/... +``` + +```bash +$ bazel build -c opt --copt=-O3 heron/... +``` + +If you wish to add the code syntax style check, add `--config=stylecheck`. + +### Step 9 --- Build the binary packages ```bash $ bazel build scripts/packages:binpkgs diff --git a/website2/docs/compiling-osx.md b/website2/docs/compiling-osx.md index 251464c63cf..cdc56e9335a 100644 --- a/website2/docs/compiling-osx.md +++ b/website2/docs/compiling-osx.md @@ -32,11 +32,12 @@ install it using this one-liner: $ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" ``` -### Step 2 -- Install Bazel +### Step 2 -- Install Bazelisk + +Bazelisk helps automate the management of Bazel versions + ```bash -wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/4.2.2/bazel-4.2.2-installer-darwin-x86_64.sh -chmod +x /tmp/bazel.sh -/tmp/bazel.sh --user +brew install bazelisk ``` ### Step 2 --- Install other required libraries @@ -75,14 +76,26 @@ to install those dependencies. ### Step 6 --- Build the project ```bash -$ bazel build --config=darwin heron/... +$ bazel build heron/... +``` + +This will build in the Bazel default `fastbuild` mode. Production release packages include additional performance optimizations not enabled by default. To enable production optimizations, include the `opt` flag. This defaults to optimization level `-O2`. The second option overrides the setting to bump it to `-CO3`. + +```bash +$ bazel build -c opt heron/... ``` +```bash +$ bazel build -c opt --copt=-O3 heron/... +``` + +If you wish to add the code syntax style check, add `--config=stylecheck`. + ### Step 7 --- Build the packages ```bash -$ bazel build --config=darwin scripts/packages:binpkgs -$ bazel build --config=darwin scripts/packages:tarpkgs +$ bazel build scripts/packages:binpkgs +$ bazel build scripts/packages:tarpkgs ``` This will install Heron packages in the `bazel-bin/scripts/packages/` directory. diff --git a/website2/docs/compiling-overview.md b/website2/docs/compiling-overview.md index 619561434eb..51a3a9e7502 100644 --- a/website2/docs/compiling-overview.md +++ b/website2/docs/compiling-overview.md @@ -86,31 +86,22 @@ $ ./bazel_configure.py ## Building -### Bazel OS Environments - -Bazel builds are specific to a given OS. When building you must specify an -OS-specific configuration using the `--config` flag. The following OS values -are supported: - -* `darwin` (Mac OS X) -* `ubuntu` (Ubuntu 20.04) -* `debian` (Debian10) -* `rocky8` (Rocky 8) +```bash +$ bazel build heron/... +``` -For example, on Mac OS X (`darwin`), the following command will build all -packages: +This will build in the Bazel default `fastbuild` mode. Production release packages include additional performance optimizations not enabled by default. To enable production optimizations, include the `opt` flag. This defaults to optimization level `-O2`. The second option overrides the setting to bump it to `-CO3`. ```bash -$ bazel build --config=darwin heron/... +$ bazel build -c opt heron/... ``` -Production release packages include additional performance optimizations -not enabled by default. Enabling these optimizations increases build time. -To enable production optimizations, include the `opt` flag: ```bash -$ bazel build -c opt --config=PLATFORM heron/... +$ bazel build -c opt --copt=-O3 heron/... ``` +If you wish to add the code syntax style check, add `--config=stylecheck`. + ### Building All Components The Bazel build process can produce either executable install scripts or @@ -118,8 +109,8 @@ bundled tars. To build executables or tars for all Heron components at once, use the following `bazel build` commands, respectively: ```bash -$ bazel build --config=PLATFORM scripts/packages:binpkgs -$ bazel build --config=PLATFORM scripts/packages:tarpkgs +$ bazel build scripts/packages:binpkgs +$ bazel build scripts/packages:tarpkgs ``` Resulting artifacts can be found in subdirectories below the `bazel-bin` @@ -134,7 +125,7 @@ Tracker](user-manuals-heron-tracker-runbook)) by passing a target to the `bazel build` command. For example, the following command would build the Heron Tracker: ```bash -$ bazel build --config=darwin heron/tools/tracker/src/python:heron-tracker +$ bazel build heron/tools/tracker/src/python:heron-tracker ``` ## Testing Heron diff --git a/website2/docs/compiling-running-tests.md b/website2/docs/compiling-running-tests.md index 8f18b3d23a6..bd1aba66010 100644 --- a/website2/docs/compiling-running-tests.md +++ b/website2/docs/compiling-running-tests.md @@ -29,14 +29,14 @@ as described in [Compiling Heron](compiling-overview). The following command will run all tests: ```bash -$ bazel test --config=darwin heron/... +$ bazel test heron/... ``` To run a specific [test target](http://bazel.io/docs/test-encyclopedia.html), pass the test target name. ```bash -$ bazel test --config=darwin heron/statemgrs/tests/java:localfs-statemgr_unittest +$ bazel test heron/statemgrs/tests/java:localfs-statemgr_unittest ``` ### Discovering Unit Test Targets @@ -86,6 +86,6 @@ Integration tests are divided into two categories: To run the failure integration tests on a Mac OS X, do the following: ```bash - $ bazel build --config=darwin integration_test/src/... + $ bazel build integration_test/src/... $ ./bazel-bin/integration_test/src/python/local_test_runner/local-test-runner ``` From ac43676e40f346d4cadc01829f054f88be0162f2 Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Mon, 28 Feb 2022 04:25:43 -0500 Subject: [PATCH 12/18] travis build.sh cleanup --- scripts/travis/build.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/travis/build.sh b/scripts/travis/build.sh index 90ab1d6dfa6..7a10504995c 100755 --- a/scripts/travis/build.sh +++ b/scripts/travis/build.sh @@ -78,10 +78,10 @@ end_timer "$T" # run heron unit tests T="heron test non-flaky" start_timer "$T" -${UTILS}/save-logs.py "heron_test_non_flaky.txt" bazel\ - test\ - --test_summary=detailed --test_output=errors\ - --config=stylecheck --test_tag_filters=-flaky heron/... \ +${UTILS}/save-logs.py "heron_test_non_flaky.txt" bazel \ + test --config=stylecheck \ + --test_summary=detailed --test_output=errors \ + --test_tag_filters=-flaky heron/... \ heronpy/... examples/... storm-compatibility-examples/v0.10.2/... \ eco-storm-examples/... eco-heron-examples/... contrib/... end_timer "$T" @@ -90,10 +90,10 @@ end_timer "$T" # which should be fixed. For now, run them serially T="heron test flaky" start_timer "$T" -${UTILS}/save-logs.py "heron_test_flaky.txt" bazel\ - test\ - --test_summary=detailed --test_output=errors\ - --config=stylecheck --test_tag_filters=flaky --jobs=1 heron/... \ +${UTILS}/save-logs.py "heron_test_flaky.txt" bazel \ + test --config=stylecheck \ + --test_summary=detailed --test_output=errors \ + --test_tag_filters=flaky --jobs=1 heron/... \ heronpy/... examples/... storm-compatibility-examples/v0.10.2/... \ eco-storm-examples/... eco-heron-examples/... end_timer "$T" From ea0f7443bede8ea5f46672e02eb84fe5990951c2 Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Sun, 6 Mar 2022 09:05:01 -0500 Subject: [PATCH 13/18] Some Dockerfile cleanup --- docker/compile/Dockerfile.rocky8 | 4 ++-- docker/dist/Dockerfile.dist.rocky8 | 2 +- docker/test/Dockerfile.rocky8 | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/compile/Dockerfile.rocky8 b/docker/compile/Dockerfile.rocky8 index a6724e91f34..e55a2f144dc 100644 --- a/docker/compile/Dockerfile.rocky8 +++ b/docker/compile/Dockerfile.rocky8 @@ -17,8 +17,8 @@ FROM rockylinux:8.5 -RUN yum -y upgrade -RUN yum -y install \ +RUN yum -y upgrade \ + && yum -y install \ ant \ automake \ curl \ diff --git a/docker/dist/Dockerfile.dist.rocky8 b/docker/dist/Dockerfile.dist.rocky8 index 3b87b635d07..a48c3c7a1a9 100644 --- a/docker/dist/Dockerfile.dist.rocky8 +++ b/docker/dist/Dockerfile.dist.rocky8 @@ -18,7 +18,7 @@ #syntax=docker/dockerfile:1.2 FROM rockylinux:8.5 -RUN yum -y install epel-release \ +RUN yum -y upgrade \ && yum -y install \ curl \ java-11-openjdk-headless \ diff --git a/docker/test/Dockerfile.rocky8 b/docker/test/Dockerfile.rocky8 index cc7386d6117..35c7135e598 100644 --- a/docker/test/Dockerfile.rocky8 +++ b/docker/test/Dockerfile.rocky8 @@ -20,8 +20,8 @@ FROM rockylinux:8.5 # This is passed to the heron build command via the --config flag ENV TARGET_PLATFORM linux -RUN yum -y upgrade -RUN yum -y install \ +RUN yum -y upgrade \ + && yum -y install \ automake \ curl \ cmake \ From 18a50342ae8725837c4cb88be10ca02b3fba30a7 Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Thu, 24 Mar 2022 01:29:40 -0400 Subject: [PATCH 14/18] Removing OS detect script --- scripts/detect_os_type.sh | 33 ------------------------ scripts/run_integration_test.sh | 1 - scripts/run_integration_topology_test.sh | 1 - 3 files changed, 35 deletions(-) delete mode 100755 scripts/detect_os_type.sh diff --git a/scripts/detect_os_type.sh b/scripts/detect_os_type.sh deleted file mode 100755 index 0ce323388fa..00000000000 --- a/scripts/detect_os_type.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -e -# Copyright 2015 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -function platform { - PLATFORM=darwin - case "$OSTYPE" in - darwin*) - PLATFORM=darwin - ;; - linux*) - PLATFORM=linux - ;; - *) - echo "WARNING: Your platform is not currently supported!" >&2 - echo "Currently supported platforms are:" >&2 - echo " darwin, ubuntu, rocky and centos" - exit 1 - ;; - esac - echo ${PLATFORM} -} diff --git a/scripts/run_integration_test.sh b/scripts/run_integration_test.sh index 5557b849904..19cb16c1fb3 100755 --- a/scripts/run_integration_test.sh +++ b/scripts/run_integration_test.sh @@ -75,7 +75,6 @@ set -e # building tar packages DIR=`dirname $0` -source ${DIR}/detect_os_type.sh bazel run -- scripts/packages:heron-install.sh --user bazel build {heron/...,scripts/packages:tarpkgs,integration_test/src/...} diff --git a/scripts/run_integration_topology_test.sh b/scripts/run_integration_topology_test.sh index 318e1ccde9c..53f32edce89 100755 --- a/scripts/run_integration_topology_test.sh +++ b/scripts/run_integration_topology_test.sh @@ -37,7 +37,6 @@ set -e # building tar packages DIR=`dirname $0` -source ${DIR}/detect_os_type.sh bazel run -- scripts/packages:heron-install.sh --user bazel build {heron/...,scripts/packages:tarpkgs,integration_test/src/...} From 8439f75b7aa1cd716a55afebad79fff8b3f18797 Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Thu, 24 Mar 2022 01:29:57 -0400 Subject: [PATCH 15/18] Fix for build issue --- WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index 73467abd961..149ab3bc8b2 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -163,7 +163,7 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") git_repository( name = "com_github_johnynek_bazel_jar_jar", commit = "171f268569384c57c19474b04aebe574d85fde0d", # Latest commit SHA as at 2019/02/13 - remote = "git://github.com/johnynek/bazel_jar_jar.git", + remote = "https://github.com/johnynek/bazel_jar_jar.git", shallow_since = "1594234634 -1000", ) From 30698540026e3a1be27df4114dc21bc5c8f7af3a Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Thu, 24 Mar 2022 22:56:13 -0400 Subject: [PATCH 16/18] Remove referenced to deleted bazelrc from the various Dockerfiles --- docker/compile/Dockerfile.centos7 | 1 - docker/compile/Dockerfile.debian10 | 1 - docker/compile/Dockerfile.rocky8 | 1 - docker/compile/Dockerfile.ubuntu18.04 | 1 - docker/compile/Dockerfile.ubuntu20.04 | 1 - docker/compile/Dockerfile.ubuntu22.04 | 1 - docker/test/Dockerfile.centos7 | 1 - docker/test/Dockerfile.ubuntu18.04 | 1 - website2/docs/compiling-docker.md | 3 +-- website2/website/scripts/Dockerfile.ubuntu18.04 | 1 - 10 files changed, 1 insertion(+), 11 deletions(-) diff --git a/docker/compile/Dockerfile.centos7 b/docker/compile/Dockerfile.centos7 index cd965523274..183a19c2d77 100644 --- a/docker/compile/Dockerfile.centos7 +++ b/docker/compile/Dockerfile.centos7 @@ -68,5 +68,4 @@ RUN /usr/bin/ln -sfT /opt/rh/rh-python38/root/usr/bin/python3 /usr/bin/python3 \ && /usr/bin/ln -sfT /opt/rh/devtoolset-8/root/bin/g++ /usr/bin/g++ \ && /usr/bin/ln -sfT /opt/rh/devtoolset-8/root/bin/cpp /usr/bin/cpp -ADD bazelrc /root/.bazelrc ADD scripts/compile-platform.sh /compile-platform.sh diff --git a/docker/compile/Dockerfile.debian10 b/docker/compile/Dockerfile.debian10 index e5086591f6a..c80fd111a5c 100644 --- a/docker/compile/Dockerfile.debian10 +++ b/docker/compile/Dockerfile.debian10 @@ -49,5 +49,4 @@ RUN wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/ && /tmp/bazel.sh \ && rm -rf /tmp/bazel.sh -ADD bazelrc /root/.bazelrc ADD scripts/compile-platform.sh /compile-platform.sh diff --git a/docker/compile/Dockerfile.rocky8 b/docker/compile/Dockerfile.rocky8 index e55a2f144dc..5a8f7c8450e 100644 --- a/docker/compile/Dockerfile.rocky8 +++ b/docker/compile/Dockerfile.rocky8 @@ -54,5 +54,4 @@ RUN wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/ && /tmp/bazel.sh \ && rm -rf /tmp/bazel.sh -ADD bazelrc /root/.bazelrc ADD scripts/compile-platform.sh /compile-platform.sh diff --git a/docker/compile/Dockerfile.ubuntu18.04 b/docker/compile/Dockerfile.ubuntu18.04 index b92cbc37d45..72bde0343fc 100644 --- a/docker/compile/Dockerfile.ubuntu18.04 +++ b/docker/compile/Dockerfile.ubuntu18.04 @@ -54,5 +54,4 @@ RUN wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/ && /tmp/bazel.sh \ && rm -rf /tmp/bazel.sh -ADD bazelrc /root/.bazelrc ADD scripts/compile-platform.sh /compile-platform.sh diff --git a/docker/compile/Dockerfile.ubuntu20.04 b/docker/compile/Dockerfile.ubuntu20.04 index 238ffa8b230..98d401c39c8 100644 --- a/docker/compile/Dockerfile.ubuntu20.04 +++ b/docker/compile/Dockerfile.ubuntu20.04 @@ -51,5 +51,4 @@ RUN wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/ && /tmp/bazel.sh \ && rm -rf /tmp/bazel.sh -ADD bazelrc /root/.bazelrc ADD scripts/compile-platform.sh /compile-platform.sh diff --git a/docker/compile/Dockerfile.ubuntu22.04 b/docker/compile/Dockerfile.ubuntu22.04 index 195c96ec603..a38e63bd712 100644 --- a/docker/compile/Dockerfile.ubuntu22.04 +++ b/docker/compile/Dockerfile.ubuntu22.04 @@ -51,5 +51,4 @@ RUN wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/ && /tmp/bazel.sh \ && rm -rf /tmp/bazel.sh -ADD bazelrc /root/.bazelrc ADD scripts/compile-platform.sh /compile-platform.sh diff --git a/docker/test/Dockerfile.centos7 b/docker/test/Dockerfile.centos7 index f9085b61ef8..a6ba1194b0f 100644 --- a/docker/test/Dockerfile.centos7 +++ b/docker/test/Dockerfile.centos7 @@ -57,5 +57,4 @@ RUN wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/ && chmod +x /tmp/bazel.sh \ && /tmp/bazel.sh -ADD bazelrc /root/.bazelrc ADD scripts/test-platform.sh /test-platform.sh diff --git a/docker/test/Dockerfile.ubuntu18.04 b/docker/test/Dockerfile.ubuntu18.04 index 98053f1fa50..0b4a714b23c 100644 --- a/docker/test/Dockerfile.ubuntu18.04 +++ b/docker/test/Dockerfile.ubuntu18.04 @@ -47,5 +47,4 @@ RUN wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/ && chmod +x /tmp/bazel.sh \ && /tmp/bazel.sh -ADD bazelrc /root/.bazelrc ADD scripts/test-platform.sh /test-platform.sh diff --git a/website2/docs/compiling-docker.md b/website2/docs/compiling-docker.md index 1abfb28a04c..5d8bd0420d3 100644 --- a/website2/docs/compiling-docker.md +++ b/website2/docs/compiling-docker.md @@ -243,9 +243,8 @@ RUN wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/ && /tmp/bazel.sh ``` -### Step 6 --- Add the `bazelrc` configuration file for Bazel and the `compile.sh` script (from the `docker` folder) that compiles Heron +### Step 6 --- Add the `compile.sh` script (from the `docker` folder) that compiles Heron ```dockerfile -ADD bazelrc /root/.bazelrc ADD compile.sh /compile.sh ``` diff --git a/website2/website/scripts/Dockerfile.ubuntu18.04 b/website2/website/scripts/Dockerfile.ubuntu18.04 index b9e091f9961..397f83fb42e 100644 --- a/website2/website/scripts/Dockerfile.ubuntu18.04 +++ b/website2/website/scripts/Dockerfile.ubuntu18.04 @@ -56,6 +56,5 @@ RUN wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/ && chmod +x /tmp/bazel.sh \ && /tmp/bazel.sh --user -ADD bazelrc /root/.bazelrc ENV PATH="/root/bin:${PATH}" From a7b7d11c3c67392e97bfd1f7e18048670abbba09 Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Fri, 25 Mar 2022 01:24:46 -0400 Subject: [PATCH 17/18] Added comment to give hint about need to upgrade Docker version for anyone trying to build Ubuntu 22.04 image --- docker/compile/Dockerfile.ubuntu22.04 | 1 + docker/dist/Dockerfile.dist.ubuntu20.04 | 1 + 2 files changed, 2 insertions(+) diff --git a/docker/compile/Dockerfile.ubuntu22.04 b/docker/compile/Dockerfile.ubuntu22.04 index a38e63bd712..6a5ec9c1c48 100644 --- a/docker/compile/Dockerfile.ubuntu22.04 +++ b/docker/compile/Dockerfile.ubuntu22.04 @@ -22,6 +22,7 @@ ARG DEBIAN_FRONTEND=noninteractive ENV LC_ALL C.UTF-8 ENV LANG C.UTF-8 +# Docker 20.10.10 or older will not work RUN apt-get update && apt-get -y install \ ant \ g++ \ diff --git a/docker/dist/Dockerfile.dist.ubuntu20.04 b/docker/dist/Dockerfile.dist.ubuntu20.04 index e04ec0165d3..6c8c0467fcb 100644 --- a/docker/dist/Dockerfile.dist.ubuntu20.04 +++ b/docker/dist/Dockerfile.dist.ubuntu20.04 @@ -23,6 +23,7 @@ ARG DEBIAN_FRONTEND=noninteractive ENV LC_ALL C.UTF-8 ENV LANG C.UTF-8 +# Docker 20.10.10 or older will not work RUN apt-get -y update \ && apt-get -y install \ curl \ From e1e53cfc2dc3e4814869dcbf76e295b0c9ddbb20 Mon Sep 17 00:00:00 2001 From: Windham Wong Date: Sat, 26 Mar 2022 01:01:30 +0800 Subject: [PATCH 18/18] Create Dockerfile.dist.ubuntu22.04 (#3800) Add back missing Dockerfile for Ubuntu22.04 --- docker/dist/Dockerfile.dist.ubuntu22.04 | 66 +++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 docker/dist/Dockerfile.dist.ubuntu22.04 diff --git a/docker/dist/Dockerfile.dist.ubuntu22.04 b/docker/dist/Dockerfile.dist.ubuntu22.04 new file mode 100644 index 00000000000..21c034f9df2 --- /dev/null +++ b/docker/dist/Dockerfile.dist.ubuntu22.04 @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +#syntax=docker/dockerfile:1.2 +FROM ubuntu:22.04 + +ARG DEBIAN_FRONTEND=noninteractive + +ENV LC_ALL C.UTF-8 +ENV LANG C.UTF-8 + +# Docker 20.10.10 or older will not work +RUN apt-get -y update \ + && apt-get -y install \ + curl \ + openjdk-11-jdk-headless \ + netcat-openbsd \ + python3 \ + python3-distutils \ + supervisor \ + unzip \ + && apt-get clean + +ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 +RUN update-ca-certificates -f + +# run Heron installer +RUN --mount=type=bind,source=artifacts,target=/tmp/heron /tmp/heron/heron-install.sh \ + && rm -f /usr/local/heron/dist/heron-core.tar.gz + +WORKDIR /heron + +RUN ln -s /usr/local/heron/dist/heron-core /heron \ + && mkdir -p /heron/heron-tools \ + && ln -s /usr/local/heron/bin /heron/heron-tools \ + && ln -s /usr/local/heron/conf /heron/heron-tools \ + && ln -s /usr/local/heron/dist /heron/heron-tools \ + && ln -s /usr/local/heron/lib /heron/heron-tools \ + && ln -s /usr/local/heron/release.yaml /heron/heron-tools \ + && ln -s /usr/local/heron/examples /heron \ + && ln -s /usr/local/heron/release.yaml /heron + +ENV HERON_HOME /heron/heron-core/ +RUN export HERON_HOME + +# install Zookeeper +ARG ZK_DIST=zookeeper-3.5.8 + +RUN --mount=type=bind,source=dist,target=/opt/heron-docker/ sh /opt/heron-docker/scripts/install-zookeeper.sh $ZK_DIST + +ADD dist/conf/zookeeper.conf /opt/zookeeper/conf/zookeeper.conf +ADD dist/conf/sandbox.conf /etc/supervisor/conf.d/