-
Notifications
You must be signed in to change notification settings - Fork 30
122 lines (105 loc) · 6.72 KB
/
build.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
name: Build & test
on: [push]
env:
BAZEL_REPO_CACHE: ~/repo-cache
BAZEL_DISK_CACHE: ~/disk-cache
BAZEL_ARGS: --repository_cache=$BAZEL_REPO_CACHE --disk_cache=$BAZEL_DISK_CACHE
HADOOP_VERSION: 2.10.1
SPARK_VERSION: 2.4.8
SFL4J_VERSION: 1.7.30
# Bump this number to invalidate the GH actions cache
cache-version: 0
jobs:
build_and_test_with_linux:
name: Build and Test with Linux Runner
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Install NixOS
uses: cachix/install-nix-action@v18
with:
nix_path: nixpkgs=./nixpkgs.nix
- name: Configure
run: |
mkdir -p ~/repo-cache ~/disk-cache
echo "build --host_platform=@rules_haskell//haskell/platforms:linux_x86_64_nixpkgs $BAZEL_ARGS" > .bazelrc.local
- name: Mount Bazel cache
uses: actions/cache/restore@v3
if: github.ref != 'refs/heads/master'
id: restore-cache
with:
path: |
~/repo-cache
~/disk-cache
key: repo-cache-${{ runner.os }}-nixpkgs-${{ env.cache-version }}-${{ github.run_id }}-${{ github.run_attempt }}
restore-keys: |
repo-cache-${{ runner.os }}-nixpkgs-${{ env.cache-version }}-
# - run: echo "build --host_platform=@rules_haskell//haskell/platforms:linux_x86_64_nixpkgs" > .bazelrc.local
- run: nix-shell --pure --run 'bazel build //apps/hello:sparkle-example-hello_deploy.jar'
- run: nix-shell --pure --run 'bazel build //apps/rdd-ops:sparkle-example-rddops_deploy.jar'
- run: nix-shell --pure --run 'bazel build //apps/dataframe:sparkle-example-dataframe_deploy.jar'
- run: nix-shell --pure --run 'bazel build //apps/osthreads:sparkle-example-osthreads_deploy.jar'
- run: nix-shell --pure --run 'bazel build //apps/lda:sparkle-example-lda_deploy.jar'
- run: nix-shell --pure --run 'bazel build //apps/argv:sparkle-example-argv_deploy.jar'
- run: nix-shell --pure --run "bazel build //apps/bench:sparkle-benchmarks_deploy.jar"
- run: nix-shell --pure --run 'bazel build //apps/hello-safe:hello-safe_deploy.jar'
- run: nix-shell --pure --run 'bazel build //apps/dataframe-safe:dataframe-safe_deploy.jar'
- run: nix-shell --pure --run "bazel build //apps/hello-deltalake:sparkle-example-hello-deltalake_deploy.jar"
- run: nix-shell --pure --run "bazel build //apps/deltalake-glow:sparkle-example-deltalake-glow_deploy.jar"
- run: nix-shell --pure --run 'bazel run spark-submit -- --packages com.amazonaws:aws-java-sdk:1.11.920,org.apache.hadoop:hadoop-aws:2.8.4 $(pwd)/bazel-bin/apps/hello/sparkle-example-hello_deploy.jar'
- run: nix-shell --pure --run 'bazel run spark-submit -- $(pwd)/bazel-bin/apps/rdd-ops/sparkle-example-rddops_deploy.jar'
- run: nix-shell --pure --run 'bazel run spark-submit -- $(pwd)/bazel-bin/apps/dataframe/sparkle-example-dataframe_deploy.jar'
- run: nix-shell --pure --run 'bazel run spark-submit -- $(pwd)/bazel-bin/apps/argv/sparkle-example-argv_deploy.jar a b +RTS -s -RTS c'
- run: nix-shell --pure --run 'bazel run spark-submit -- --packages com.amazonaws:aws-java-sdk:1.11.920,org.apache.hadoop:hadoop-aws:2.8.4 $(pwd)/bazel-bin/apps/hello/sparkle-example-hello_deploy.jar $(pwd)/bazel-bin/apps/lda/sparkle-example-lda_deploy.jar'
- run: nix-shell --pure --run "bazel run spark-submit -- --packages com.amazonaws:aws-java-sdk:1.11.920,org.apache.hadoop:hadoop-aws:2.8.4 $(pwd)/bazel-bin/apps/lda/sparkle-example-lda_deploy.jar"
- run: nix-shell --pure --run 'bazel run spark-submit -- --packages com.amazonaws:aws-java-sdk:1.11.920,org.apache.hadoop:hadoop-aws:2.8.4 $(pwd)/bazel-bin/apps/hello-safe/hello-safe_deploy.jar'
- run: nix-shell --pure --run 'bazel run spark-submit -- $(pwd)/bazel-bin/apps/dataframe-safe/dataframe-safe_deploy.jar'
- run: nix-shell --pure --run "bazel run spark-submit -- --packages io.delta:delta-core_2.11:0.4.0 $(pwd)/bazel-bin/apps/hello-deltalake/sparkle-example-hello-deltalake_deploy.jar"
- run: nix-shell --pure --run "bazel run spark-submit-with-data -- --packages io.delta:delta-core_2.11:0.4.0,io.projectglow:glow-spark2_2.11:1.1.2 $(pwd)/bazel-bin/apps/deltalake-glow/sparkle-example-deltalake-glow_deploy.jar"
- run: spark-submit bazel-bin/apps/osthreads/sparkle-example-osthreads_deploy.jar | tee out.txt || grep "Job | pool | start time (s) | end time (s)" out.txt
- uses: actions/cache/save@v3
if: github.ref == 'refs/heads/master'
with:
path: |
~/repo-cache
~/disk-cache
key: repo-cache-${{ runner.os }}-nixpkgs-${{ env.cache-version }}-${{ github.run_id }}-${{ github.run_attempt }}
build_and_test_with_macos:
name: Build and Test with MacOs Runner
runs-on: macos-11
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Install NixOS
uses: cachix/install-nix-action@v18
with:
nix_path: nixpkgs=./nixpkgs.nix
- name: Install cachix
uses: cachix/cachix-action@v10
with:
name: tweag
- name: Run cachix
run: cachix watch-store tweag &
- name: Configure
run: mkdir -p ~/repo-cache ~/disk-cache
- name: Prefetch Stackage snapshot
run: nix-shell --pure --run "cmd='bazel fetch @stackage//... $BAZEL_ARGS'; \$cmd || \$cmd || \$cmd"
- name: Build all
run: |
while true; do echo "."; sleep 60; done &
nix-shell --pure --run "bazel build //apps/hello:sparkle-example-hello_deploy.jar $BAZEL_ARGS"
nix-shell --pure --run "bazel build //apps/rdd-ops:sparkle-example-rddops_deploy.jar $BAZEL_ARGS"
nix-shell --pure --run "bazel build //apps/osthreads:sparkle-example-osthreads_deploy.jar $BAZEL_ARGS"
- name: Install Apache Spark and Hadoop
run: |
curl -OL https://repo1.maven.org/maven2/org/slf4j/slf4j-api/${SFL4J_VERSION}/slf4j-api-${SFL4J_VERSION}.jar
curl -OL https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-without-hadoop.tgz
tar -xzf spark-${SPARK_VERSION}-bin-without-hadoop.tgz
curl -OL https://archive.apache.org/dist/hadoop/core/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz
tar -xzf hadoop-${HADOOP_VERSION}.tar.gz
- name: Run tests
run: |
export SPARK_DIST_CLASSPATH=$PWD/slf4j-api-${SFL4J_VERSION}.jar:$(hadoop-${HADOOP_VERSION}/bin/hadoop classpath)
export PATH="$PWD/spark-${SPARK_VERSION}-bin-without-hadoop/bin:$PATH"
spark-submit -v --executor-cores 1 --packages com.amazonaws:aws-java-sdk:1.11.920,org.apache.hadoop:hadoop-aws:2.8.4 bazel-bin/apps/hello/sparkle-example-hello_deploy.jar