workflow-ci.yaml

name: CI

on: [push]

jobs:
  build-and-test:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        scala: [ '2.11', '2.12' ]
        spark: [ '2.4.8', '3.1.2' ]
        hadoop: [ '2.10.1', '3.3.1' ]
        include:
          - scala: '2.11'
            spark: '3.1.2'
            skip: true
    name: CI scala_${{ matrix.scala }} spark-${{ matrix.spark }} hadoop-${{ matrix.hadoop }}
    steps:
      - name: Cache Maven packages
        uses: actions/cache@v2.1.6
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2
      - uses: actions/checkout@v2
      - name: Set up JDK 8
        uses: actions/setup-java@v2.3.0
        with:
          distribution: 'zulu'
          java-version: 8
      - name: Build
        if: '!matrix.skip'
        run: |
          export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
          export MAVEN_CLI_OPTS="-B --no-transfer-progress"
          mvn $MAVEN_CLI_OPTS -P CI,scala-${{ matrix.scala }} -Dspark.version=${{ matrix.spark }} -Dhadoop.version=${{ matrix.hadoop }} clean verify site
          rm -rf ~/.m2/repository/io/github/phymbert
      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v2.0.3
        if: '!matrix.skip'
        with:
          flags: unit-tests-spark_${{ matrix.spark }}_${{ matrix.scala }}
  run-examples-spark-standalone:
    needs: build-and-test
    runs-on: ubuntu-latest
    name: Run examples Spark Standalone
    steps:
      - name: Cache Maven packages
        uses: actions/cache@v2.1.6
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2
      - uses: actions/checkout@v2
      - name: Start spark stack
        run: |
          docker-compose -f integration/docker-compose.yml up -d
      - name: Set up JDK 8
        uses: actions/setup-java@v2.3.0
        with:
          distribution: 'zulu'
          java-version: 8
      - name: Build examples jar
        run: |
          export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
          export MAVEN_CLI_OPTS="-B --no-transfer-progress"
          mvn $MAVEN_CLI_OPTS clean install -P scala-2.11 -Dspark.version=2.4.8 -Dhadoop.version=2.10.1 -DskipTests=true
      - name: Run examples
        run: |
          timeout 30 sh -c 'until nc -z $0 $1; do sleep 1; done' localhost 7077
          timeout 10 sh -c 'until nc -z $0 $1; do sleep 1; done' localhost 8080
          timeout 10 sh -c 'until nc -z $0 $1; do sleep 1; done' localhost 8081
          timeout 10 sh -c 'until nc -z $0 $1; do sleep 1; done' localhost 8082
          timeout 10 sh -c 'until nc -z $0 $1; do sleep 1; done' localhost 8083
          docker cp examples/target/spark-search-examples_*.jar spark-master:/spark-search-examples.jar
          docker exec spark-master /spark/bin/spark-submit --master spark://spark-master:7077 --deploy-mode client --class all.examples.org.apache.spark.search.rdd.SearchRDDExamples /spark-search-examples.jar
          docker exec spark-master /spark/bin/spark-submit --master spark://spark-master:7077 --deploy-mode client --class all.examples.org.apache.spark.search.rdd.SearchRDDJavaExamples /spark-search-examples.jar

  run-examples-yarn:
    needs: build-and-test
    runs-on: ubuntu-latest
    name: Run examples Yarn
    steps:
      - name: Cache Maven packages
        uses: actions/cache@v2.1.6
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2
      - uses: actions/checkout@v2
      - name: Start spark hadoop stack
        run: |
          docker-compose -f integration/docker-compose-yarn.yml up -d --build
      - name: Set up JDK 8
        uses: actions/setup-java@v2.3.0
        with:
          distribution: 'zulu'
          java-version: 8
      - name: Build examples jar
        run: |
          export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
          export MAVEN_CLI_OPTS="-B --no-transfer-progress"
          mvn $MAVEN_CLI_OPTS clean install -P scala-2.11 -Dspark.version=2.4.8 -Dhadoop.version=2.10.1 -DskipTests=true
      - name: Run examples
        run: |
          timeout 30 sh -c 'until nc -z $0 $1; do sleep 1; done' localhost 50070
          docker cp examples/target/spark-search-examples_*.jar node-master:/spark-search-examples.jar
          docker exec node-master /opt/spark/bin/spark-submit --master yarn --deploy-mode client --conf spark.hadoop.yarn.timeline-service.enabled=false --class all.examples.org.apache.spark.search.rdd.SearchRDDExamples /spark-search-examples.jar
          docker exec node-master /opt/spark/bin/spark-submit --master yarn --deploy-mode client --conf spark.hadoop.yarn.timeline-service.enabled=false --class all.examples.org.apache.spark.search.rdd.SearchRDDJavaExamples /spark-search-examples.jar