Jenkinsfile.nightly

#!groovy

/**
 * This program and the accompanying materials are made available under the terms of the
 * Eclipse Public License v2.0 which accompanies this distribution, and is available at
 * https://www.eclipse.org/legal/epl-v20.html
 *
 * SPDX-License-Identifier: EPL-2.0
 *
 * Copyright IBM Corporation 2018, 2019
 */

node('zowe-jenkins-agent-dind-wdc') {
  def lib = library("jenkins-library").org.zowe.jenkins_shared_library

  def SLACK_CHANNEL = '#zowe-build'
  def ZOWE_RELEASE_REPOSITORY = 'libs-release-local'
  def ZOWE_RELEASE_PATH = '/org/zowe/nightly'
  def ZOWE_BUILD_NAME = 'zowe-install-packaging :: staging'
  def ZOWE_BUILD_REPOSITORY = 'libs-snapshot-local'
  def ZOWE_CLI_RELEASE_PATH = '/org/zowe/nightly/cli'
  def ZOWE_CLI_BUILD_NAME = 'Zowe CLI Bundle :: master'
  def ZOWE_CLI_BUILD_REPOSITORY = 'libs-snapshot-local'
  def IS_BUILD_UNSTABLE = false

  def isStagingBranch = env && env.BRANCH_NAME == 'staging'

  def sourceRegBuildInfo
  def sourceSmpeBuildInfo
  def sourceDockerAmd64BuildInfo
  def sourceContainerizationBuildInfo
  def cliSourceBuildInfo
  def cliPluginsSourceBuildInfo

  def testRegBuildErrorUrl
  def testSmpeBuildErrorUrl
  def testDockerBuildErrorUrl
  def testMultipleNodeErrorUrl
  def testMultipleKeyringErrorUrl
  def testExternalCertificates

  def targetSmpeFullPath
  def targetRegFullPath
  def targetDockeramd64FullPath
  def cliTargetFullPath
  def cliPluginsTargetFullPath

  def pipeline = lib.pipelines.generic.GenericPipeline.new(this)

  pipeline.admins.add("jackjia", "markackert")

  // we have extra parameters for the pipeline
  pipeline.addBuildParameter(
    booleanParam(
      name: 'TEST_RUN',
      description: 'Test run will skip test cases and will not send out Slack message.',
      defaultValue: isStagingBranch ? false : true
    )
  )

  // we want to run daily on master branch
//   if (isStagingBranch) {
//     pipeline.addBuildOption(pipelineTriggers([cron("TZ=America/New_York\nH 23 * * *")]))
//   }

  // get the day of the week
  def currentDayofWeek = sh(script: 'date +"%a"', returnStdout: true).trim()
  boolean fullTestRun = false
  boolean keyringRun = false
  // only run full test suite on Saturdays
  if (currentDayofWeek.equals('Sat')) {
    fullTestRun = true
  }
  if (currentDayofWeek.equals('Wed')) {
    keyringRun = true
  }

  pipeline.setup(
    packageName: 'org.zowe',
    github: [
      email                      : lib.Constants.DEFAULT_GITHUB_ROBOT_EMAIL,
      usernamePasswordCredential : lib.Constants.DEFAULT_GITHUB_ROBOT_CREDENTIAL,
    ],
    artifactory: [
      url                        : lib.Constants.DEFAULT_LFJ_ARTIFACTORY_URL,
      usernamePasswordCredential : lib.Constants.DEFAULT_LFJ_ARTIFACTORY_ROBOT_CREDENTIAL,
    ]
  )

  pipeline.createStage(
    name          : "Pre-Check",
    isSkippable   : true,
    stage         : {
      echo "Checking last build of \"${ZOWE_BUILD_NAME}\" ..."

      // check artifactory for latest build with SMP/E and docker amd64 image
      try {
        def smpeBuild = pipeline.artifactory.getArtifact([
          'pattern'      : "${ZOWE_BUILD_REPOSITORY}/*/zowe-smpe-*.zip",
          'build-name'   : ZOWE_BUILD_NAME
        ])

        def dockerAmd64Build = pipeline.artifactory.getArtifact([
          'pattern'      : "${ZOWE_BUILD_REPOSITORY}/*/server-bundle.amd64-*.tar",
          'build-name'   : ZOWE_BUILD_NAME
        ])

        def containerizationBuild = pipeline.artifactory.getArtifact([
          'pattern'      : "${ZOWE_BUILD_REPOSITORY}/*/zowe-containerization-*.zip",
          'build-name'   : ZOWE_BUILD_NAME
        ])

        // extract build information
        def buildNumber = smpeBuild['build.number']
        def buildTimestamp = smpeBuild['build.timestamp']
        long buildTimestampInt = buildTimestamp as Long
        def currentTimestamp = sh(script: 'date +"%s"', returnStdout: true).trim()
        long currentTimestampInt = currentTimestamp as Long
        def timeElapse = (currentTimestampInt * 1000 - buildTimestampInt) / (3600 * 1000)
        echo "Build #${buildNumber} is ${timeElapse} hours ago"
        if (timeElapse < 12) { // how many hours we consider it's too old?
          echo "Will skip re-build."
          sourceSmpeBuildInfo = smpeBuild
          sourceDockerAmd64BuildInfo = dockerAmd64Build
          sourceContainerizationBuildInfo = containerizationBuild
        } else {
          echo "Will need to re-build."
        }
      } catch (es) {
        echo "getArtifact zowe-smpe-*.zip, or server-bundle.amd64-*.tar, or zowe-containerization-*.zip failed: ${es}"
        echo "Will need to re-build."
      }

      echo "Checking last CLI build of \"${ZOWE_CLI_BUILD_NAME}\" ..."

      // check artifactory for latest CLI build
      try {
        cliSourceBuildInfo = pipeline.artifactory.getArtifact([
          'pattern'      : "${ZOWE_CLI_BUILD_REPOSITORY}/org/zowe/cli/zowe-cli-package/*/zowe-cli-package-1*.zip",
          //'build-name'   : ZOWE_CLI_BUILD_NAME      // we no longer rely on jenkins to find artifacts
        ])
      } catch (e) {
        // CLI build missing, will not promote CLI
        echo "Ignored: ${e}"
      }
      try {
        cliPluginsSourceBuildInfo = pipeline.artifactory.getArtifact([
          'pattern'      : "${ZOWE_CLI_BUILD_REPOSITORY}/*/zowe-cli-plugins-1*.zip",
          //'build-name'   : ZOWE_CLI_BUILD_NAME      // we no longer rely on jenkins to find artifacts
        ])
      } catch (e) {
        // CLI build missing, will not promote CLI
        echo "Ignored: ${e}"
      }
    },
    timeout: [time: 2, unit: 'MINUTES']
  )

  pipeline.build(
    name          : "Zowe",
    timeout       : [time: 1, unit: 'HOURS'],
    isSkippable   : false,
    operation     : {
      if (!sourceSmpeBuildInfo || !sourceSmpeBuildInfo['path']) {
        echo "Building new driver ..."

        // run build
        def build_result = build(
          job: '/zowe-install-packaging/staging',
          parameters: [
            booleanParam(name: 'BUILD_SMPE', value: true),
            booleanParam(name: 'BUILD_DOCKER', value: true),
            booleanParam(name: 'BUILD_KUBERNETES', value: true),
            booleanParam(name: 'Skip Stage: Test Convenience Build', value: true),
          ],
          propagate: false
        )
        echo "Build result: ${build_result.result}"
        if (build_result.result != 'SUCCESS') {
          error "Failed to build a new Zowe driver, check failure details at ${build_result.absoluteUrl}"
        }

        // load build info
        sourceSmpeBuildInfo = pipeline.artifactory.getArtifact([
          'pattern'      : "${ZOWE_BUILD_REPOSITORY}/*/zowe-smpe-*.zip",
          'build-name'   : ZOWE_BUILD_NAME
        ])
        sourceDockerAmd64BuildInfo = pipeline.artifactory.getArtifact([
          'pattern'      : "${ZOWE_BUILD_REPOSITORY}/*/server-bundle.amd64-*.tar",
          'build-name'   : ZOWE_BUILD_NAME
        ])
        sourceContainerizationBuildInfo = pipeline.artifactory.getArtifact([
          'pattern'      : "${ZOWE_BUILD_REPOSITORY}/*/zowe-containerization-*.zip",
          'build-name'   : ZOWE_BUILD_NAME
        ])
        
      } else {
        echo "No need to build new driver."
      }
    }
  )

  pipeline.createStage(
    name          : "Load Build Info",
    isSkippable   : true,
    shouldExecute : {
      return sourceSmpeBuildInfo && sourceSmpeBuildInfo['path']
    },
    stage         : {
      sourceRegBuildInfo = pipeline.artifactory.getArtifact([
        'pattern'      : "${ZOWE_BUILD_REPOSITORY}/*/zowe-*.pax",
        'build-name'   : ZOWE_BUILD_NAME,
        'build-number' : sourceSmpeBuildInfo['build.number']
      ])
    },
    timeout: [time: 2, unit: 'MINUTES']
  )

  pipeline.test(
    name              : "Zowe docker Amd-64 image",
    shouldExecute : {
      return sourceDockerAmd64BuildInfo && sourceDockerAmd64BuildInfo['path'] && sourceRegBuildInfo && sourceRegBuildInfo['path']
    },
    operation         : {
      def docker_amd64_url = pipeline.artifactory.url + '/' + sourceDockerAmd64BuildInfo.path
      if (params.TEST_RUN) {
        echo "Skip testing with parameters:\n\n" +
             "STARTED_BY_AUTOMATION: true\n" +
             "TEST_SCOPE: test docker\n" +
             "ZOWE_ARTIFACTORY_PATTERN: '${sourceRegBuildInfo.path}'\n" +
             "ZOWE_CLI_PLUGINS_ARTIFACTORY_PATTERN: '${cliPluginsSourceBuildInfo ? cliPluginsSourceBuildInfo.path : ''}'\n" +
             "ZOWE_ARTIFACTORY_BUILD: ''\n" +
             "ZOWE_CLI_ARTIFACTORY_PATTERN: '${cliSourceBuildInfo ? cliSourceBuildInfo.path : ''}'\n" +
             "ZOWE_CLI_ARTIFACTORY_BUILD: ''\n" +
             "ZOWE_DOCKER_ARTIFACTORY_BUILD: '${docker_amd64_url}'\n" +
             "INSTALL_TEST_DEBUG_INFORMATION: 'zowe-install-test:*'\n" +
             "SANITY_TEST_DEBUG_INFORMATION: 'zowe-sanity-test:*'\n" +
             "Skip Stage: Lint: true\n" +
             "Skip Stage: Audit: true\n" +
             "Skip Stage: SonarQube Scan: true"
      } else {
        def testParameters = [
          booleanParam(name: 'STARTED_BY_AUTOMATION', value: true),
          string(name: 'TEST_SERVER', value: 'marist-4'),
          string(name: 'TEST_SCOPE', value: 'test docker'),
          string(name: 'ZOWE_ARTIFACTORY_PATTERN', value: sourceRegBuildInfo.path),
          string(name: 'ZOWE_ARTIFACTORY_BUILD', value: ''),
          string(name: 'ZOWE_DOCKER_ARTIFACTORY_BUILD', value: docker_amd64_url),
          string(name: 'CLIENT_NODE_VERSION', value: 'v12.18.3'),
          string(name: 'INSTALL_TEST_DEBUG_INFORMATION', value: 'zowe-install-test:*'),
          string(name: 'SANITY_TEST_DEBUG_INFORMATION', value: 'zowe-sanity-test:*'),
          booleanParam(name: 'Skip Stage: Lint', value: true),
          booleanParam(name: 'Skip Stage: Audit', value: true),
          booleanParam(name: 'Skip Stage: SonarQube Scan', value: true)
        ]
        if (cliSourceBuildInfo && cliSourceBuildInfo.path) {
          testParameters.add(string(name: 'ZOWE_CLI_ARTIFACTORY_PATTERN', value: cliSourceBuildInfo.path))
          testParameters.add(string(name: 'ZOWE_CLI_ARTIFACTORY_BUILD', value: ''))
        }
        // FIXME: add back after those 2 parameters are added to zowe-install-test
        // if (cliPluginsSourceBuildInfo && cliPluginsSourceBuildInfo.path) {
        //   testParameters.add(string(name: 'ZOWE_CLI_PLUGINS_ARTIFACTORY_PATTERN', value: cliPluginsSourceBuildInfo.path))
        //   testParameters.add(string(name: 'ZOWE_CLI_ARTIFACTORY_BUILD', value: ''))
        // }
        def test_result = build(
            job: '/zowe-install-test/staging',
            parameters: testParameters,
            propagate: false
          )
        echo "Test result: ${test_result.result}"
        if (test_result.result != 'SUCCESS') {
          testDockerBuildErrorUrl = test_result.absoluteUrl
          echo "Test failed on Docker Image ${sourceDockerAmd64BuildInfo.url}, check failure details at ${test_result.absoluteUrl}"
          IS_BUILD_UNSTABLE = true
        }
      }
    },
    allowMissingJunit : true,
    timeout: [time: 4, unit: 'HOURS']
  )

  pipeline.test(
    name              : "Zowe Regular Build",
    shouldExecute : {
      return sourceRegBuildInfo && sourceRegBuildInfo['path']
    },
    operation         : {
      if (params.TEST_RUN) {
        echo "Skip testing with parameters:\n\n" +
             "STARTED_BY_AUTOMATION: true\n" +
             "TEST_SCOPE: bundle: convenience build on multiple security systems\n" +
             "ZOWE_ARTIFACTORY_PATTERN: '${sourceRegBuildInfo.path}'\n" +
             "ZOWE_ARTIFACTORY_BUILD: ''\n" +
             "ZOWE_CLI_ARTIFACTORY_PATTERN: '${cliSourceBuildInfo ? cliSourceBuildInfo.path : ''}'\n" +
             "ZOWE_CLI_PLUGINS_ARTIFACTORY_PATTERN: '${cliPluginsSourceBuildInfo ? cliPluginsSourceBuildInfo.path : ''}'\n" +
             "ZOWE_CLI_ARTIFACTORY_BUILD: ''\n" +
             "INSTALL_TEST_DEBUG_INFORMATION: 'zowe-install-test:*'\n" +
             "SANITY_TEST_DEBUG_INFORMATION: 'zowe-sanity-test:*'\n" +
             "Skip Stage: Lint: true\n" +
             "Skip Stage: Audit: true\n" +
             "Skip Stage: SonarQube Scan: true"
      } else {
        def testParameters = [
          booleanParam(name: 'STARTED_BY_AUTOMATION', value: true),
          string(name: 'TEST_SCOPE', value: 'bundle: convenience build on multiple security systems'),
          string(name: 'ZOWE_ARTIFACTORY_PATTERN', value: sourceRegBuildInfo.path),
          string(name: 'ZOWE_ARTIFACTORY_BUILD', value: ''),
          string(name: 'CLIENT_NODE_VERSION', value: 'v12.18.3'),
          string(name: 'INSTALL_TEST_DEBUG_INFORMATION', value: 'zowe-install-test:*'),
          string(name: 'SANITY_TEST_DEBUG_INFORMATION', value: 'zowe-sanity-test:*'),
          booleanParam(name: 'Skip Stage: Lint', value: true),
          booleanParam(name: 'Skip Stage: Audit', value: true),
          booleanParam(name: 'Skip Stage: SonarQube Scan', value: true)
        ]
        if (cliSourceBuildInfo && cliSourceBuildInfo.path) {
          testParameters.add(string(name: 'ZOWE_CLI_ARTIFACTORY_PATTERN', value: cliSourceBuildInfo.path))
          testParameters.add(string(name: 'ZOWE_CLI_ARTIFACTORY_BUILD', value: ''))
        }
        // FIXME: add back after those 2 parameters are added to zowe-install-test
        // if (cliPluginsSourceBuildInfo && cliPluginsSourceBuildInfo.path) {
        //   testParameters.add(string(name: 'ZOWE_CLI_PLUGINS_ARTIFACTORY_PATTERN', value: cliPluginsSourceBuildInfo.path))
        //   testParameters.add(string(name: 'ZOWE_CLI_ARTIFACTORY_BUILD', value: ''))
        // }
        def test_result = build(
            job: '/zowe-install-test/staging',
            parameters: testParameters,
            propagate: false
          )
        echo "Test result: ${test_result.result}"
        if (test_result.result != 'SUCCESS') {
          testRegBuildErrorUrl = test_result.absoluteUrl
          echo "Test failed on regular build ${sourceRegBuildInfo.path}, check failure details at ${test_result.absoluteUrl}"
          IS_BUILD_UNSTABLE = true
        }
      }
    },
    allowMissingJunit : true,
    timeout: [time: 2, unit: 'HOURS']
  )

  pipeline.test(
    name              : "Zowe SMP/e Build",
    shouldExecute : {
      return sourceSmpeBuildInfo && sourceSmpeBuildInfo['path']
    },
    operation         : {
      if (params.TEST_RUN) {
        echo "Skip testing with parameters:\n\n" +
             "STARTED_BY_AUTOMATION: true\n" +
             "TEST_SCOPE: bundle: smpe ptf on multiple security systems\n" +
             "ZOWE_ARTIFACTORY_PATTERN: '${sourceSmpeBuildInfo.path}'\n" +
             "ZOWE_CLI_PLUGINS_ARTIFACTORY_PATTERN: '${cliPluginsSourceBuildInfo ? cliPluginsSourceBuildInfo.path : ''}'\n" +
             "ZOWE_ARTIFACTORY_BUILD: ''\n" +
             "ZOWE_CLI_ARTIFACTORY_PATTERN: '${cliSourceBuildInfo ? cliSourceBuildInfo.path : ''}'\n" +
             "ZOWE_CLI_ARTIFACTORY_BUILD: ''\n" +
             "INSTALL_TEST_DEBUG_INFORMATION: 'zowe-install-test:*'\n" +
             "SANITY_TEST_DEBUG_INFORMATION: 'zowe-sanity-test:*'\n" +
             "Skip Stage: Lint: true\n" +
             "Skip Stage: Audit: true\n" +
             "Skip Stage: SonarQube Scan: true"
      } else {
        def testParameters = [
          booleanParam(name: 'STARTED_BY_AUTOMATION', value: true),
          string(name: 'TEST_SCOPE', value: 'bundle: smpe ptf on multiple security systems'),
          string(name: 'ZOWE_ARTIFACTORY_PATTERN', value: sourceSmpeBuildInfo.path),
          string(name: 'ZOWE_ARTIFACTORY_BUILD', value: ''),
          string(name: 'CLIENT_NODE_VERSION', value: 'v12.18.3'),
          string(name: 'INSTALL_TEST_DEBUG_INFORMATION', value: 'zowe-install-test:*'),
          string(name: 'SANITY_TEST_DEBUG_INFORMATION', value: 'zowe-sanity-test:*'),
          booleanParam(name: 'Skip Stage: Lint', value: true),
          booleanParam(name: 'Skip Stage: Audit', value: true),
          booleanParam(name: 'Skip Stage: SonarQube Scan', value: true)
        ]
        if (cliSourceBuildInfo && cliSourceBuildInfo.path) {
          testParameters.add(string(name: 'ZOWE_CLI_ARTIFACTORY_PATTERN', value: cliSourceBuildInfo.path))
          testParameters.add(string(name: 'ZOWE_CLI_ARTIFACTORY_BUILD', value: ''))
        }
        // FIXME: add back after those 2 parameters are added to zowe-install-test
        // if (cliPluginsSourceBuildInfo && cliPluginsSourceBuildInfo.path) {
        //   testParameters.add(string(name: 'ZOWE_CLI_PLUGINS_ARTIFACTORY_PATTERN', value: cliPluginsSourceBuildInfo.path))
        //   testParameters.add(string(name: 'ZOWE_CLI_ARTIFACTORY_BUILD', value: ''))
        // }
        def test_result = build(
            job: '/zowe-install-test/staging',
            parameters: testParameters,
            propagate: false
          )
        echo "Test result: ${test_result.result}"
        if (test_result.result != 'SUCCESS') {
          testSmpeBuildErrorUrl = test_result.absoluteUrl
          echo "Test failed on SMP/e build ${sourceSmpeBuildInfo.path}, check failure details at ${test_result.absoluteUrl}"
          IS_BUILD_UNSTABLE = true
        }
      }
    },
    allowMissingJunit : true,
    timeout: [time: 4, unit: 'HOURS']
  )

  pipeline.test(
    name              : "Zowe Multiple Node Versions Build",
    shouldExecute : {
      return fullTestRun && sourceRegBuildInfo && sourceRegBuildInfo['path']
    },
    operation         : {
      if (params.TEST_RUN) {
        echo "Skip testing with parameters:\n\n" +
             "STARTED_BY_AUTOMATION: true\n" +
             "TEST_SCOPE: bundle: multiple node versions\n" +
             "ZOWE_ARTIFACTORY_PATTERN: '${sourceRegBuildInfo.path}'\n" +
             "ZOWE_ARTIFACTORY_BUILD: ''\n" +
             "ZOWE_CLI_ARTIFACTORY_PATTERN: '${cliSourceBuildInfo ? cliSourceBuildInfo.path : ''}'\n" +
             "ZOWE_CLI_PLUGINS_ARTIFACTORY_PATTERN: '${cliPluginsSourceBuildInfo ? cliPluginsSourceBuildInfo.path : ''}'\n" +
             "ZOWE_CLI_ARTIFACTORY_BUILD: ''\n" +
             "INSTALL_TEST_DEBUG_INFORMATION: 'zowe-install-test:*'\n" +
             "SANITY_TEST_DEBUG_INFORMATION: 'zowe-sanity-test:*'\n" +
             "Skip Stage: Lint: true\n" +
             "Skip Stage: Audit: true\n" +
             "Skip Stage: SonarQube Scan: true"
      } else {
        def testParameters = [
          booleanParam(name: 'STARTED_BY_AUTOMATION', value: true),
          string(name: 'TEST_SCOPE', value: 'bundle: multiple node versions'),
          string(name: 'ZOWE_ARTIFACTORY_PATTERN', value: sourceRegBuildInfo.path),
          string(name: 'ZOWE_ARTIFACTORY_BUILD', value: ''),
          string(name: 'CLIENT_NODE_VERSION', value: 'v12.18.3'),
          string(name: 'INSTALL_TEST_DEBUG_INFORMATION', value: 'zowe-install-test:*'),
          string(name: 'SANITY_TEST_DEBUG_INFORMATION', value: 'zowe-sanity-test:*'),
          booleanParam(name: 'Skip Stage: Lint', value: true),
          booleanParam(name: 'Skip Stage: Audit', value: true),
          booleanParam(name: 'Skip Stage: SonarQube Scan', value: true)
        ]
        if (cliSourceBuildInfo && cliSourceBuildInfo.path) {
          testParameters.add(string(name: 'ZOWE_CLI_ARTIFACTORY_PATTERN', value: cliSourceBuildInfo.path))
          testParameters.add(string(name: 'ZOWE_CLI_ARTIFACTORY_BUILD', value: ''))
        }

        def test_result = build(
            job: '/zowe-install-test/staging',
            parameters: testParameters,
            propagate: false
          )
        echo "Test result: ${test_result.result}"
        if (test_result.result != 'SUCCESS') {
          testMultipleNodeErrorUrl = test_result.absoluteUrl
          echo "Test failed on multiple node versions build ${sourceRegBuildInfo.path}, check failure details at ${test_result.absoluteUrl}"
          IS_BUILD_UNSTABLE = true
        }
      }
    },
    allowMissingJunit : true,
    timeout: [time: 2, unit: 'HOURS']
  )

  pipeline.test(
    name              : "Zowe Multiple Keyring Modes Build",
    shouldExecute : {
      return (keyringRun || fullTestRun) && sourceRegBuildInfo && sourceRegBuildInfo['path']
    },
    operation         : {
      if (params.TEST_RUN) {
        echo "Skip testing with parameters:\n\n" +
             "STARTED_BY_AUTOMATION: true\n" +
             "TEST_SCOPE: bundle: keyring on multiple security systems\n" +
             "ZOWE_ARTIFACTORY_PATTERN: '${sourceRegBuildInfo.path}'\n" +
             "ZOWE_ARTIFACTORY_BUILD: ''\n" +
             "ZOWE_CLI_ARTIFACTORY_PATTERN: '${cliSourceBuildInfo ? cliSourceBuildInfo.path : ''}'\n" +
             "ZOWE_CLI_PLUGINS_ARTIFACTORY_PATTERN: '${cliPluginsSourceBuildInfo ? cliPluginsSourceBuildInfo.path : ''}'\n" +
             "ZOWE_CLI_ARTIFACTORY_BUILD: ''\n" +
             "INSTALL_TEST_DEBUG_INFORMATION: 'zowe-install-test:*'\n" +
             "SANITY_TEST_DEBUG_INFORMATION: 'zowe-sanity-test:*'\n" +
             "Skip Stage: Lint: true\n" +
             "Skip Stage: Audit: true\n" +
             "Skip Stage: SonarQube Scan: true"
      } else {
        def testParameters = [
          booleanParam(name: 'STARTED_BY_AUTOMATION', value: true),
          string(name: 'TEST_SCOPE', value: 'bundle: keyring on multiple security systems'),
          string(name: 'ZOWE_ARTIFACTORY_PATTERN', value: sourceRegBuildInfo.path),
          string(name: 'ZOWE_ARTIFACTORY_BUILD', value: ''),
          string(name: 'CLIENT_NODE_VERSION', value: 'v12.18.3'),
          string(name: 'INSTALL_TEST_DEBUG_INFORMATION', value: 'zowe-install-test:*'),
          string(name: 'SANITY_TEST_DEBUG_INFORMATION', value: 'zowe-sanity-test:*'),
          booleanParam(name: 'Skip Stage: Lint', value: true),
          booleanParam(name: 'Skip Stage: Audit', value: true),
          booleanParam(name: 'Skip Stage: SonarQube Scan', value: true)
        ]
        if (cliSourceBuildInfo && cliSourceBuildInfo.path) {
          testParameters.add(string(name: 'ZOWE_CLI_ARTIFACTORY_PATTERN', value: cliSourceBuildInfo.path))
          testParameters.add(string(name: 'ZOWE_CLI_ARTIFACTORY_BUILD', value: ''))
        }

        def test_result = build(
            job: '/zowe-install-test/staging',
            parameters: testParameters,
            propagate: false
          )
        echo "Test result: ${test_result.result}"
        if (test_result.result != 'SUCCESS') {
          testMultipleKeyringErrorUrl = test_result.absoluteUrl
          echo "Test failed on multiple keyring modes build ${sourceRegBuildInfo.path}, check failure details at ${test_result.absoluteUrl}"
          IS_BUILD_UNSTABLE = true
        }
      }
    },
    allowMissingJunit : true,
    timeout: [time: 2, unit: 'HOURS']
  )

  pipeline.test(
    name              : "Zowe Using External Certificates",
    shouldExecute : {
      return fullTestRun && sourceRegBuildInfo && sourceRegBuildInfo['path']
    },
    operation         : {
      if (params.TEST_RUN) {
        echo "Skip testing with parameters:\n\n" +
             "STARTED_BY_AUTOMATION: true\n" +
             "TEST_SERVER: marist-4\n" +
             "TEST_SCOPE: external certificate\n" +
             "ZOWE_ARTIFACTORY_PATTERN: '${sourceRegBuildInfo.path}'\n" +
             "ZOWE_ARTIFACTORY_BUILD: ''\n" +
             "ZOWE_CLI_ARTIFACTORY_PATTERN: '${cliSourceBuildInfo ? cliSourceBuildInfo.path : ''}'\n" +
             "ZOWE_CLI_PLUGINS_ARTIFACTORY_PATTERN: '${cliPluginsSourceBuildInfo ? cliPluginsSourceBuildInfo.path : ''}'\n" +
             "ZOWE_CLI_ARTIFACTORY_BUILD: ''\n" +
             "INSTALL_TEST_DEBUG_INFORMATION: 'zowe-install-test:*'\n" +
             "SANITY_TEST_DEBUG_INFORMATION: 'zowe-sanity-test:*'\n" +
             "Skip Stage: Lint: true\n" +
             "Skip Stage: Audit: true\n" +
             "Skip Stage: SonarQube Scan: true"
      } else {
        def testParameters = [
          booleanParam(name: 'STARTED_BY_AUTOMATION', value: true),
          string(name: 'TEST_SERVER', value: 'marist-4'),
          string(name: 'TEST_SCOPE', value: 'external certificate'),
          string(name: 'ZOWE_ARTIFACTORY_PATTERN', value: sourceRegBuildInfo.path),
          string(name: 'ZOWE_ARTIFACTORY_BUILD', value: ''),
          string(name: 'CLIENT_NODE_VERSION', value: 'v12.18.3'),
          string(name: 'INSTALL_TEST_DEBUG_INFORMATION', value: 'zowe-install-test:*'),
          string(name: 'SANITY_TEST_DEBUG_INFORMATION', value: 'zowe-sanity-test:*'),
          booleanParam(name: 'Skip Stage: Lint', value: true),
          booleanParam(name: 'Skip Stage: Audit', value: true),
          booleanParam(name: 'Skip Stage: SonarQube Scan', value: true)
        ]
        if (cliSourceBuildInfo && cliSourceBuildInfo.path) {
          testParameters.add(string(name: 'ZOWE_CLI_ARTIFACTORY_PATTERN', value: cliSourceBuildInfo.path))
          testParameters.add(string(name: 'ZOWE_CLI_ARTIFACTORY_BUILD', value: ''))
        }

        def test_result = build(
            job: '/zowe-install-test/staging',
            parameters: testParameters,
            propagate: false
          )
        echo "Test result: ${test_result.result}"
        if (test_result.result != 'SUCCESS') {
          testExternalCertificates = test_result.absoluteUrl
          echo "Test failed on using external certificate on build ${sourceRegBuildInfo.path}, check failure details at ${test_result.absoluteUrl}"
          IS_BUILD_UNSTABLE = true
        }
      }
    },
    allowMissingJunit : true,
    timeout: [time: 1, unit: 'HOURS']
  )

  pipeline.createStage(
    name          : "Promote",
    isSkippable   : true,
    shouldExecute : {
      return sourceSmpeBuildInfo && sourceSmpeBuildInfo['path'] &&
             sourceRegBuildInfo && sourceRegBuildInfo['path'] &&
             sourceContainerizationBuildInfo && sourceContainerizationBuildInfo['path']
    },
    stage         : {
      // promote Zowe nightly build
      targetSmpeFullPath = pipeline.artifactory.promote([
        'source'     : sourceSmpeBuildInfo,
        'targetPath' : "${ZOWE_RELEASE_REPOSITORY}${ZOWE_RELEASE_PATH}/"
      ])
      targetRegFullPath = pipeline.artifactory.promote([
        'source'     : sourceRegBuildInfo,
        'targetPath' : "${ZOWE_RELEASE_REPOSITORY}${ZOWE_RELEASE_PATH}/"
      ])
      targetDockeramd64FullPath = pipeline.artifactory.promote([
        'source'     : sourceDockerAmd64BuildInfo,
        'targetPath' : "${ZOWE_RELEASE_REPOSITORY}${ZOWE_RELEASE_PATH}/"
      ])
      targetContainerizationFullPath = pipeline.artifactory.promote([
        'source'     : sourceContainerizationBuildInfo,
        'targetPath' : "${ZOWE_RELEASE_REPOSITORY}${ZOWE_RELEASE_PATH}/"
      ])

      // promote Zowe CLI nightly build
      if (cliSourceBuildInfo) {
        try {
          cliTargetFullPath = pipeline.artifactory.promote([
            'source'     : cliSourceBuildInfo,
            'targetPath' : "${ZOWE_RELEASE_REPOSITORY}${ZOWE_CLI_RELEASE_PATH}/"
          ])
        } catch (errCli) {
          echo "promote CLI core package failed: ${errCli}"
        }
      }
      if (cliPluginsSourceBuildInfo) {
        try {
          cliPluginsTargetFullPath = pipeline.artifactory.promote([
            'source'     : cliPluginsSourceBuildInfo,
            'targetPath' : "${ZOWE_RELEASE_REPOSITORY}${ZOWE_CLI_RELEASE_PATH}/"
          ])
        } catch (errCliPlugin) {
          echo "promote CLI plugins package failed: ${errCliPlugin}"
        }
      }
    },
    timeout: [time: 10, unit: 'MINUTES']
  )

  pipeline.createStage(
    name          : "Message",
    isSkippable   : true,
    stage         : {
      // update build description with build ID
      currentBuild.description = "${ZOWE_BUILD_NAME}#${sourceRegBuildInfo['build.number']}".toString()

      def slackColor = 'good'
      def message = "Zowe Build ${ZOWE_BUILD_NAME}#${sourceRegBuildInfo['build.number']} is promoted as nightly build. You can download from:\n\n" +
                    "${lib.Constants.DEFAULT_LFJ_ARTIFACTORY_URL}/${targetRegFullPath}\n\n" +
                    "${lib.Constants.DEFAULT_LFJ_ARTIFACTORY_URL}/${targetSmpeFullPath}\n\n" +
                    "${lib.Constants.DEFAULT_LFJ_ARTIFACTORY_URL}/${targetDockeramd64FullPath}\n\n" +
                    "${lib.Constants.DEFAULT_LFJ_ARTIFACTORY_URL}/${targetContainerizationFullPath}\n\n"
      if (cliSourceBuildInfo) {
        if (cliTargetFullPath) {
          message += "\n\n" +
                      "The CLI Core Package ${ZOWE_CLI_BUILD_NAME}#${cliSourceBuildInfo['build.number']} is promoted as nightly CLI build. You can download from:\n\n" +
                      "${lib.Constants.DEFAULT_LFJ_ARTIFACTORY_URL}/${cliTargetFullPath}"
        } else {
          slackColor = 'warning'
          message += "\n\n" +
                      "!!WARNING!! Failed to promote CLI Core Package"
        }
      }
      if (cliPluginsSourceBuildInfo) {
        if (cliPluginsTargetFullPath) {
          message += "\n\n" +
                      "The CLI Plugins Package ${ZOWE_CLI_BUILD_NAME}#${cliPluginsSourceBuildInfo['build.number']} is promoted as nightly CLI build. You can download from:\n\n" +
                      "${lib.Constants.DEFAULT_LFJ_ARTIFACTORY_URL}/${cliPluginsTargetFullPath}"
        } else {
          slackColor = 'warning'
          message += "\n\n" +
                      "!!WARNING!! Failed to promote CLI Plguins Package"
        }
      }
      if (testRegBuildErrorUrl) {
        slackColor = 'warning'
        message = message + "\n\n" +
                  "!!WARNING!! Smoke test FAILED on the regular build! Check test result here: " +
                  testRegBuildErrorUrl + "\n"
      }

      if (testSmpeBuildErrorUrl) {
        slackColor = 'warning'
        message = message + "\n\n" +
                  "!!WARNING!! Smoke test FAILED on the SMP/e build! Check test result here: " +
                  testSmpeBuildErrorUrl + "\n"
      }

      if (testDockerBuildErrorUrl) {
        slackColor = 'warning'
        message = message + "\n\n" +
                  "!!WARNING!! Smoke test FAILED on the docker amd-64 image! Check test result here: " +
                  testDockerBuildErrorUrl + "\n"
      }
 
      if (testMultipleNodeErrorUrl) {
        slackColor = 'warning'
        message = message + "\n\n" +
                  "!!WARNING!! Smoke test FAILED on the multiple node versions build! Check test result here: " +
                  testMultipleNodeErrorUrl + "\n"
      }

      if (testMultipleKeyringErrorUrl) {
        slackColor = 'warning'
        message = message + "\n\n" +
                  "!!WARNING!! Smoke test FAILED on the multiple keyring modes build! Check test result here: " +
                  testMultipleKeyringErrorUrl + "\n"
      }

      if (testExternalCertificates) {
        slackColor = 'warning'
        message = message + "\n\n" +
                  "!!WARNING!! Smoke test FAILED on using external certificate! Check test result here: " +
                  testExternalCertificates + "\n"
      }

      echo "*************************************************************************************************\n" +
           message + "\n" +
           "*************************************************************************************************"

      def slackMessage = "${message}\n\nCheck pipeline detail: ${env.BUILD_URL}"
      if (IS_BUILD_UNSTABLE) {
        currentBuild.result = 'UNSTABLE'
      }
      if (params.TEST_RUN) {
        echo "Slack message in ${slackColor}:\n${slackMessage}"
      } else {
        slackSend channel: SLACK_CHANNEL,
                  color: slackColor,
                  message: slackMessage
      }
    },
    timeout: [time: 2, unit: 'MINUTES']
  )

  pipeline.end()
}