packages.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
import org.ajoberstar.grgit.*
import groovy.json.JsonOutput

buildscript {
  repositories {
    mavenCentral()
  }
  dependencies {
    classpath 'org.ajoberstar:grgit:0.4.1'
    // To handle symbolic links when cloning git repos, the jgit.java7 jar
    // must be added to the classpath.
    classpath 'org.eclipse.jgit:org.eclipse.jgit.java7:3.6.2.201501210735-r'
  }
}

apply plugin: 'java'
apply plugin: 'groovy'

def PACKAGES_GROUP = 'package'

final String VERBOSE = "verbose"
final String DEPS_BUILD = "buildwithdeps"
final boolean buildDependencies = false || System.getProperty(DEPS_BUILD) == 'true'
final String BOM = System.getProperty('bomfile') ?: "$rootDir/bigtop.bom"
def final config  = new ConfigSlurper().parse(new URL("file:$BOM"))

config.bigtop.builddir = projectDir.absolutePath + "/build"
config.bigtop.outputdir = projectDir.absolutePath + "/output"
config.bigtop.distdir = projectDir.absolutePath + "/dist"
config.bigtop.dldir = projectDir.absolutePath + "/dl"
if(!config.bigtop.buildstamp) config.bigtop.buildstamp = 1

def bomVersions = []

def final BASE_DIR = projectDir.absolutePath
def final REPO_DIR = "$BASE_DIR/bigtop-repos"
def final BUILD_DIR = config.bigtop.builddir
def final OUTPUT_DIR = config.bigtop.outputdir
def final DIST_DIR = config.bigtop.distdir
def final DL_DIR = config.bigtop.dldir
def final BIGTOP_BUILD_STAMP = System.getenv('BIGTOP_BUILD_STAMP') ?: config.bigtop.buildstamp

// List of tasks derived from the BOM file is used to preserve the natural order
def bomTasks = []

// Package building and logic around it

def touchTargetFile = { fileName ->
  // to comply with make build
  GFileUtils.touch new File(fileName)
}
def ifExists = { url ->
  if (url == null) return
  URLConnection uCon = new URL(url).openConnection()
  return (uCon as HttpURLConnection).responseCode == 200
}
def safeDelete = { fileName ->
  exec {
    workingDir '.'
    commandLine 'rm', '-rf', fileName
  }
}
def getDate() {
  new Date().format('E, dd MMM yyyy HH:mm:ss Z')
}
/**
 * To avoid breaking the compat with existing packages let's use the old style names
 */
def toOldStyleName = { newname ->
  newname.toUpperCase().replaceAll("\\-", "_")
}
def setDefaults = { comp ->
  // The closure parameter is a component in the config.bigtop.components
  if (!comp.pkg) {
    comp.pkg = comp.name
  }
  if (!comp.version.pkg) {
    comp.version.pkg = comp.version.base
  }
}
def nativePackaging = {
  def devNull = new org.apache.bigtop.NullOutputStream()
  def result = exec {
    commandLine "/bin/bash", "-c",
    """dpkg-query -S /bin/sh && exit 1
       rpm -qf /bin/sh && exit 2
       exit 0
    """
    ignoreExitValue true
    errorOutput devNull
    standardOutput devNull
  }
  [false, [pkg:"deb", repo:"apt"], [pkg:"rpm", repo:"yum"]][result.getExitValue()]
}.call()

def generateDistributionsFile = { file ->
  def writer = new File(file);
  writer.text ="""Origin: Bigtop
Label: Bigtop
Suite: stable
Codename: bigtop
Version: ${config.bigtop.version}
Architectures: amd64 ppc64el source
Components: contrib
Description: Bigtop
""";
}

task "packages-help" (description: "All package build related tasks information", group: PACKAGES_GROUP) << {
  config.bigtop.components.each { label, comp ->
    println (comp.name + "\n\t[" + tasks.findAll { alltask -> alltask.name.startsWith(comp.name)}*.name.join(", ") + "]")
  }
}

task "bom-json" (description: "List the components of the stack in json format") << {
  def componentObjects = config.bigtop.components.sort().collect {
    setDefaults(it.value)
    [
      name: [
        project: it.value.name,
        pkg: it.value.pkg,
        relNotes: it.value.relNotes,
      ],
      tarball: [
        destination: it.value.tarball.destination,
        source: it.value.tarball.source,
      ],
      url: [
        site: it.value.url.site,
        archive: it.value.url.archive,
      ],
      version: [
        base: it.value.version.base,
        pkg: it.value.version.pkg,
        release: it.value.version.release,
      ],
      git: [
         repo: it.value.git.repo,
         ref: it.value.git.ref,
         dir: it.value.git.dir,
      ]
    ]
  }
  def fullDefinition = [
    version: config.bigtop.version,
    components: componentObjects
  ]
  def json = JsonOutput.toJson(fullDefinition)
  println JsonOutput.prettyPrint(json)
}

task "all-components" (description: "List the components of the stack") << {
  println "${project.name} ${config.bigtop.version} stack includes the following components"
  config.bigtop.components.sort().each { label, comp ->
    println sprintf ('\t%1$s %2$s', comp.name.padRight(20), comp.version.base.padLeft(15))
  }
}

def genTasks = { target ->
  Task t = task "${target}-download" (dependsOn: "${target}_vardefines",
      description: "Download $target artifacts",
      group: PACKAGES_GROUP) << {

    def final TARBALL_DST = config.bigtop.components[target].tarball.destination
    def final DOWNLOAD_DST = config.bigtop.components[target].downloaddst
    def final DOWNLOAD_URL = config.bigtop.components[target].downloadurl

    def final GIT_REPO = config.bigtop.components[target].git.repo
    def final GIT_REF = config.bigtop.components[target].git.ref
    def final GIT_DIR = config.bigtop.components[target].git.dir

    if (!DOWNLOAD_URL)
      return

    mkdir(DL_DIR)
    if (TARBALL_DST?.isEmpty() || new File(DOWNLOAD_DST)?.exists() || new File(config.bigtop.components[target].targetdl)?.exists()) {
      println "\tFile $DOWNLOAD_DST appears to be already downloaded. Exiting..."
      return
    }
    if (GIT_REPO && GIT_REF) {
      def dir = GIT_DIR
      if (dir == null || dir.isEmpty()) {
        dir = TARBALL_DST.substring(0, TARBALL_DST.lastIndexOf(".t"))
      }
      delete("${DL_DIR}/${dir}")
      Grgit.clone(
        uri: GIT_REPO,
        refToCheckout: GIT_REF,
        dir: new File("${DL_DIR}/${dir}")
      )
      delete("${DL_DIR}/${dir}/.git")
      exec {
        workingDir DL_DIR
        commandLine "tar -czf ${TARBALL_DST} ${dir}".split()
      }
      delete("${DL_DIR}/${dir}")
    }
    else {
      download {
        src DOWNLOAD_URL
        dest DOWNLOAD_DST
      }
    }
    touchTargetFile(config.bigtop.components[target].targetdl)
  }
  task "${target}-tar" (dependsOn: ["${target}_vardefines", "${target}-download"],
      description: "Preparing a tarball for $target artifacts",
      group: PACKAGES_GROUP) << {
    if (new File(config.bigtop.components[target].targettar)?.exists()) {
      println "\tNothing to do. Exiting..."
      return
    }
    def final TAR_DIR = config.bigtop.components[target].tardir
    def final TARBALL_SRC = config.bigtop.components[target].tarball.source ?: ""
    def final TARBALL_DST = config.bigtop.components[target].tarball.destination
    def final DOWNLOAD_DST = config.bigtop.components[target].downloaddst ?: ""
    def final SEED_TAR = config.bigtop.components[target].seedtar

    safeDelete(TAR_DIR); mkdir(TAR_DIR)

    if (TARBALL_SRC.isEmpty() || TARBALL_SRC.endsWith('.zip')) {
      if (TARBALL_SRC.isEmpty()) {
        // if no tar file needed (i.e. bigtop-utils)
        // create some contents to pack later
        copy {
          from 'LICENSE'
          into TAR_DIR
        }
      } else {
        // i.e. a ZIP file
        exec {
          workingDir TAR_DIR
          commandLine "unzip $DOWNLOAD_DST".split(' ')
        }
        def unpacked = new File(TAR_DIR)
        // check wether we have to move contents one level up
        if (unpacked.list().size() == 1) {
          def TOP_LEVEL_DIR = unpacked.list()[0]
          new File("$TAR_DIR/$TOP_LEVEL_DIR").list({ d, f ->
            (f != "." && f != "..")}).each { f ->
              new File("$TAR_DIR/$TOP_LEVEL_DIR/$f").renameTo("$TAR_DIR/$f")
          }
          safeDelete(TOP_LEVEL_DIR)
        }
      }
      // create SEED_TAR
      exec {
        workingDir "$TAR_DIR/.."
        commandLine "tar -czf $SEED_TAR ${new File("$TAR_DIR/..").list().join(' ')}".split(' ')
      }
    } else {
      println "Copy $DOWNLOAD_DST to $SEED_TAR"
      copy {
        from DOWNLOAD_DST
        into config.bigtop.builddir + "/$target/tar/"
        rename TARBALL_DST, SEED_TAR
      }
    }
    touchTargetFile(config.bigtop.components[target].targettar)
  }

  // Keeping the reference to deb task to be used later for correct sequencing
  Task tdeb = task "$target-deb"(dependsOn: "${target}-sdeb",
      description: "Building DEB for $target artifacts",
      group: PACKAGES_GROUP) << {
    if (new File(config.bigtop.components[target].targetdeb)?.exists()) {
      println "\tNothing to do. Exiting..."
      return
    }
    def final PKG_NAME = config.bigtop.components[target].pkg
    def final PKG_VERSION = config.bigtop.components[target].version.pkg
    def final PKG_OUTPUT_DIR = config.bigtop.components[target].outputdir
    def final BASE_VERSION = config.bigtop.components[target].version.base
    def final SRCDEB = "${PKG_NAME}_$PKG_VERSION-${BIGTOP_BUILD_STAMP}.dsc"
    def final HADOOP_VERSION = config.bigtop.components["hadoop"].version.pkg

    exec {
      workingDir PKG_OUTPUT_DIR
      commandLine "dpkg-source -x $SRCDEB".split(' ')
    }
// Order of debuild parameters is important; hence specifying explicitely rather
// than in an array of args
    def command = """debuild \
--preserve-envvar PATH \
--preserve-envvar MAVEN3_HOME \
--preserve-envvar MAVEN_OPTS \
--preserve-envvar JAVA_HOME \
--preserve-envvar BIGTOP_JDK \
--set-envvar=HADOOP_VERSION=$HADOOP_VERSION \
--set-envvar=${toOldStyleName(target)}_BASE_VERSION=$BASE_VERSION \
--set-envvar=${toOldStyleName(target)}_VERSION=$PKG_VERSION \
--set-envvar=${toOldStyleName(target)}_RELEASE=$BIGTOP_BUILD_STAMP \
-uc -us -b
"""
    exec {
      workingDir "$PKG_OUTPUT_DIR/$PKG_NAME-$PKG_VERSION"
      commandLine command.split(' ')
    }
    exec {
      workingDir "$PKG_OUTPUT_DIR"
      commandLine 'rm','-rf',"$PKG_NAME-$PKG_VERSION"
    }
    touchTargetFile(config.bigtop.components[target].targetdeb)
  }
  // Guarantee that tasks are ran in the order set by BOM file dependencies section
  if (buildDependencies)
    config.bigtop.dependencies.each { dependsOn, dependees ->
      if (dependees.contains(tdeb.name-"-deb")) tdeb.dependsOn "$dependsOn-deb"
    }
  // If dependencies aren't explicitly set, the default build order is defined
  // by the sequence of the components in the BOM file
  if (!config.bigtop.dependencies && bomTasks.size() > 0)
    tdeb.mustRunAfter "${bomTasks.get(bomTasks.size() - 1)}-deb".toLowerCase()
   task "$target-sdeb" (dependsOn: ["${target}_vardefines",  "${target}-tar"],
      description: "Building SDEB for $target artifacts",
      group: PACKAGES_GROUP
  ) << {
    if (new File(config.bigtop.components[target].targetsdeb)?.exists()) {
      println "\tNothing to do. Exiting..."
      return
    }
    def final TARBALL_SRC = config.bigtop.components[target].tarball.source
    def final TARBALL_DST = config.bigtop.components[target].tarball.destination ?: TARBALL_SRC
    def final PKG_BUILD_DIR = config.bigtop.components[target].builddir
    def final NAME = config.bigtop.components[target].name
    def final PKG_NAME = config.bigtop.components[target].pkg
    def final SEED_TAR = config.bigtop.components[target].seedtar
    def final PKG_VERSION = config.bigtop.components[target].version.pkg
    def final PKG_OUTPUT_DIR = config.bigtop.components[target].outputdir
    delete ("$PKG_BUILD_DIR/deb")
    def final DEB_BLD_DIR = "$PKG_BUILD_DIR/deb/$NAME-${PKG_VERSION}"
    def final DEB_PKG_DIR = "$PKG_BUILD_DIR/deb/$PKG_NAME-${PKG_VERSION}-${BIGTOP_BUILD_STAMP}"
    mkdir (DEB_BLD_DIR)
    copy {
      from SEED_TAR
      into "$PKG_BUILD_DIR/deb/"
      rename TARBALL_DST, "${PKG_NAME}_${PKG_VERSION}.orig.tar.gz"
    }
    exec {
      workingDir DEB_BLD_DIR
      commandLine "tar --strip-components 1 -xf $DEB_BLD_DIR/../${PKG_NAME}_${PKG_VERSION}.orig.tar.gz".split(' ')
    }
    fileTree ("${BASE_DIR}/bigtop-packages/src/deb/$NAME") {
      include '**/*'
    }.copy { into "$DEB_BLD_DIR/debian" }
    copy {
      from "${BASE_DIR}/bigtop-packages/src/templates/init.d.tmpl"
      into "$DEB_BLD_DIR/debian"
    }
    fileTree ("$BASE_DIR/bigtop-packages/src/common/$NAME") {
      include '**/*'
    }.copy { into "$DEB_BLD_DIR/debian" }
    // Prepeare bom file with all the versions
    def bomWriter = new File("$DEB_BLD_DIR/debian/bigtop.bom").newWriter()
    bomVersions.each { bomWriter << "$it\n"}
    bomWriter.close()
    // Create changelog
    def changelog = new File("$DEB_BLD_DIR/debian/changelog").newWriter()
    changelog << "$PKG_NAME ($PKG_VERSION-$BIGTOP_BUILD_STAMP) stable; urgency=low\n"
    changelog << "  Clean build\n"
    changelog << " -- Bigtop <dev@bigtop.apache.org>  ${getDate()}\n"
    changelog.close()

    // Move patches and create "series"
    def patches = new File( "$DEB_BLD_DIR/debian").list({ d, f -> f ==~ /patch.*diff/}).sort()
    if (patches.size() > 0) {
      mkdir("$DEB_BLD_DIR/debian/patches")
      def seriesWriter = new File("$DEB_BLD_DIR/debian/patches/series").newWriter()
      patches.each { f ->
        def res = new File("$DEB_BLD_DIR/debian/$f").renameTo( "$DEB_BLD_DIR/debian/patches/$f")
        seriesWriter << "$f\n"
      }
      seriesWriter.close()
    }
    // Deleting obsolete files
    delete fileTree (dir: "$DEB_BLD_DIR/debian", includes: ['*.ex', '*.EX', '*.~'])
    // Creating source package
    exec {
      workingDir DEB_BLD_DIR
      commandLine "dpkg-buildpackage -uc -us -sa -S".split(' ')
    }
    mkdir(PKG_OUTPUT_DIR)
    fileTree (dir: "$DEB_PKG_DIR/..", includes: ['*.dsc', '*.diff.gz', '*.debian.tar.gz', '*.debian.tar.xz', "*_source.changes", "*.orig.tar.gz" ]).copy {
      into PKG_OUTPUT_DIR
    }
    safeDelete(DEB_BLD_DIR)
    touchTargetFile(config.bigtop.components[target].targetsdeb)
  }

  // Keeping the reference to task to be used later for correct sequencing
  Task trpm = task "$target-rpm" (dependsOn: ["${target}-srpm"],
      description: "Building RPM for $target artifacts",
      group: PACKAGES_GROUP) << {
    if (new File(config.bigtop.components[target].targetrpm)?.exists()) {
      println "\tNothing to do. Exiting..."
      return
    }
    def final PKG_BUILD_DIR = config.bigtop.components[target].builddir
    def final NAME = config.bigtop.components[target].name
    def final PKG_NAME = config.bigtop.components[target].pkg
    def final PKG_OUTPUT_DIR = config.bigtop.components[target].outputdir
    def final PKG_VERSION = config.bigtop.components[target].version.pkg
    def final BASE_VERSION = config.bigtop.components[target].version.base
    def final HADOOP_VERSION = config.bigtop.components["hadoop"].version.pkg
    def RELEASE_DIST = "rpmbuild --eval '%{?dist}' 2>/dev/null".execute().text.trim().replaceAll("'",'')
    def SRCRPM="$PKG_OUTPUT_DIR/$PKG_NAME-${PKG_VERSION}-$BIGTOP_BUILD_STAMP${RELEASE_DIST}.src.rpm"

    def command = [
        '--define', "_topdir $PKG_BUILD_DIR/rpm/",
        '--define', "${NAME}_base_version $BASE_VERSION",
        '--define', "hadoop_version ${HADOOP_VERSION}",
        '--define', "${NAME}_version ${PKG_VERSION}",
        '--define', "${NAME}_release ${BIGTOP_BUILD_STAMP}%{?dist}",
        '--rebuild', SRCRPM,
    ]
    exec {
      workingDir BASE_DIR
      executable 'rpmbuild'
      args command
    }
    fileTree ("$PKG_BUILD_DIR/rpm/RPMS") {
      include '**/*'
    }.copy { into PKG_OUTPUT_DIR }
    touchTargetFile(config.bigtop.components[target].targetrpm)
  }
  // Guarantee that tasks are ran in the order set by BOM file dependencies section
  if (buildDependencies)
    config.bigtop.dependencies.each { dependsOn, dependees ->
      if (dependees.contains(trpm.name - "-rpm")) trpm.dependsOn "$dependsOn-rpm"
    }
  // If dependencies aren't explicitly set, the default build order is defined
  // by the sequence of the components in the BOM file
  if (!config.bigtop.dependencies && bomTasks.size() > 0)
    trpm.mustRunAfter "${bomTasks.get(bomTasks.size() - 1)}-rpm".toLowerCase()
  task "$target-srpm" (dependsOn: ["${target}_vardefines" , "${target}-tar"],
      description: "Building SRPM for $target artifacts",
      group: PACKAGES_GROUP) << {
    if (new File(config.bigtop.components[target].targetsrpm)?.exists()) {
      println "\tNothing to do. Exiting..."
      return
    }
    def final NAME = config.bigtop.components[target].name
    def final PKG_NAME = config.bigtop.components[target].pkg
    def final PKG_NAME_FOR_PKG = NAME.replaceAll("-", "_")
    def final PKG_BUILD_DIR = config.bigtop.components[target].builddir
    def final SEED_TAR = config.bigtop.components[target].seedtar
    def final PKG_VERSION = config.bigtop.components[target].version.pkg
    def final BASE_VERSION = config.bigtop.components[target].version.base
    def final PKG_OUTPUT_DIR = config.bigtop.components[target].outputdir
    safeDelete ("$PKG_BUILD_DIR/rpm")
    ['INSTALL','SOURCES','BUILD','SRPMS','RPMS'].each { rpmdir ->
      mkdir("$PKG_BUILD_DIR/rpm/$rpmdir")
    }
    fileTree ("${BASE_DIR}/bigtop-packages/src/rpm/$NAME") {
      include '**/*'
    }.copy { into "$PKG_BUILD_DIR/rpm" }
    copy {
      from SEED_TAR
      into "$PKG_BUILD_DIR/rpm/SOURCES"
    }
    copy {
      from "${BASE_DIR}/bigtop-packages/src/templates/init.d.tmpl"
      into "$PKG_BUILD_DIR/rpm/SOURCES"
    }
    fileTree ("$BASE_DIR/bigtop-packages/src/common/$NAME") {
      include '**/*'
    }.copy { into "$PKG_BUILD_DIR/rpm/SOURCES" }
    // Writing bigtop.bom files with all the versions
    def bomWriter = new File("$PKG_BUILD_DIR/rpm/SOURCES/bigtop.bom").newWriter()
    bomVersions.each { bomWriter << "$it\n"}
    bomWriter.close()

    def specFileName = "${PKG_BUILD_DIR}/rpm/SPECS/${NAME}.spec"
    def specTmpFileName = "${PKG_BUILD_DIR}/rpm/SPECS/tmp_${NAME}.spec"
    def specFile = new File(specFileName)
    def specWriter = new File(specTmpFileName).newWriter()
    def patches = new File("${PKG_BUILD_DIR}/rpm/SOURCES").list({ d, f -> f ==~ /patch.*diff/}).sort()
    specFile.eachLine { line ->
      if (line.startsWith("#BIGTOP_PATCH_FILES")) {
        def patchNum = 0
        patches.each { p ->
          specWriter << "Patch$patchNum: $p\n"
          patchNum++
        }
      } else {
        if (line.startsWith("#BIGTOP_PATCH_COMMANDS")) {
          def patchNum = 0
          patches.each { p ->
            specWriter << "%patch$patchNum -p1\n"
            patchNum++
          }
        } else {
          specWriter << "$line\n"
        }
      }
    }
    specWriter.close()
    specFile.delete()
    new File(specTmpFileName).renameTo(specFileName)

    def command = [
        '--define', "_topdir $PKG_BUILD_DIR/rpm/",
        '--define', "${PKG_NAME_FOR_PKG}_base_version $BASE_VERSION",
        '--define', "${PKG_NAME_FOR_PKG}_version ${PKG_VERSION}",
        '--define', "${PKG_NAME_FOR_PKG}_release ${BIGTOP_BUILD_STAMP}%{?dist}",
        '-bs', '--nodeps', "--buildroot=${PKG_BUILD_DIR}/rpm/INSTALL",
        specFileName,
    ]
    exec {
      workingDir BASE_DIR
      executable 'rpmbuild'
      args command
    }
    mkdir(PKG_OUTPUT_DIR)
    def RELEASE_DIST = "rpmbuild --eval '%{?dist}' 2>/dev/null".execute().text.trim().replaceAll("'",'')
    copy {
      from "$PKG_BUILD_DIR/rpm/SRPMS/${PKG_NAME}-${PKG_VERSION}-${BIGTOP_BUILD_STAMP}${RELEASE_DIST}.src.rpm"
      into PKG_OUTPUT_DIR
    }
    touchTargetFile(config.bigtop.components[target].targetsrpm)
  }
  if (nativePackaging) {
    def ptype = nativePackaging.pkg
    task "$target-pkg" (dependsOn: "$target-$ptype",
        description: "Invoking a native binary packaging target $ptype",
        group: PACKAGES_GROUP) << {
    }
    task "$target-spkg" (dependsOn: "$target-s$ptype",
        description: "Invoking a native binary packaging target s$ptype",
        group: PACKAGES_GROUP) << {
    }
  }
  task "$target-version" (description: "Show version of $target component", group: PACKAGES_GROUP) << {
    println "Base: ${config.bigtop.components[target].version.base}"
  }
  task "${target}_vardefines" << {
    setDefaults(config.bigtop.components[target])

    config.bigtop.components[target].package.release = '1'

    config.bigtop.components[target].builddir = config.bigtop.builddir + "/$target"
    config.bigtop.components[target].outputdir = config.bigtop.outputdir + "/$target"
    config.bigtop.components[target].srcdir = config.bigtop.builddir + "/source"
    config.bigtop.components[target].tardir = config.bigtop.builddir + "/$target/tar/${target}-${config.bigtop.components[target].version.base}"
    config.bigtop.components[target].seedtar = config.bigtop.builddir + "/$target/tar/" + config.bigtop.components[target].tarball.destination

    config.bigtop.components[target].downloadurl =
        (!config.bigtop.components[target].url.isEmpty() &&
            !config.bigtop.components[target].url.site.isEmpty() &&
            !config.bigtop.components[target].tarball.source.isEmpty()) ?
            config.bigtop.components[target].url.site + '/' + config.bigtop.components[target].tarball.source : null
        if (!config.bigtop.components[target].tarball.destination.isEmpty())
          config.bigtop.components[target].downloaddst = DL_DIR + '/' + config.bigtop.components[target].tarball.destination

    // test that the download url will return http 200.  If it does not, use the ARCHIVE url instead of the MIRROR SITE url
    if (!config.bigtop.components[target].url.isEmpty() && !ifExists(config.bigtop.components[target].downloadurl)) {
      config.bigtop.components[target].downloadurl = config.bigtop.components[target].url.archive + '/' + config.bigtop.components[target].tarball.source
    }

    config.bigtop.components[target].targetdl  = config.bigtop.components[target].builddir + '/.download'
    config.bigtop.components[target].targettar  = config.bigtop.components[target].builddir + '/.tar'
    config.bigtop.components[target].targetsrpm  = config.bigtop.components[target].builddir + '/.srpm'
    config.bigtop.components[target].targetrpm  = config.bigtop.components[target].builddir + '/.rpm'
    config.bigtop.components[target].targetsdeb  = config.bigtop.components[target].builddir + '/.sdeb'
    config.bigtop.components[target].targetdeb  = config.bigtop.components[target].builddir + '/.deb'
    config.bigtop.components[target].targetrelnotes  = config.bigtop.components[target].builddir + '/.relnotes'

    if (System.getProperty(VERBOSE)) {
      println "$target properties (explcit and derived)"
      config.bigtop.components[target].toProperties(target).each {k ,v ->
        println "$k\t= $v"
      }
    }
  }

  task "$target-info" (dependsOn: "${target}_vardefines",
      description: "Info about $target component build",
      group: PACKAGES_GROUP) << {
    println "Info for package $target"
    println "  Will download from URL: ${config.bigtop.components[target].downloadurl}"
    println "  To destination file: ${config.bigtop.components[target].downloaddst}"
    println "  Then unpack into ${config.bigtop.components[target].srcdir}"
    println "  And create a seed tarball ${config.bigtop.components[target].seedtar}"

    println "Version: " + config.bigtop.components[target].version.base
    //TODO more about stamping
  }
  task "$target-relnotes" (description: "Preparing release notes for $target. No yet implemented!!!", group: PACKAGES_GROUP)<< {
  }
  task "$target-clean" (dependsOn: "${target}_vardefines",
      description: "Removing $target component build and output directories",
      group: PACKAGES_GROUP) << {
    safeDelete(config.bigtop.components[target].builddir)
    safeDelete(config.bigtop.components[target].outputdir)
  }
  task "$target-help" (description: "List of available tasks for $target", group: PACKAGES_GROUP) << {
    println (target + "\n\t[" + tasks.findAll { alltask -> alltask.name.startsWith(target)}*.name.join(", ") + "]")
  }

  bomTasks.add(target)
}

// Let's enforce some of the configuration requirements
private void doValidateBOM(config) {
  assert config.bigtop.version
  assert config.bigtop.stack.jdk
  assert config.bigtop.stack.scala
  assert config.bigtop.apache.size() != 0
}

// We need to make sure that all dynamic tasks are available for invocation
project.afterEvaluate {
  doValidateBOM(config)
  config.bigtop.components.each { component_label, comp ->
    assert component_label == comp.name
    genTasks(comp.name)
  }
  // Versions need to be preserved for more than just component:
  //  - there are JDK version requirement
  //  - possibly more in the future
  bomVersions = config.bigtop.components.collect {
    "${toOldStyleName(it.value.name)}_VERSION=${it.value.version.base}"
  }
  bomVersions += config.bigtop.stack.collect { // Version of the stack have different syntax
    "${toOldStyleName(it.key)}_VERSION=${it.value.version_base}"
  }
  if (System.getProperty(VERBOSE))println "BIGTOP_BOM_VERSIONS:\n${bomVersions.join(' ')}"
  // Putting all targets of different types into one common target
  task "srpm" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-srpm")}*.name,
      description: "Build all SRPM packages for the stack components",
      group: PACKAGES_GROUP
  ) << { }
  task "rpm" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-rpm")}*.name,
      description: "Build all RPM packages for the stack",
      group: PACKAGES_GROUP
  ) << { }
  task "sdeb" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-sdeb")}*.name,
      description: "Build all SDEB packages for the stack components",
      group: PACKAGES_GROUP
  ) << { }
  task "deb" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-deb")}*.name,
      description: "Build all DEB packages for the stack components",
      group: PACKAGES_GROUP
  ) << { }
  task "pkgs" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-pkg")}*.name,
      description: "Build all native packages for the stack components",
      group: PACKAGES_GROUP
  ) << { }

  task allclean (dependsOn: [clean, tasks.findAll { alltask -> alltask.name.endsWith("-clean")}*.name],
      description: "Removing $BUILD_DIR, $OUTPUT_DIR, and $DIST_DIR.\n\t\t" +
        "Cleaning all components' build and output directories.",
      group: PACKAGES_GROUP) << {
    safeDelete(BUILD_DIR)
    safeDelete(OUTPUT_DIR)
    safeDelete(DIST_DIR)
  }
  task realclean (dependsOn: allclean,
      description: "Removing $BUILD_DIR, $OUTPUT_DIR, $DIST_DIR, and $DL_DIR",
      group: PACKAGES_GROUP) << {
    delete (DL_DIR)
  }
}

task "apt" (
    description: "Creating APT repository",
    group: PACKAGES_GROUP) << {

  delete ( "$OUTPUT_DIR/apt")
  mkdir ("$OUTPUT_DIR/apt/conf")

  generateDistributionsFile("$OUTPUT_DIR/apt/conf/distributions");

  fileTree (OUTPUT_DIR) {
    include "*/*.changes"
  }.each  { changeFile ->
    exec {
      workingDir BUILD_DIR
      commandLine "reprepro -Vb $OUTPUT_DIR/apt include bigtop $changeFile".split(' ')
    }
  }
}

task "yum" (
    description: "Creating YUM repository",
    group: PACKAGES_GROUP) << {
  delete ( "$OUTPUT_DIR/repodata")
  exec {
    workingDir BUILD_DIR
    commandLine "createrepo -o $OUTPUT_DIR $OUTPUT_DIR".split(' ')
  }
}

if (nativePackaging) {
  task "repo" (dependsOn: nativePackaging.repo,
      description: "Invoking a native repository target ${nativePackaging.repo}",
      group: PACKAGES_GROUP) << { }
}