Skip to content

Commit

Permalink
feat: when only metadata files have changed, run only metadata tests (#…
Browse files Browse the repository at this point in the history 
…11)

* feat: only run user tests and build docker if needed

Fixes #9

* fix: group docker build and delivery in a single stage

Fixes #9
  • Loading branch information
@alvarolopez
alvarolopez authored Aug 14, 2024
1 parent 28c9448 commit 840d6ff
Showing 1 changed file with 149 additions and 114 deletions.
263 changes: 149 additions & 114 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,38 +126,49 @@ pipeline {
}
}
}
stage("User-defined module pipeline job") {
stage("Check if only metadata files have changed") {
steps {
//sh 'printenv'
script {
build(job: "/AI4OS-HUB-TEST/" + env.JOB_NAME.drop(10))
// Check if only metadata files have been changed

// Get the list of changed files
changed_files = sh (returnStdout: true, script: "git diff --name-only HEAD^ HEAD").trim()

// we need to check here if the change only affects any of the metadata files, but not the code
// we can't use "git diff --name-only HEAD^ HEAD" as it will return all files changed in the commit
// we need to check if the metadata files are present in the list of changed files

need_build = true

// Check if metadata files are present in the list of changed files
if (changed_files.contains("metadata.json") || changed_files.contains("ai4-metadata.json") || changed_files.contains("ai4-metadata.yml")) {
// Convert to an array and pop items
changed_files = changed_files.tokenize()
changed_files.removeAll(["metadata.json", "ai4-metadata.json", "ai4-metadata.yml"])
// now check if the list is empty
if (changed_files.size() == 0) {
need_build = false
}
}
}
}
}
stage("Docker Variable initialization") {
stage("User-defined module pipeline job") {
when {
expression {env.MODULES.contains(env.THIS_REPO)}
}
environment {
AI4OS_REGISTRY_CREDENTIALS = credentials('AIOS-registry-credentials')
expression {need_build}
}
steps {
//sh 'printenv'
script {
withFolderProperties{
docker_registry = env.AI4OS_REGISTRY
docker_repository = env.AI4OS_REGISTRY_REPOSITORY
}
docker_ids = []

docker_registry_credentials = env.AI4OS_REGISTRY_CREDENTIALS

// Check here if variables exist
build(job: "/AI4OS-HUB-TEST/" + env.JOB_NAME.drop(10))
}
}
}

stage('AI4OS Hub Docker images build') {
stage("Docker build and delivery") {
when {
expression {env.MODULES.contains(env.THIS_REPO)}

anyOf {
branch 'cicd'
branch 'main'
Expand All @@ -166,114 +177,138 @@ pipeline {
branch 'dev'
branch 'release/*'
}
expression {env.MODULES.contains(env.THIS_REPO)}

expression {need_build}
}
steps {
checkout scm

script {

// define docker tag depending on the branch/release
if ( env.BRANCH_NAME.startsWith("release/") ) {
docker_tag = env.BRANCH_NAME.drop(8)
stages {
stage("Docker Variable initialization") {
environment {
AI4OS_REGISTRY_CREDENTIALS = credentials('AIOS-registry-credentials')
}
else if ( env.BRANCH_NAME in ['master', 'main'] ) {
docker_tag = "latest"
}
else {
docker_tag = env.BRANCH_NAME
steps {
script {
withFolderProperties{
docker_registry = env.AI4OS_REGISTRY
docker_repository = env.AI4OS_REGISTRY_REPOSITORY
}
docker_ids = []

docker_registry_credentials = env.AI4OS_REGISTRY_CREDENTIALS

// Check here if variables exist
}
}
docker_tag = docker_tag.toLowerCase()
}

// get docker image name from metadata.json
meta = readJSON file: "metadata.json"
image_name = meta["sources"]["docker_registry_repo"].split("/")[1]
stage('AI4OS Hub Docker images build') {
steps {
checkout scm

script {

// define docker tag depending on the branch/release
if ( env.BRANCH_NAME.startsWith("release/") ) {
docker_tag = env.BRANCH_NAME.drop(8)
}
else if ( env.BRANCH_NAME in ['master', 'main'] ) {
docker_tag = "latest"
}
else {
docker_tag = env.BRANCH_NAME
}
docker_tag = docker_tag.toLowerCase()

// get docker image name from metadata.json
meta = readJSON file: "metadata.json"
image_name = meta["sources"]["docker_registry_repo"].split("/")[1]

// use preconfigured in Jenkins docker_repository
// XXX may confuse users? (e.g. expect xyz/myimage, but we push to ai4hub/myimage)
image = docker_repository + "/" + image_name + ":" + docker_tag
// use preconfigured in Jenkins docker_repository
// XXX may confuse users? (e.g. expect xyz/myimage, but we push to ai4hub/myimage)
image = docker_repository + "/" + image_name + ":" + docker_tag

// build docker images
// by default we expect Dockerfile name and location in the git repo root
def dockerfile = "Dockerfile"
def base_cpu_tag = ""
def base_gpu_tag = ""
def build_cpu_tag = false
def build_gpu_tag = false
// try to load constants if defined in user's repository ($jenkinsconstants_file has to be in the repo root)
jenkinsconstants_file = "JenkinsConstants.groovy"
try {
def constants = load jenkinsconstants_file
// $jenkinsconstants_file may have all constants or only "dockefile" or only both base_cpu|gpu_tag:
try {
dockerfile = constants.dockerfile
} catch (e) {}
// let's define that if used, both "base_cpu_tag" && "base_gpu_tag" are required:
try {
base_cpu_tag = constants.base_cpu_tag
} catch (e) {}
try {
base_gpu_tag = constants.base_gpu_tag
} catch (e) {}
if (!base_cpu_tag && !base_gpu_tag) {
throw new Exception("Neither \"base_cpu_tag\" nor \"base_gpu_tag\" is defined. Using default tag from ${dockerfile}")
}
if (!base_cpu_tag || !base_gpu_tag) {
throw new Exception("Check ${jenkinsconstants_file}: If separate tags for CPU and GPU are needed, both \"base_cpu_tag\" and \"base_gpu_tag\" are required!")
}
// if no Exception so far, allow building "-cpu" and "-gpu" versions
build_cpu_tag = true
build_gpu_tag = true
} catch (err) {
// if $jenkinsconstants_file not found or one of base_cpu|gpu_tag is not defined or none of them, build docker image with default params
println("[WARNING] Exception: ${err}")
println("[INFO] Using default parameters for Docker image building")
image_id = docker.build(image, "--no-cache --force-rm -f ${dockerfile} .")
}
// build "-cpu" image, if configured
if (build_cpu_tag) {
image_id = docker.build(image, "--no-cache --force-rm --build-arg tag=${base_cpu_tag} -f ${dockerfile} .")
// define additional docker_tag_cpu to mark it as "cpu" version
docker_tag_cpu = (docker_tag == "latest") ? "cpu" : (docker_tag + "-cpu")
image_cpu = docker_repository + "/" + image_name + ":" + docker_tag_cpu
sh "docker tag ${image} ${image_cpu}"
docker_ids.add(image_cpu)
}
// build docker images
// by default we expect Dockerfile name and location in the git repo root
def dockerfile = "Dockerfile"
def base_cpu_tag = ""
def base_gpu_tag = ""
def build_cpu_tag = false
def build_gpu_tag = false
// try to load constants if defined in user's repository ($jenkinsconstants_file has to be in the repo root)
jenkinsconstants_file = "JenkinsConstants.groovy"
try {
def constants = load jenkinsconstants_file
// $jenkinsconstants_file may have all constants or only "dockefile" or only both base_cpu|gpu_tag:
try {
dockerfile = constants.dockerfile
} catch (e) {}
// let's define that if used, both "base_cpu_tag" && "base_gpu_tag" are required:
try {
base_cpu_tag = constants.base_cpu_tag
} catch (e) {}
try {
base_gpu_tag = constants.base_gpu_tag
} catch (e) {}
if (!base_cpu_tag && !base_gpu_tag) {
throw new Exception("Neither \"base_cpu_tag\" nor \"base_gpu_tag\" is defined. Using default tag from ${dockerfile}")
}
if (!base_cpu_tag || !base_gpu_tag) {
throw new Exception("Check ${jenkinsconstants_file}: If separate tags for CPU and GPU are needed, both \"base_cpu_tag\" and \"base_gpu_tag\" are required!")
}
// if no Exception so far, allow building "-cpu" and "-gpu" versions
build_cpu_tag = true
build_gpu_tag = true
} catch (err) {
// if $jenkinsconstants_file not found or one of base_cpu|gpu_tag is not defined or none of them, build docker image with default params
println("[WARNING] Exception: ${err}")
println("[INFO] Using default parameters for Docker image building")
image_id = docker.build(image, "--no-cache --force-rm -f ${dockerfile} .")
}
// build "-cpu" image, if configured
if (build_cpu_tag) {
image_id = docker.build(image, "--no-cache --force-rm --build-arg tag=${base_cpu_tag} -f ${dockerfile} .")
// define additional docker_tag_cpu to mark it as "cpu" version
docker_tag_cpu = (docker_tag == "latest") ? "cpu" : (docker_tag + "-cpu")
image_cpu = docker_repository + "/" + image_name + ":" + docker_tag_cpu
sh "docker tag ${image} ${image_cpu}"
docker_ids.add(image_cpu)
}

// check that in the built image (cpu or default), DEEPaaS API starts as expected
// EXCLUDE "cicd" branch
// do it with only "cpu|default" image:
// a) can stop before proceeding with "gpu" version b) "gpu" may fail without GPU hardware anyway
if (env.BRANCH_NAME != 'cicd') {
sh "git clone https://github.com/ai4os/ai4os-hub-check-artifact"
sh "bash ai4os-hub-check-artifact/check-artifact ${image}"
sh "rm -rf ai4os-hub-check-artifact"
}
// check that in the built image (cpu or default), DEEPaaS API starts as expected
// EXCLUDE "cicd" branch
// do it with only "cpu|default" image:
// a) can stop before proceeding with "gpu" version b) "gpu" may fail without GPU hardware anyway
if (env.BRANCH_NAME != 'cicd') {
sh "git clone https://github.com/ai4os/ai4os-hub-check-artifact"
sh "bash ai4os-hub-check-artifact/check-artifact ${image}"
sh "rm -rf ai4os-hub-check-artifact"
}

docker_ids.add(image)
docker_ids.add(image)

// finally, build "-gpu" image, if configured
if (build_gpu_tag) {
// define additional docker_tag_gpu to mark "gpu" version
docker_tag_gpu = (docker_tag == "latest") ? "gpu" : (docker_tag + "-gpu")
image = docker_repository + "/" + image_name + ":" + docker_tag_gpu
image_id = docker.build(image, "--no-cache --force-rm --build-arg tag=${base_gpu_tag} -f ${dockerfile} .")
docker_ids.add(image)
}
// finally, build "-gpu" image, if configured
if (build_gpu_tag) {
// define additional docker_tag_gpu to mark "gpu" version
docker_tag_gpu = (docker_tag == "latest") ? "gpu" : (docker_tag + "-gpu")
image = docker_repository + "/" + image_name + ":" + docker_tag_gpu
image_id = docker.build(image, "--no-cache --force-rm --build-arg tag=${base_gpu_tag} -f ${dockerfile} .")
docker_ids.add(image)
}

}
}
}
}
}
stage('AI4OS Hub Docker delivery to registry') {
when {
expression {env.MODULES.contains(env.THIS_REPO)}
expression {docker_ids.size() > 0}
}
steps {
script {
docker.withRegistry(docker_registry, docker_registry_credentials) {
docker_ids.each {
docker.image(it).push()
stage('AI4OS Hub Docker delivery to registry') {
when {
expression {docker_ids.size() > 0}
}
steps {
script {
docker.withRegistry(docker_registry, docker_registry_credentials) {
docker_ids.each {
docker.image(it).push()
}
}
}
}
}
Expand Down

0 comments on commit 840d6ff

Please sign in to comment.