-
Notifications
You must be signed in to change notification settings - Fork 1
/
Jenkinsfile
357 lines (326 loc) · 16.4 KB
/
Jenkinsfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
// vim: filetype=groovy
@Library([
'github.com/indigo-dc/jenkins-pipeline-library@release/2.1.1',
]) _
def projectConfig
pipeline {
agent {
label 'docker'
}
environment {
// Remove .git from the GIT_URL link
THIS_REPO = "${env.GIT_URL.endsWith(".git") ? env.GIT_URL[0..-5] : env.GIT_URL}"
// Get list of AI4OS Hub repositories from "modules-catalog/.gitmodules"
MODULES_CATALOG_URL = "https://raw.githubusercontent.com/ai4os-hub/modules-catalog/master/.gitmodules"
MODULES = sh (returnStdout: true, script: "curl -s ${MODULES_CATALOG_URL}").trim()
METADATA_FILE = "metadata.json"
}
stages {
stage('Metadata tests') {
parallel {
stage('AI4OS Hub metadata V1 validation') {
when {
expression {env.MODULES.contains(env.THIS_REPO)}
}
agent {
docker {
image 'ai4oshub/ci-images:python3.12'
}
}
steps {
script {
// Check if .metadata.json is present in the repository
if (!fileExists("metadata.json")) {
error("metadata.json file not found in the repository")
}
env.METADATA_FILE = "metadata.json"
}
script {
sh "ai4-metadata validate --metadata-version 1.0.0 metadata.json"
}
}
}
stage('AI4OS Hub metadata V2 validation (JSON)') {
when {
expression {env.MODULES.contains(env.THIS_REPO)}
// Check if metadata.json is present in the repository
expression {fileExists("ai4-metadata.json")}
}
agent {
docker {
image 'ai4oshub/ci-images:python3.12'
}
}
steps {
script {
// Check if metadata files are present in the repository
if (!fileExists("ai4-metadata.json")) {
error("ai4-metadata.json file not found in the repository")
}
if (fileExists("ai4-metadata.yml")) {
error("Both ai4-metadata.json and ai4-metadata.yml files found in the repository")
}
env.METADATA_FILE = "ai4-metadata.json"
}
script {
sh "ai4-metadata validate --metadata-version 2.0.0 ai4-metadata.json"
}
}
}
stage('AI4OS Hub metadata V2 validation (YAML)') {
when {
expression {env.MODULES.contains(env.THIS_REPO)}
// Check if metadata.json is present in the repository
expression {fileExists("ai4-metadata.yml")}
}
agent {
docker {
image 'ai4oshub/ci-images:python3.12'
}
}
steps {
script {
if (!fileExists("ai4-metadata.yml")) {
error("ai4-metadata.yml file not found in the repository")
}
if (fileExists("ai4-metadata.json")) {
error("Both ai4-metadata.json and ai4-metadata.yml files found in the repository")
}
}
script {
sh "ai4-metadata validate --metadata-version 2.0.0 ai4-metadata.yml"
}
}
}
stage("License validation") {
steps {
script {
// Check if LICENSE file is present in the repository
if (!fileExists("LICENSE")) {
error("LICENSE file not found in the repository")
}
}
}
}
}
}
stage("Check if only metadata files have changed") {
steps {
script {
// Check if only metadata files have been changed
// Get the list of changed files
changed_files = sh (returnStdout: true, script: "git diff --name-only HEAD^ HEAD").trim()
// we need to check here if the change only affects any of the metadata files, but not the code
// we can't use "git diff --name-only HEAD^ HEAD" as it will return all files changed in the commit
// we need to check if the metadata files are present in the list of changed files
need_build = true
// Check if metadata files are present in the list of changed files
if (changed_files.contains("metadata.json") || changed_files.contains("ai4-metadata.json") || changed_files.contains("ai4-metadata.yml")) {
// Convert to an array and pop items
changed_files = changed_files.tokenize()
changed_files.removeAll(["metadata.json", "ai4-metadata.json", "ai4-metadata.yml"])
// now check if the list is empty
if (changed_files.size() == 0) {
need_build = false
}
}
}
}
}
// Let's run user tests for all repos
stage("User-defined module pipeline job") {
when {
anyOf {
expression {need_build}
triggeredBy 'UserIdCause'
}
}
steps {
//sh 'printenv'
script {
build(job: "/AI4OS-HUB-TEST/" + env.JOB_NAME.drop(10))
}
}
}
stage("Docker build and delivery") {
when {
expression {env.MODULES.contains(env.THIS_REPO)}
anyOf {
branch 'cicd'
branch 'main'
branch 'master'
branch 'test'
branch 'dev'
branch 'release/*'
}
anyOf {
expression {need_build}
triggeredBy 'UserIdCause'
}
}
stages {
stage("Docker Variable initialization") {
environment {
AI4OS_REGISTRY_CREDENTIALS = credentials('AIOS-registry-credentials')
}
steps {
script {
withFolderProperties{
docker_registry = env.AI4OS_REGISTRY
docker_repository = env.AI4OS_REGISTRY_REPOSITORY
}
docker_ids = []
docker_registry_credentials = env.AI4OS_REGISTRY_CREDENTIALS
// Check here if variables exist
}
}
}
stage('AI4OS Hub Docker images build') {
steps {
checkout scm
script {
// define docker tag depending on the branch/release
if ( env.BRANCH_NAME.startsWith("release/") ) {
docker_tag = env.BRANCH_NAME.drop(8)
}
else if ( env.BRANCH_NAME in ['master', 'main'] ) {
docker_tag = "latest"
}
else {
docker_tag = env.BRANCH_NAME
}
docker_tag = docker_tag.toLowerCase()
// get docker image name from metadata.json / ai4-metadata.json
meta = readJSON file: env.METADATA_FILE
if (env.METADATA_FILE == "metadata.json") {
image_name = meta["sources"]["docker_registry_repo"].split("/")[1]
} else {
image_name = meta["links"]["docker_image"].split("/")[1]
}
// use preconfigured in Jenkins docker_repository
// XXX may confuse users? (e.g. expect xyz/myimage, but we push to ai4hub/myimage)
image = docker_repository + "/" + image_name + ":" + docker_tag
// build docker images
// by default we expect Dockerfile name and location in the git repo root
def dockerfile = "Dockerfile"
def base_cpu_tag = ""
def base_gpu_tag = ""
def build_cpu_tag = false
def build_gpu_tag = false
// try to load constants if defined in user's repository ($jenkinsconstants_file has to be in the repo root)
jenkinsconstants_file = "JenkinsConstants.groovy"
try {
def constants = load jenkinsconstants_file
// $jenkinsconstants_file may have all constants or only "dockefile" or only both base_cpu|gpu_tag:
try {
dockerfile = constants.dockerfile
} catch (e) {}
// let's define that if used, both "base_cpu_tag" && "base_gpu_tag" are required:
try {
base_cpu_tag = constants.base_cpu_tag
} catch (e) {}
try {
base_gpu_tag = constants.base_gpu_tag
} catch (e) {}
if (!base_cpu_tag && !base_gpu_tag) {
throw new Exception("Neither \"base_cpu_tag\" nor \"base_gpu_tag\" is defined. Using default tag from ${dockerfile}")
}
if (!base_cpu_tag || !base_gpu_tag) {
throw new Exception("Check ${jenkinsconstants_file}: If separate tags for CPU and GPU are needed, both \"base_cpu_tag\" and \"base_gpu_tag\" are required!")
}
// if no Exception so far, allow building "-cpu" and "-gpu" versions
build_cpu_tag = true
build_gpu_tag = true
} catch (err) {
// if $jenkinsconstants_file not found or one of base_cpu|gpu_tag is not defined or none of them, build docker image with default params
println("[WARNING] Exception: ${err}")
println("[INFO] Using default parameters for Docker image building. Using ${env.BRANCH_NAME} branch")
image_id = docker.build(image,
"--no-cache --force-rm --build-arg branch=${env.BRANCH_NAME} -f ${dockerfile} .")
}
// build "-cpu" image, if configured
if (build_cpu_tag) {
image_id = docker.build(image,
"--no-cache --force-rm --build-arg branch=${env.BRANCH_NAME} --build-arg tag=${base_cpu_tag} -f ${dockerfile} .")
// define additional docker_tag_cpu to mark it as "cpu" version
docker_tag_cpu = (docker_tag == "latest") ? "cpu" : (docker_tag + "-cpu")
image_cpu = docker_repository + "/" + image_name + ":" + docker_tag_cpu
sh "docker tag ${image} ${image_cpu}"
docker_ids.add(image_cpu)
}
// check that in the built image (cpu or default), DEEPaaS API starts as expected
// EXCLUDE "cicd" branch
// do it with only "cpu|default" image:
// a) can stop before proceeding with "gpu" version b) "gpu" may fail without GPU hardware anyway
if (env.BRANCH_NAME != 'cicd') {
sh "rm -rf ai4os-hub-check-artifact"
sh "git clone https://github.com/ai4os/ai4os-hub-check-artifact"
sh "bash ai4os-hub-check-artifact/check-artifact ${image}"
sh "rm -rf ai4os-hub-check-artifact"
}
docker_ids.add(image)
// finally, build "-gpu" image, if configured
if (build_gpu_tag) {
// define additional docker_tag_gpu to mark "gpu" version
docker_tag_gpu = (docker_tag == "latest") ? "gpu" : (docker_tag + "-gpu")
image = docker_repository + "/" + image_name + ":" + docker_tag_gpu
image_id = docker.build(image,
"--no-cache --force-rm --build-arg branch=${env.BRANCH_NAME} --build-arg tag=${base_gpu_tag} -f ${dockerfile} .")
docker_ids.add(image)
}
}
}
}
stage('AI4OS Hub Docker delivery to registry') {
when {
expression {docker_ids.size() > 0}
}
steps {
script {
docker.withRegistry(docker_registry, docker_registry_credentials) {
docker_ids.each {
docker.image(it).push()
}
}
}
}
}
}
}
stage('Update OSCAR services') {
when {
expression {env.MODULES.contains(env.THIS_REPO)}
}
agent {
docker {
image 'ai4oshub/ci-images:python3.12'
}
}
environment {
OSCAR_SERVICE_TOKEN = credentials('oscar-service-token')
}
steps {
withFolderProperties {
script {
dir("_ai4os-hub-qa") {
git branch: "master",
url: 'https://github.com/ai4os/ai4os-hub-qa'
}
sh "./_ai4os-hub-qa/scripts/oscar_update.py"
}
}
}
}
}
post {
// Clean after build
always {
cleanWs(cleanWhenNotBuilt: false,
deleteDirs: true,
disableDeferredWipeout: true,
notFailBuild: true,
patterns: [[pattern: '.gitignore', type: 'INCLUDE'],
[pattern: '.propsfile', type: 'EXCLUDE']])
}
}
}