Create a PR main.tf template and use tfvars to specify test envs

jenkins_pipelines/environments/common/TESTING-DONT-REVIEW-pipeline-pull-request-light.groovy

@@ -0,0 +1,226 @@
+def run(params) {
+    timestamps {
+        // Start pipeline with default values
+        arch= 'x86_64'
+        built = false
+        deployed = false
+        tests_passed = false
+        sumaform_backend = 'libvirt'
+        terraform_bin = '/usr/bin/terraform'
+        terraform_bin_plugins = '/usr/bin'
+        service_pack_migration = false
+//        terracumber_gitrepo = 'https://github.com/uyuni-project/terracumber.git'
+        terracumber_gitrepo = 'https://github.com/maximenoel8/terracumber.git'
+        terracumber_ref = 'support_tfvars'
+        terraform_init = true
+        rake_namespace = 'cucumber'
+        rake_parallel_namespace = 'parallel'
+        jenkins_workspace = '/home/jenkins/jenkins-build/workspace/'
+        pull_request_repo = 'https://github.com/uyuni-project/uyuni.git'
+        builder_api = 'https://api.opensuse.org'
+        build_url = 'https://build.opensuse.org'
+        builder_project = 'systemsmanagement:Uyuni:Master:PR'
+        source_project = 'systemsmanagement:Uyuni:Master'
+        sumaform_tools_project = 'systemsmanagement:sumaform:tools'
+        test_packages_project = 'systemsmanagement:Uyuni:Test-Packages:Pool'
+        build_repo = 'openSUSE_Leap_15.5'
+        environment_workspace = null
+        url_prefix="https://ci.suse.de/view/Manager/view/Uyuni/job/${env.JOB_NAME}"
+        env.common_params = ''
+        fqdn_jenkins_node = sh(script: "hostname -f", returnStdout: true).trim()
+        env_number = 10
+        tfvariables_file  = 'susemanager-ci/terracumber_config/tf_files/variables/PR-testing-variables.tf'
+        tfvars_product_version = "susemanager-ci/terracumber_config/tf_files/tfvars/PR-testing-${product_version}.tfvars"
+        tfvars_platform_localisation = "susemanager-ci/terracumber_config/tf_files/tfvars/PR-testing-${platform_localisation}-environments.tfvars"
+        tf_local_variables = 'susemanager-ci/terracumber_config/tf_files/tfvars/PR-testing-additionnal-repos.tf'
+        try {
+            stage('Checkout CI tools') {
+                ws(environment_workspace){
+                    if(must_test) {
+                        git url: terracumber_gitrepo, branch: terracumber_ref
+                        dir("susemanager-ci") {
+                            checkout scm
+                        }
+
+                        // Define test environment parameters
+                        env.resultdir = "${WORKSPACE}/results"
+                        env.resultdirbuild = "${resultdir}/${BUILD_NUMBER}"
+                        env.tf_file = "susemanager-ci/terracumber_config/tf_files/PR-testing-template.tf"
+                        env.common_params = "--outputdir ${resultdir} --tf ${tf_file} --gitfolder ${resultdir}/sumaform --tf_variables_description_file=${tfvariables_file}"
+
+                        if (params.terraform_parallelism) {
+                            env.common_params = "${env.common_params} --parallelism ${params.terraform_parallelism}"
+                        }
+
+                        // Clean up old results
+                        sh "if [ -d ${resultdir} ];then ./clean-old-results -r ${resultdir};fi"
+
+                        // Create a directory for  to place the directory with the build results (if it does not exist)
+                        sh "mkdir -p ${resultdir}"
+
+                        // Clone sumaform
+                        sh "set +x; source /home/jenkins/.credentials set -x; ./terracumber-cli ${common_params} --gitrepo ${sumaform_gitrepo} --gitref ${sumaform_ref} --runstep gitsync"
+                    }
+                }
+            }
+            stage('Deploy') {
+                ws(environment_workspace){
+                    if(must_test) {
+                        // Delete old terraform.tfvars
+                        sh "rm -f ${env.resultdir}/sumaform/terraform.tfvars"
+                        // Merge product en platform variables into terraform.tfvars
+                        sh "cat ${tfvars_product_version} ${tfvars_platform_localisation} >> ${env.resultdir}/sumaform/terraform.tfvars"
+                        // Add environment to use in tfvars
+                        sh "echo 'ENVIRONMENT = \'${env_number}\'' >> ${env.resultdir}/sumaform/terraform.tfvars"
+                        // Copy the variable declaration file
+                        sh "cp ${tf_local_variables} ${env.resultdir}/sumaform/"
+
+                        // Add all
+                        // Passing the built repository by parameter using a environment variable to terraform file
+                        // TODO: We will need to add a logic to replace the host, when we use IBS for spacewalk
+                        sh "echo -e \\\n############ Repositories variables ############\\\n >> ${env.resultdir}/sumaform/terraform.tfvars"
+                        sh "echo PULL_REQUEST_REPO = \\\"http://download.suse.de/ibs/SUSE/Products/SLE-Module-Development-Tools/15-SP4/x86_64/product/\\\" >> ${env.resultdir}/sumaform/terraform.tfvars"
+                        sh "echo MASTER_REPO = \\\"http://download.suse.de/ibs/SUSE/Products/SLE-Module-Development-Tools/15-SP4/x86_64/product/\\\" >> ${env.resultdir}/sumaform/terraform.tfvars"
+                        sh "echo MASTER_OTHER_REPO = \\\"http://download.suse.de/ibs/SUSE/Products/SLE-Module-Development-Tools/15-SP4/x86_64/product/\\\" >> ${env.resultdir}/sumaform/terraform.tfvars"
+                        sh "echo MASTER_SUMAFORM_TOOLS_REPO = \\\"http://download.suse.de/ibs/SUSE/Products/SLE-Module-Development-Tools/15-SP4/x86_64/product/\\\" >> ${env.resultdir}/sumaform/terraform.tfvars"
+                        sh "echo TEST_PACKAGES_REPO = \\\"http://download.suse.de/ibs/SUSE/Products/SLE-Module-Development-Tools/15-SP4/x86_64/product/\\\" >> ${env.resultdir}/sumaform/terraform.tfvars"
+                        sh "echo UPDATE_REPO = \\\"http://download.suse.de/ibs/SUSE/Products/SLE-Module-Development-Tools/15-SP4/x86_64/product/\\\" >> ${env.resultdir}/sumaform/terraform.tfvars"
+
+                        if (additional_repo_url == '') {
+                            echo "Adding dummy repo for update repo"
+                            sh "echo ADDITIONAL_REPO_URL = \\\"${additional_repo}\\\" >> ${env.resultdir}/sumaform/terraform.tfvars"
+                        } else {
+                            echo "Adding ${additional_repo_url}"
+                            sh "echo ADDITIONAL_REPO_URL = \\\"${additional_repo_url}\\\" >> ${env.resultdir}/sumaform/terraform.tfvars"
+                        }
+
+                        sh "echo SLE_CLIENT_REPO = \\\"http://download.suse.de/ibs/SUSE/Products/SLE-Module-Development-Tools/15-SP4/x86_64/product/\\\" >> ${env.resultdir}/sumaform/terraform.tfvars"
+                        sh "echo RHLIKE_CLIENT_REPO = \\\"http://download.suse.de/ibs/SUSE/Products/SLE-Module-Development-Tools/15-SP4/x86_64/product/\\\" >> ${env.resultdir}/sumaform/terraform.tfvars"
+                        sh "echo DEBLIKE_CLIENT_REPO = \\\"http://download.suse.de/ibs/SUSE/Products/SLE-Module-Development-Tools/15-SP4/x86_64/product/\\\" >> ${env.resultdir}/sumaform/terraform.tfvars"
+                        sh "echo OPENSUSE_CLIENT_REPO = \\\"http://download.suse.de/ibs/SUSE/Products/SLE-Module-Development-Tools/15-SP4/x86_64/product/\\\" >> ${env.resultdir}/sumaform/terraform.tfvars"
+
+                        // Provision the environment
+                        if (terraform_init) {
+                            env.TERRAFORM_INIT = '--init'
+                        } else {
+                            env.TERRAFORM_INIT = ''
+                        }
+                        sh "set +x; source /home/jenkins/.credentials set -x; export TF_VAR_CUCUMBER_GITREPO=${cucumber_gitrepo}; export TF_VAR_CUCUMBER_BRANCH=${cucumber_ref}; export TERRAFORM=${terraform_bin}; export TERRAFORM_PLUGINS=${terraform_bin_plugins}; ./terracumber-cli ${common_params} --logfile ${resultdirbuild}/sumaform.log ${env.TERRAFORM_INIT} --taint '.*(domain|main_disk).*' --runstep provision"
+                        deployed = true
+
+                        // Collect and tag Flaky tests from the GitHub Board
+                        def statusCode = sh script:"./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'cd /root/spacewalk/testsuite; export BUILD_NUMBER=${BUILD_NUMBER}; rake utils:collect_and_tag_flaky_tests'", returnStatus:true
+                        sh "exit 0"
+                    }
+                }
+            }
+            stage('Sanity Check') {
+                ws(environment_workspace){
+                    if(must_test) {
+                        sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'cd /root/spacewalk/testsuite; rake cucumber:sanity_check'"
+                    }
+                }
+            }
+            stage('Core - Setup') {
+                ws(environment_workspace){
+                    if(must_test) {
+                        sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'cd /root/spacewalk/testsuite; rake cucumber:core'"
+                        sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'cd /root/spacewalk/testsuite; rake cucumber:reposync'"
+                    }
+                }
+            }
+            stage('Core - Initialize clients') {
+                ws(environment_workspace){
+                    if(must_test) {
+                        namespace = rake_namespace
+                        sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'cd /root/spacewalk/testsuite; rake ${namespace}:init_clients'"
+                    }
+                }
+            }
+            stage('Secondary features') {
+                ws(environment_workspace){
+                    if(must_test && ( params.functional_scopes || run_all_scopes) ) {
+                        def statusCode1 = sh script:"./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd '${secondary_exports} cd /root/spacewalk/testsuite; rake cucumber:secondary'", returnStatus:true
+                        def statusCode2 = sh script:"./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd '${secondary_exports} cd /root/spacewalk/testsuite; rake ${rake_namespace}:secondary_parallelizable'", returnStatus:true
+                        sh "exit \$(( ${statusCode1}|${statusCode2} ))"
+                    }
+                }
+            tests_passed = true
+            }
+        }
+        finally {
+            stage('Clean up lockfiles') {
+                if(running_same_pr == "no"){
+                    sh(script: "rm -f ${env.suma_pr_lockfile}")
+                }
+                if(environment_workspace){
+                    ws(environment_workspace){
+                        if (env.env_file) {
+                            if (tests_passed || !deployed){
+                                println("Unlock environment")
+                                sh "rm -f ${env_file}*"
+                            } else {
+                                println("Keep the environment locked for 24 hours so you can debug")
+                                sh "echo \"rm -f ${env_file}*\" | at now +24 hour"
+                                sh "echo keep:24h >> ${env_file}.info"
+                                sh "python3 ${WORKSPACE}/product/susemanager-utils/testing/automation/run-command-in-server.py --command=\"chmod 755 /tmp/set_custom_header.sh;/tmp/set_custom_header.sh -e ${env_number} -m ${email_to} -t 24\" --username=\"root\" --password=\"linux\" -v -i suma-pr${env_number}-srv.mgr.prv.suse.net"
+                            }
+                        }
+                    }
+                }
+            }
+            stage('Get test results') {
+                if(environment_workspace && common_params != ''){
+                    ws(environment_workspace){
+                        def error = 0
+                        if(must_test) {
+                            if (deployed) {
+                                try {
+                                    sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'cd /root/spacewalk/testsuite; rake cucumber:finishing_pr'"
+                                } catch(Exception ex) {
+                                    println("ERROR: rake cucumber:finishing_pr failed")
+                                    error = 1
+                                }
+                                try {
+                                    sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'cd /root/spacewalk/testsuite; rake utils:generate_test_report'"
+                                } catch(Exception ex) {
+                                    println("ERROR: rake utils:generate_test_repor failed")
+                                    error = 1
+                                }
+                                sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep getresults"
+                                publishHTML( target: [
+                                        allowMissing: true,
+                                        alwaysLinkToLastBuild: false,
+                                        keepAll: true,
+                                        reportDir: "${resultdirbuild}/cucumber_report/",
+                                        reportFiles: 'cucumber_report.html',
+                                        reportName: "TestSuite Report for Pull Request ${builder_project}:${pull_request_number}"]
+                                )
+                                junit allowEmptyResults: true, testResults: "results/${BUILD_NUMBER}/results_junit/*.xml"
+                            }
+                            if (fileExists("results/${BUILD_NUMBER}")) {
+                                archiveArtifacts artifacts: "results/${BUILD_NUMBER}/**/*"
+                            }
+                            if (email_to != '') {
+                                sh " export TF_VAR_MAIL_TO=${email_to};export TF_VAR_URL_PREFIX=${url_prefix}; ./terracumber-cli ${common_params} --logfile ${resultdirbuild}/mail.log --runstep mail"
+                            }
+                            // Clean up old results
+                            sh "./clean-old-results -r ${resultdir} -s 10"
+                            sh "exit ${error}"
+                        }
+                    }
+                }
+            }
+            stage('Remove build project') {
+                if(environment_workspace){
+                    ws(environment_workspace){
+                        sh "rm -rf ${environment_workspace}/repos/${builder_project}:${pull_request_number}/${build_repo}/${arch}"
+                        sh "rm -rf ${builder_project}:${pull_request_number}"
+                    }
+                }
+            }
+        }
+    }
+}
+
+return this

jenkins_pipelines/environments/common/pipeline-pull-request-envs.groovy

@@ -27,8 +27,11 @@ if (env.JOB_NAME == "uyuni-prs-ci-tests-jordi") {
     first_env = 10;
     last_env = 10;
     // if you change the sumaform repo or reference, you need to remove the sumaform directory from the results folder
-    sumaform_gitrepo = "https://github.com/witekest/sumaform.git";
-    sumaform_ref = "server_monitoring";
+    // /!\ Temporary change /!\
+    //sumaform_gitrepo = "https://github.com/witekest/sumaform.git";
+    //sumaform_ref = "server_monitoring";
+    sumaform_gitrepo = "https://github.com/uyuni-project/sumaform.git";
+    sumaform_ref = "master";
 } else if (env.JOB_NAME == "uyuni-prs-ci-tests-ion") {
     first_env = 7;
     last_env = 7;