From cc0e669a02aaf8ea73ece3075f49ff618c43ac36 Mon Sep 17 00:00:00 2001 From: Yaroslav Stefinko Date: Sun, 27 Oct 2024 20:20:27 +0200 Subject: [PATCH] CI clean-up (#533) Remove some redundant files from GitHub. Relates-To: DATASDK-42 Signed-off-by: Yaroslav Stefinko --- .gitlab-ci.yml | 170 ------------------ scripts/linux/nv/gitlab_test_performance.sh | 100 ----------- .../linux/nv/gitlab_test_publish_dryrun.sh | 38 ---- .../wv/gitlab_test_performance_extended.sh | 101 ----------- scripts/misc/artifactory_upload.sh | 46 ----- scripts/misc/full_report_generation.sh | 56 ------ 6 files changed, 511 deletions(-) delete mode 100644 .gitlab-ci.yml delete mode 100755 scripts/linux/nv/gitlab_test_performance.sh delete mode 100755 scripts/linux/nv/gitlab_test_publish_dryrun.sh delete mode 100755 scripts/linux/wv/gitlab_test_performance_extended.sh delete mode 100755 scripts/misc/artifactory_upload.sh delete mode 100755 scripts/misc/full_report_generation.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 43ecc166..00000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,170 +0,0 @@ -image: ${DOCKER_REGISTRY}/${DOCKER_IMAGE}:${DOCKER_IMAGE_VERSION} - -include: - - template: Security/SAST.gitlab-ci.yml - - template: Security/Secret-Detection.gitlab-ci.yml - - template: Security/Dependency-Scanning.gitlab-ci.yml - -stages: - - test - - translate_report - - deploy - -regression_test_job: - stage: test - when: manual - tags: - - docker-prod - image: ${DOCKER_REGISTRY}/${DOCKER_IMAGE_PERFORMANCE}:${DOCKER_IMAGE_PERFORMANCE_VERSION} - script: - - npm install -g yarn - - yarn - - yarn bootstrap - - npm run build --loglevel verbose - - npm run test --loglevel verbose - - npm run integration-test --loglevel verbose - - npm run functional-test --loglevel verbose - - npm run api-breaks-test --loglevel verbose - - cp ./tests/integration/api-breaks/xunit.xml $CI_PROJECT_DIR/api-breaks-xunit.xml - - cp ./tests/integration/xunit.xml $CI_PROJECT_DIR/integration-xunit.xml - - cp ./tests/functional/xunit.xml $CI_PROJECT_DIR/functional-xunit.xml - - cp ./@here/olp-sdk-authentication/xunit.xml $CI_PROJECT_DIR/olp-sdk-authentication-xunit.xml - - cp ./@here/olp-sdk-core/xunit.xml $CI_PROJECT_DIR/olp-sdk-core-xunit.xml - - cp ./@here/olp-sdk-dataservice-api/xunit.xml $CI_PROJECT_DIR/olp-sdk-dataservice-api-xunit.xml - - cp ./@here/olp-sdk-dataservice-read/xunit.xml $CI_PROJECT_DIR/olp-sdk-dataservice-read-xunit.xml - - cp ./@here/olp-sdk-dataservice-write/xunit.xml $CI_PROJECT_DIR/olp-sdk-dataservice-write-xunit.xml - - cp ./@here/olp-sdk-fetch/xunit.xml $CI_PROJECT_DIR/olp-sdk-fetch-xunit.xml - only: - - master - - branches - artifacts: - when: always - paths: - - ./*.xml - expire_in: 1 year # save for 1 year as job artifacts - -regression_generate_report_job: - stage: translate_report - when: manual - tags: - - docker-prod - image: python:3.8 - before_script: - - pip install junit2html==26 - script: - - python -m junit2htmlreport --report-matrix api-breaks-index.html ./api-breaks-xunit.xml - - python -m junit2htmlreport --report-matrix fetch-index.html ./olp-sdk-fetch-xunit.xml - - python -m junit2htmlreport --report-matrix integration-index.html ./integration-xunit.xml - - python -m junit2htmlreport --report-matrix functional-index.html ./functional-xunit.xml - - python -m junit2htmlreport --report-matrix authentication-index.html ./olp-sdk-authentication-xunit.xml - - python -m junit2htmlreport --report-matrix core-index.html ./olp-sdk-core-xunit.xml - - python -m junit2htmlreport --report-matrix dataservice-api-index.html ./olp-sdk-dataservice-api-xunit.xml - - python -m junit2htmlreport --report-matrix dataservice-read-index.html ./olp-sdk-dataservice-read-xunit.xml - - python -m junit2htmlreport --report-matrix dataservice-write-index.html ./olp-sdk-dataservice-write-xunit.xml - - ./scripts/misc/full_report_generation.sh - - if $(ls $CI_PROJECT_DIR/index.html &>/dev/null); then tar -czf ${CI_JOB_NAME}_test_reports.tar.gz --directory=$CI_PROJECT_DIR index.html ; fi; - - $CI_PROJECT_DIR/scripts/misc/artifactory_upload.sh edge-sdks/sdk-for-typescript/test-reports/$CI_JOB_NAME/$CI_JOB_ID/${CI_JOB_NAME}_test_reports.tar.gz ${CI_JOB_NAME}_test_reports.tar.gz - artifacts: - paths: - - .public - - ./*.html - - ./*.xml - when: always - expire_in: 3 yrs # save our zip for 3 years as job artifacts - only: - refs: - - master - - branches - -nightly_perf_publish_test_job: - stage: test - tags: - - docker-prod - image: ${DOCKER_REGISTRY}/${DOCKER_IMAGE_PERFORMANCE}:${DOCKER_IMAGE_PERFORMANCE_VERSION} - script: - - $CI_PROJECT_DIR/scripts/linux/nv/gitlab_test_performance.sh - - $CI_PROJECT_DIR/scripts/linux/nv/gitlab_test_publish_dryrun.sh - - if $(ls $CI_PROJECT_DIR/*.html &>/dev/null); then tar -czf ${CI_JOB_NAME}_test_reports.tar.gz --directory=$CI_PROJECT_DIR *.html ; fi; - - $CI_PROJECT_DIR/scripts/misc/artifactory_upload.sh edge-sdks/sdk-for-typescript/test-reports/$CI_JOB_NAME/$CI_JOB_ID/${CI_JOB_NAME}_test_reports.tar.gz ${CI_JOB_NAME}_test_reports.tar.gz - only: - refs: - - master - - schedules - variables: - - $NIGHTLY - artifacts: - when: always - paths: - - ./*.gz - - ./*.html - - cache - - heaptrack - - reports - expire_in: 1 year # save our archives for 1 year as job artifacts - -weekly_perf_test_job: - stage: test - tags: - - docker-prod - image: ${DOCKER_REGISTRY}/${DOCKER_IMAGE_PERFORMANCE}:${DOCKER_IMAGE_PERFORMANCE_VERSION} - script: - - $CI_PROJECT_DIR/scripts/linux/wv/gitlab_test_performance_extended.sh - only: - refs: - - master - - schedules - variables: - - $WEEKLY - artifacts: - when: always - paths: - - ./*.gz - - ./*.html - - cache - - heaptrack - - reports - expire_in: 1 year # save our archives for 1 year as job artifacts - - # next stages will run on NV and WK jobs - -generate_perf_report_job: - stage: translate_report - tags: - - docker-prod - image: python:3.6 - when: always - before_script: - - pip install junit2html - script: - - if [ "$NIGHTLY" == "1" ]; then cat heaptrack_report.html >> reports/index.html; fi - - if [ "$WEEKLY" == "1" ]; then cat heaptrack_report.html >> reports/index.html; fi - - mkdir -p .public - - cp reports/*.html .public/ - artifacts: - paths: - - .public - only: - refs: - - master - - schedules - variables: - - $WEEKLY - - $NIGHTLY - -pages: - stage: deploy - tags: - - docker-prod - when: always - script: mv .public public - artifacts: - paths: - - public - expire_in: 1 year - only: - refs: - - master - - schedules - variables: - - $WEEKLY - - $NIGHTLY diff --git a/scripts/linux/nv/gitlab_test_performance.sh b/scripts/linux/nv/gitlab_test_performance.sh deleted file mode 100755 index c53279cf..00000000 --- a/scripts/linux/nv/gitlab_test_performance.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash -ex -# -# Copyright (C) 2019 HERE Europe B.V. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# License-Filename: LICENSE - -# For core dump backtrace -ulimit -c unlimited - -# Build the SDK -echo ">>> Building ... >>>" -npm install -g yarn -yarn -yarn bootstrap -npm run build -echo ">>> Building SDK done... >>>" - -(cd tests/performance/ && npx tsc) - -# Start local server -echo ">>> Local Server starting for further performance test ... >>>" -node tests/utils/mocked-olp-server/server.js > /dev/null 2>/dev/null & SERVER_PID=$! - -# Node can start server in 1 second, but not faster. -# Add waiter for server to be started. No other way to solve that. -# Curl returns code 1 - means server still down. Curl returns 0 when server is up -RC=1 -while [[ ${RC} -ne 0 ]]; -do - set +e - curl -s http://localhost:3000 - RC=$? - sleep 0.15 - set -e -done -echo ">>> Local Server started for further performance test ... >>>" - -echo ">>> Start performance tests ... >>>" - -heaptrack node ./tests/performance/shortMemoryTest.js 2>> errors.txt || TEST_FAILURE=1 -mv heaptrack.node.* short_test.gz - -heaptrack node ./tests/performance/shortCacheInMemoryTest.js 2>> errors.txt || TEST_FAILURE=1 -mv heaptrack.node.* short_cacheInMemory_test.gz - -echo ">>> Finished performance tests . >>>" - -if [[ ${TEST_FAILURE} == 1 ]]; then - echo "Printing error.txt ###########################################" - cat errors.txt - echo "End of error.txt #############################################" - echo "CRASH ERROR. One of test groups contains crash. Report was not generated for that group ! " -else - echo "OK. Full list of tests passed. " -fi - -mkdir -p reports/heaptrack -mkdir heaptrack - -# Third party dependency needed for pretty graph generation below -git clone --depth=1 https://github.com/brendangregg/FlameGraph - -heaptrack_print --print-leaks \ - --print-flamegraph heaptrack/flamegraph_short_test.data \ - --file short_test.gz > reports/heaptrack/report_short_test.txt - # Pretty graph generation - ./FlameGraph/flamegraph.pl --title="Flame Graph: short_test" heaptrack/flamegraph_short_test.data > reports/heaptrack/flamegraph_short_test.svg - cat reports/heaptrack/flamegraph_short_test.svg >> heaptrack_report.html - -cp heaptrack_report.html reports - -heaptrack_print --print-leaks \ - --print-flamegraph heaptrack/flamegraph_short_cacheInMemory_test.data \ - --file short_cacheInMemory_test.gz > reports/heaptrack/report_cacheInMemory_short_test.txt - # Pretty graph generation - ./FlameGraph/flamegraph.pl --title="Flame Graph: short_test" heaptrack/flamegraph_short_cacheInMemory_test.data > reports/heaptrack/flamegraph_short_cacheInMemory_test.svg - cat reports/heaptrack/flamegraph_short_cacheInMemory_test.svg >> heaptrack_cacheInMemory_report.html - -cp heaptrack_cacheInMemory_report.html reports - -ls -la heaptrack -ls -la - -# Gracefully stop local server -kill -15 ${SERVER_PID} -# Waiter for all processes to be exited correctly -wait diff --git a/scripts/linux/nv/gitlab_test_publish_dryrun.sh b/scripts/linux/nv/gitlab_test_publish_dryrun.sh deleted file mode 100755 index b156e81b..00000000 --- a/scripts/linux/nv/gitlab_test_publish_dryrun.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -ex -# -# Copyright (C) 2019-2020 HERE Europe B.V. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# License-Filename: LICENSE - -# Test the publish process of SDK, by running dry run test -echo ">>> Testing ... >>>" - -# olp-sdk-authentication publish test dry-run -cd @here/olp-sdk-authentication && npm install && npm publish --dry-run && cd - - -# olp-sdk-dataservice-read publish test dry-run -cd @here/olp-sdk-dataservice-read && npm install && npm publish --dry-run && cd - - -# olp-sdk-dataservice-api publish test dry-run -cd @here/olp-sdk-dataservice-api && npm install && npm publish --dry-run && cd - - -# olp-sdk-fetch publish test dry-run -cd @here/olp-sdk-fetch && npm install && npm publish --dry-run && cd - - -# olp-sdk-dataservice-write publish test dry-run -cd @here/olp-sdk-dataservice-write && npm install && npm publish --dry-run && cd - - -echo ">>> Testing SDK done... >>>" diff --git a/scripts/linux/wv/gitlab_test_performance_extended.sh b/scripts/linux/wv/gitlab_test_performance_extended.sh deleted file mode 100755 index b71f65f4..00000000 --- a/scripts/linux/wv/gitlab_test_performance_extended.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash -ex -# -# Copyright (C) 2019 HERE Europe B.V. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# License-Filename: LICENSE - -# For core dump backtrace -ulimit -c unlimited - -# Build the SDK -echo ">>> Building ... >>>" -npm install -g yarn -yarn -yarn bootstrap -npm run build -echo ">>> Building SDK done... >>>" - -(cd tests/performance/ && npx tsc) - -# Start local server -echo ">>> Local Server starting for further performance test ... >>>" -node tests/utils/mocked-olp-server/server.js > /dev/null 2>/dev/null & SERVER_PID=$! - -# Node can start server in 1 second, but not faster. -# Add waiter for server to be started. No other way to solve that. -# Curl returns code 1 - means server still down. Curl returns 0 when server is up -RC=1 -while [[ ${RC} -ne 0 ]]; -do - set +e - curl -s http://localhost:3000 - RC=$? - sleep 0.15 - set -e -done -echo ">>> Local Server started for further performance test ... >>>" - -echo ">>> Start performance tests ... >>>" - -heaptrack node ./tests/performance/longMemoryTest.js 2>> errors.txt || TEST_FAILURE=1 -mv heaptrack.node.* long_test.gz - -heaptrack node ./tests/performance/longCacheInMemoryTest.js 2>> errors.txt || TEST_FAILURE=1 -mv heaptrack.node.* long_cacheInMemory_test.gz - -echo ">>> Finished performance tests . >>>" - -if [[ ${TEST_FAILURE} == 1 ]]; then - echo "Printing error.txt ###########################################" - cat errors.txt - echo "End of error.txt #############################################" - echo "CRASH ERROR. One of test groups contains crash. Report was not generated for that group ! " -else - echo "OK. Full list of tests passed. " -fi - -mkdir -p reports/heaptrack -mkdir heaptrack - -# Third party dependency needed for pretty graph generation below -git clone --depth=1 https://github.com/brendangregg/FlameGraph - -heaptrack_print --print-leaks \ - --print-flamegraph heaptrack/flamegraph_long_test.data \ - --file long_test.gz > reports/heaptrack/report_long_test.txt - # Pretty graph generation - ./FlameGraph/flamegraph.pl --title="Flame Graph: long_test" heaptrack/flamegraph_long_test.data > reports/heaptrack/flamegraph_long_test.svg - cat reports/heaptrack/flamegraph_long_test.svg >> heaptrack_report.html - -cp heaptrack_report.html reports - -heaptrack_print --print-leaks \ - --print-flamegraph heaptrack/flamegraph_long_cacheInMemory_test.data \ - --file long_cacheInMemory_test.gz > reports/heaptrack/report_long_cacheInMemory_test.txt - # Pretty graph generation - ./FlameGraph/flamegraph.pl --title="Flame Graph: long_cacheInMemory_test" heaptrack/flamegraph_long_cacheInMemory_test.data > reports/heaptrack/flamegraph_long_cacheInMemory_test.svg - cat reports/heaptrack/flamegraph_long_cacheInMemory_test.svg >> heaptrack_cacheInMemory_report.html - -cp heaptrack_cacheInMemory_report.html reports - - -ls -la heaptrack -ls -la - -# Gracefully stop local server -kill -15 ${SERVER_PID} -# Waiter for all processes to be exited correctly -wait \ No newline at end of file diff --git a/scripts/misc/artifactory_upload.sh b/scripts/misc/artifactory_upload.sh deleted file mode 100755 index 2b87c69a..00000000 --- a/scripts/misc/artifactory_upload.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -e - -## -# Upload any file to HERE Artifactory -# -# Requirements: curl, $ARTIFACTORY_BOT_USER and $ARTIFACTORY_BOT_HTTP_PASSWORD as environmental variable. -# Usage: artifactory_upload.sh -# -# Script require parameters: -# param 1 - full URL to file on Artifactory -# param 2 - full or relative path to file on Disk -## -if [[ $1 == "--help" || $1 == "-help" ]]; then - echo "" - echo "### Artifactory upload script help ###" - echo "Usage: artifactory_upload.sh " - echo "" - echo "- is repo path+filename on HERE Artifactory (https://${ARTIFACTORY_HOST}/artifactory) : " - echo "for example, product/releases/test-reports/job-type/test/0/CHANGELOG.md " - echo "" - echo "- is full or relative path+filename on your local disk : " - echo "for example, /workspace/CHANGELOG.md or ./CHANGELOG.md " - echo "Be aware if file exist, it will be overwritten." - echo "### ... ###" - echo "" - exit 0 -fi - -if [[ ! -z "$ARTIFACTORY_BOT_USER" && ! -z "$ARTIFACTORY_BOT_HTTP_PASSWORD" ]]; then - if [[ $(which curl &>/dev/null;echo $?) -eq 0 ]]; then - if ! [[ $1 == " " || $2 == " " || $1 == "" || $2 == "" ]]; then - curl -u $ARTIFACTORY_BOT_USER:$ARTIFACTORY_BOT_HTTP_PASSWORD -X PUT https://${ARTIFACTORY_HOST}/artifactory/$1 -T $2 - echo "File uploaded to $1" - exit 0 - else - echo "Parameter 1 an 2 are not specified, please run: artifactory_upload.sh --help " - exit 1 - fi - else - echo "No curl installed. Visit: https://curl.se/download.html " - exit 127 - fi -else - echo "Variables ARTIFACTORY_BOT_USER/ARTIFACTORY_BOT_HTTP_PASSWORD are not defined" - exit 1 -fi diff --git a/scripts/misc/full_report_generation.sh b/scripts/misc/full_report_generation.sh deleted file mode 100755 index 12b78010..00000000 --- a/scripts/misc/full_report_generation.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -ex -# -# Copyright (C) 2021 HERE Europe B.V. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# License-Filename: LICENSE - -# Simple script that create full regression html report with list of different tests - -# Replacing default title with specific test group names -sed -i -e 's/Reports\ Matrix/API-Breaks\ Test\ Report/g' ./api-breaks-index.html -sed -i -e 's/Reports\ Matrix/Fetch\ Test\ Report/g' ./fetch-index.html -sed -i -e 's/Reports\ Matrix/Integration\ Test\ Report/g' ./integration-index.html -sed -i -e 's/Reports\ Matrix/Functional\ Test\ Report/g' ./functional-index.html -sed -i -e 's/Reports\ Matrix/Authentication\ Test\ Report/g' ./authentication-index.html -sed -i -e 's/Reports\ Matrix/Core\ Test\ Report/g' ./core-index.html -sed -i -e 's/Reports\ Matrix/Dataservice-api\ Test\ Report/g' ./dataservice-api-index.html -sed -i -e 's/Reports\ Matrix/Dataservice-read\ Test\ Report/g' ./dataservice-read-index.html -sed -i -e 's/Reports\ Matrix/Dataservice-write\ Test\ Report/g' ./dataservice-write-index.html - -# Creating new html files -echo " Full Regression Test Report

Full Regression Test Report

" > ./index.html -# Authentication -cat ./authentication-index.html >> ./index.html -# Core -cat ./core-index.html >> ./index.html -# DS API -cat ./dataservice-api-index.html >> ./index.html -# DS READ -cat ./dataservice-read-index.html >> ./index.html -# DS WRITE -cat ./dataservice-write-index.html >> ./index.html -# Fetch -cat ./fetch-index.html >> ./index.html -# Functional -cat ./functional-index.html >> ./index.html -# Integration -cat ./integration-index.html >> ./index.html -# API-Breaks -cat ./api-breaks-index.html >> ./index.html - -mkdir -p .public -cp index.html .public/ -