.gitlab-ci.yml

image: docker.uib.gmbh/opsi/dev/pybuilder:deb9-py3.11

stages:
  - test_python
  - build
  - package
  - test_package

pytest-darwin:
  stage: test_python
  tags:
    - catalina-py311-x64
  script:
    - export ZIPLAUNCHER_CLEAN_TMP=true
    - poetry lock --no-update --no-cache
    - poetry install
    - poetry run pytest --tb=short -x -o junit_family=xunit2 --junitxml=testreport_darwin.xml --cov-append --cov opsicli --cov plugins --cov-report term --cov-report xml -v tests
    - mv .coverage coverage_darwin
  artifacts:
    name: "opsi-cli_test_darwin"
    paths:
      - coverage.xml
      - testreport_darwin.xml
      - coverage_darwin
    expire_in: 3 days

pytest-windows:
  stage: test_python
  tags:
    - win10-py311-x86
  script:
    - poetry lock --no-update --no-cache
    - poetry install
    - poetry run pytest --tb=short -x -o junit_family=xunit2 --junitxml=testreport_windows.xml --cov-append --cov opsicli --cov plugins --cov-report term --cov-report xml -v tests
    - Rename-Item -Path .coverage -NewName coverage_windows
  artifacts:
    name: "opsi-cli_test_windows"
    paths:
      - coverage.xml
      - testreport_windows.xml
      - coverage_windows
    expire_in: 3 days

pytest-linux:
  stage: test_python
  services:
    - name: mysql:8.0
      command:
        - --max_connections=1000
        - --default-authentication-plugin=mysql_native_password
    - name: redislabs/redistimeseries:latest
      alias: redis
    - name: docker.uib.gmbh/opsi/opsi-server:4.3-development
      alias: opsiconfd
  variables:
    FF_NETWORK_PER_BUILD: "1"  # Create a new network for each build
    #CI_DEBUG_SERVICES: "true"  # Show debug information for services in ci
    MYSQL_ROOT_PASSWORD: "opsi"
    MYSQL_DATABASE: "opsitest"
    MYSQL_USER: "opsi"
    MYSQL_PASSWORD: "opsi"
    MYSQL_HOST: mysql
    REDIS_HOST: redis
    OPSI_HOST: opsiconfd
    OPSI_ADMIN_PASSWORD: vhahd8usaz # also hardcoded in tests/utils.py
    OPSI_HOST_ROLE: configserver
    OPSI_HOST_ID: "test.uib.gmbh"
    OPSI_TFTPBOOT: "false"
    # when restoring the test backup, this job takes > 60 seconds to start, so gitlab-ci kills it
    #OPSICONFD_RESTORE_BACKUP_URL: "http://binaryindex.uib.gmbh/development/testbackup/all/all/testbackup_all_all_1.1.tar.gz"
    # issue for individually configurable timeouts https://gitlab.com/gitlab-org/gitlab/-/issues/30353
  script:
    - poetry lock --no-update --no-cache
    - poetry install
    - poetry run pytest --tb=short -x -o junit_family=xunit2 --junitxml=testreport_linux.xml --cov-append --cov opsicli --cov plugins --cov-report term --cov-report xml -v tests
    - mv .coverage coverage_linux
  artifacts:
    name: "opsi-cli_test_linux"
    paths:
      - coverage.xml
      - testreport_linux.xml
      - coverage_linux
    expire_in: 3 days

lint-pytest:
  stage: test_python
  needs:
    # We want to combine test coverage from all test jobs
    - job: pytest-windows
      artifacts: true
    - job: pytest-darwin
      artifacts: true
    - job: pytest-linux
      artifacts: true
  script:
    - apt-get update
    - apt-get --yes --allow-downgrades install libsqlite3-0=3.16.2-5+deb9u1
    - apt-get --yes install sqlite3
    - poetry lock --no-update --no-cache
    - poetry install
    - poetry run ruff check opsicli tests
    - poetry run mypy opsicli tests
    - rm -rf .mypy_cache
    - "for plugindir in $(find plugins/*/python/*.py | cut -d'/' -f1-2  | sort | uniq); do echo $plugindir; poetry run mypy --explicit-package-bases $plugindir; done"
    # Convert file.path to a relative path in coverage databases
    - for db in coverage_*; do sqlite3 $db 'update file set path=replace(substr(path,instr(path,"opsicli")),"\","/");'; done
    - for db in coverage_*; do sqlite3 $db 'update file set path=replace(substr(path,instr(path,"plugins")),"\","/");'; done
    - echo -e '[run]\nrelative_files = True\n' > .coveragerc
    - poetry run coverage combine coverage_*
    - poetry run coverage xml
    - poetry run coverage report
    - poetry run opsi-dev-tool -l info --testreport-merge testreport_*.xml testreport.xml
  coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+)%/'
  artifacts:
    name: "opsi-cli_test"
    paths:
      - coverage.xml
    expire_in: 3 days
    reports:
      junit: testreport.xml

linux-x64-pyinstaller:
  stage: build
  script:
    - export LC_ALL=C.UTF-8
    - wget "$OPSIDEVTOOLS_URL_LINUX_X64" -O - | tar xzf -
    - mv opsi-dev-tool opsi-dev-cli
    - ./opsi-dev-cli binary pull ziplaunch
    - poetry lock --no-update --no-cache
    - poetry install
    - poetry run ./opsi-dev-cli -l info pyinstaller build
    - ./ziplaunch/ziplaunch dist/opsicli ./ziplaunch_data/ziplaunch.ini opsi-cli
    - ./opsi-cli --version
    - ./opsi-cli --non-interactive plugin new cicdtest --version=1.0
    - sed -i 's/import rich_click as click/import rich_click as click; import flask/' cicdtest/python/__init__.py
    - ./opsi-cli -l 7 plugin add cicdtest
    - time ./opsi-cli --help
    - mv ./opsi-cli opsi-cli-linux-x64.run
    - '[ "$CI_COMMIT_TAG" = "" ] && ./opsi-dev-cli -l info binary push opsi-cli-linux-x64.run --prerelease="$CI_JOB_ID"'
    - '[ "$CI_COMMIT_TAG" = "" ] || ./opsi-dev-cli -l info binary push opsi-cli-linux-x64.run'
    - mv ./opsi-cli-linux-x64.run CLIENT_DATA/opsi-cli-linux.run
  artifacts:
    name: opsi-cli-linux.run
    paths:
      - CLIENT_DATA/opsi-cli-linux.run

# arm binary pushed to binaryindex, but not included in opsi-package (no arm clients)
linux-arm64-pyinstaller:
  stage: build
  tags:
    - linux-arm64
  script:
    - export LC_ALL=C.UTF-8
    - wget "$OPSIDEVTOOLS_URL_LINUX_ARM64" -O - | tar xzf -
    - mv opsi-dev-tool opsi-dev-cli
    - ./opsi-dev-cli binary pull ziplaunch
    - poetry lock --no-update --no-cache
    - poetry install
    - poetry run ./opsi-dev-cli -l info pyinstaller build
    - ./ziplaunch/ziplaunch dist/opsicli ./ziplaunch_data/ziplaunch.ini opsi-cli
    - ./opsi-cli --version
    - ./opsi-cli --non-interactive plugin new cicdtest --version=1.0
    - sed -i 's/import rich_click as click/import rich_click as click; import flask/' cicdtest/python/__init__.py
    - ./opsi-cli -l 7 plugin add cicdtest
    - time ./opsi-cli --help
    - mv ./opsi-cli opsi-cli-linux-arm64.run
    - '[ "$CI_COMMIT_TAG" = "" ] && ./opsi-dev-cli -l info binary push opsi-cli-linux-arm64.run --prerelease="$CI_JOB_ID"'
    - '[ "$CI_COMMIT_TAG" = "" ] || ./opsi-dev-cli -l info binary push opsi-cli-linux-arm64.run'

macos-x64-pyinstaller:
  stage: build
  tags:
    - catalina-py311-x64
  script:
    - export LC_ALL=en_US.UTF-8
    - poetry lock --no-update --no-cache
    - poetry install
    - curl -o opsi-dev-tools.tar.gz "$OPSIDEVTOOLS_URL_DARWIN_X64"
    - tar -xvf opsi-dev-tools.tar.gz
    - mv opsi-dev-tool opsi-dev-cli
    - ./opsi-dev-cli binary pull ziplaunch
    - poetry run ./opsi-dev-cli -l info pyinstaller build
    #- ./opsi-dev-cli -l info apple sign dist/opsicli

    - version=$(./dist/opsicli/opsi-cli --version | cut -d ' ' -f3)
    - mkdir ziplaunch-build
    - ./opsi-dev-cli apple create-app --identifier org.opsi.opsi-cli --executable-name opsi-cli --version $version ziplaunch-build opsi-cli
    - cp ./ziplaunch_data/macos-start.sh ziplaunch-build/opsi-cli.app/Contents/MacOS/opsi-cli
    - chmod +x ziplaunch-build/opsi-cli.app/Contents/MacOS/opsi-cli
    - mv dist/opsicli ziplaunch-build/opsi-cli.app/Contents/Resources/
    - ziplaunch-build/opsi-cli.app/Contents/Resources/opsicli/opsi-cli --version
    #- ./opsi-dev-cli -l info apple sign-and-notarize --print-log ziplaunch-build/opsi-cli.app
    #- ziplaunch-build/opsi-cli.app/Contents/Resources/opsicli/opsi-cli --version

    - ./ziplaunch/ziplaunch ziplaunch-build ./ziplaunch_data/ziplaunch.ini opsi-cli
    #- ./opsi-cli --version  # is unsigned, will not be allowed to run
    - ./opsi-dev-cli -l info apple sign-and-notarize --macho-fix --print-log opsi-cli

    - ./opsi-cli --version
    - ./opsi-cli --non-interactive plugin new cicdtest --version=1.0
    - sed -i '' 's/import rich_click as click/import rich_click as click; import flask/' cicdtest/python/__init__.py
    - ./opsi-cli -l 7 plugin add cicdtest
    - time ./opsi-cli --help
    - mv ./opsi-cli opsi-cli-macos-x64
    - '[ "$CI_COMMIT_TAG" = "" ] && ./opsi-dev-cli -l info binary push ./opsi-cli-macos-x64 --prerelease="$CI_JOB_ID"'
    - '[ "$CI_COMMIT_TAG" = "" ] || ./opsi-dev-cli -l info binary push ./opsi-cli-macos-x64'
    - mv ./opsi-cli-macos-x64 CLIENT_DATA/opsi-cli-macos
  artifacts:
    name: opsi-cli-macos
    paths:
      - CLIENT_DATA/opsi-cli-macos

windows-x86-pyinstaller:
  stage: build
  tags:
    - win10-py311-x86
  script:
    # Build python environment
    - poetry lock --no-update --no-cache
    - poetry install
    # Get opsi-dev-tool
    - Invoke-WebRequest -UseBasicParsing -Uri "$OPSIDEVTOOLS_URL_WINDOWS_X86" -OutFile opsi-dev-tools.zip
    - Expand-Archive opsi-dev-tools.zip -DestinationPath .
    - Copy-Item opsi-dev-tool.exe opsi-dev-cli.exe
    # Fetch ziplaunch
    - .\opsi-dev-cli.exe binary pull ziplaunch
    # Build opsi-cli binary
    - poetry run .\opsi-dev-cli.exe -l info pyinstaller build
    # Sign opsi-cli binary
    - .\opsi-dev-cli.exe -l info signserver sign dist\opsicli\opsi-cli.exe
    # Build ziplauncher
    - $out = dist\opsicli\opsi-cli.exe --version
    - $version = $out.split()[2]
    - .\opsi-dev-cli.exe patch-file ziplaunch_data\ziplaunch.ini version=$version
    - .\ziplaunch\ziplaunch dist\opsicli .\ziplaunch_data\ziplaunch.ini opsi-cli.exe
    # Test opsi-cli binary
    - .\opsi-cli.exe --version
    - .\opsi-cli.exe --non-interactive plugin new cicdtest --version=1.0
    - (get-content cicdtest\python\__init__.py) | %{$_ -replace "import click","import click; import flask"} | Out-File -encoding ascii cicdtest\python\__init__.py
    - .\opsi-cli.exe -l 7 plugin add cicdtest
    - powershell -command "Measure-Command { .\opsi-cli.exe --help }"
    # Sign ziplauncher binary
    - .\opsi-dev-cli.exe -l info signserver sign opsi-cli.exe
    # Virus check opsi-cli binary and ziplauncher
    - .\opsi-dev-cli.exe -l info --output-format summary virustotal check dist\opsicli\opsi-cli.exe --allow-detections=10
    - .\opsi-dev-cli.exe -l info --output-format summary virustotal check opsi-cli.exe --allow-detections=10
    - move opsi-cli.exe opsi-cli-windows-x86.exe
    # Upload and archive opsi-cli binary
    - if (! $CI_COMMIT_TAG) {.\opsi-dev-cli.exe -l info binary push opsi-cli-windows-x86.exe --prerelease="$CI_JOB_ID"}
    - if ($CI_COMMIT_TAG) {.\opsi-dev-cli.exe -l info binary push opsi-cli-windows-x86.exe}
    - move opsi-cli-windows-x86.exe CLIENT_DATA\opsi-cli-windows.exe
  artifacts:
    name: opsi-cli-windows.exe
    paths:
      - CLIENT_DATA\opsi-cli-windows.exe

makeopsipackage:
  stage: package
  script:
    # Disregard any information from OPSI and write controlfile according to pyproject.toml version and CI_JOB_ID
    - version=$(cat pyproject.toml | grep -e "^version = " | head -n 1 | cut -d \" -f 2)
    - newpackage="1"
    - '[ "$CI_COMMIT_TAG" = "" ] && newpackage="$newpackage.$CI_JOB_ID"'
    - opsi-dev-cli opsi-package set-value packageVersion "$newpackage"
    - opsi-dev-cli opsi-package set-value productVersion "$version"
    - opsi-dev-cli changelog from-git output
    - cp output/changelog.md OPSI/
    - opsi-makepackage --no-set-rights -l 6
    - upload-package --operating-systems windows,linux,macos opsi-cli*.opsi
    - '[ "$CI_COMMIT_TAG" = "" ] || opsi-dev-cli -l debug release-service register-package-version opsi-cli TOOL --files=opsi-cli-windows-x86.exe,opsi-cli-linux-x64.run,opsi-cli-linux-arm64.run,opsi-cli-macos-x64 --version="$version" --changelog-file=OPSI/changelog.md'
  artifacts:
    name: controlfile
    paths:
      - OPSI/control

.runtest: &runtest |
  version=$(opsi-dev-cli opsi-package get-value productVersion)
  package=$(opsi-dev-cli opsi-package get-value packageVersion)
  version="$version-$(echo $package | tr '.' '~')"
  opsi-dev-cli -l info jenkins test --versions="$version,$version,$version,$version,$version"

test_all_systems:
  stage: test_package
  script:
    - *runtest
  only:
    - tags

test_all_systems_manual:
  stage: test_package
  script:
    - *runtest
  when: manual