Skip to content

Commit

Permalink
[GHA] OV provider interface updates (openvinotoolkit#26636)
Browse files Browse the repository at this point in the history
  • Loading branch information
akladiev authored Sep 25, 2024
1 parent 1e3160d commit 4bfe725
Show file tree
Hide file tree
Showing 9 changed files with 244 additions and 103 deletions.
10 changes: 7 additions & 3 deletions .github/actions/common/artifact_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import argparse
import os
from pathlib import Path
from .constants import EventType, ProductType
from .constants import EventType, ProductType, PlatformKey


def add_common_args(parser: argparse.ArgumentParser):
Expand All @@ -12,10 +12,14 @@ def add_common_args(parser: argparse.ArgumentParser):
default=os.getenv('GITHUB_BASE_REF') or os.getenv('GITHUB_REF_NAME'))
parser.add_argument('-e', '--event_name', help='Name of GitHub event', required=False,
default=os.getenv('GITHUB_EVENT_NAME'))
parser.add_argument('--storage_dir', help='Subdirectory name for artifacts, same as product type', required=True,
choices=[product_type.value for product_type in ProductType])
parser.add_argument('--storage_root', help='Root path of the artifacts storage', required=False,
default=os.getenv('ARTIFACTS_SHARE'))
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('-d', '--storage_dir', help='Subdirectory name for artifacts, same as product type',
choices=[platform_key.value for platform_key in ProductType])
group.add_argument('-p', '--platform', type=str,
help='Platform for which to restore artifacts. Used if storage_dir is not set',
choices=[product_type.value for product_type in PlatformKey])


def get_event_type(event_name: str = os.getenv('GITHUB_EVENT_NAME')) -> str:
Expand Down
24 changes: 24 additions & 0 deletions .github/actions/common/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,27 @@ class EventType(Enum):
'public_windows_vs2019_Debug',
)
ProductType = Enum('ProductType', {t.upper(): t for t in productTypes})


platformKeys = (
'centos7_x86_64',
'debian10_armhf',
'rhel8_x86_64',
'ubuntu20_arm64',
'ubuntu20_x86_64',
'ubuntu22_x86_64',
'ubuntu24_x86_64',
'macos_12_6_arm64',
'macos_12_6_x86_64',
'windows_x86_64',
)
PlatformKey = Enum('PlatformKey', {t.upper(): t for t in platformKeys})

PlatformMapping = {
PlatformKey.DEBIAN10_ARMHF: ProductType.PUBLIC_LINUX_DEBIAN_10_ARM_RELEASE,
PlatformKey.UBUNTU20_X86_64: ProductType.PUBLIC_LINUX_UBUNTU_20_04_X86_64_RELEASE,
PlatformKey.UBUNTU20_ARM64: ProductType.PUBLIC_LINUX_UBUNTU_20_04_ARM64_RELEASE,
PlatformKey.UBUNTU22_X86_64: ProductType.PUBLIC_LINUX_UBUNTU_22_04_X86_64_RELEASE,
PlatformKey.UBUNTU24_X86_64: ProductType.PUBLIC_LINUX_UBUNTU_24_04_X86_64_RELEASE,
PlatformKey.WINDOWS_X86_64: ProductType.PUBLIC_WINDOWS_VS2019_RELEASE,
}
157 changes: 96 additions & 61 deletions .github/actions/openvino_provider/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ name: 'OpenVINO provider'
description: 'Provides OpenVINO build artifacts from a requested source'
inputs:
revision:
description: 'Version of OpenVINO to provide - latest_available | HEAD | specific commit hash'
description: 'Version of OpenVINO to provide - latest_available_commit | HEAD | specific commit hash |
latest_nightly | specific package version (e.g. 2024.4.0rc2)'
required: true
branch_name:
description: 'Branch of OpenVINO to take the revision from if no specific hash was provided.
Expand All @@ -11,21 +12,25 @@ inputs:
ov_artifact_name:
description: "Name under which to upload provided OpenVINO build artifacts, set automatically by default"
required: false
platform:
description: "Platform for which to get artifacts: centos7 | debian10 | rhel8 | ubuntu20 |
ubuntu20 | ubuntu22 | ubuntu24 | macos | macos_12_6 | windows"
required: false
arch:
description: "Architecture for which to get artifacts: x86_64 | armhf | arm64"
default: "x86_64"
required: false
install_dir:
description: 'Local path to install OpenVINO package to. If not specified, package is installed to GitHub workspace
and uploaded to GitHub'
required: false
commit_packages_to_provide:
description: "Comma-separated list of OpenVINO packages from post-commit to provide ('all' to get everything)"
required: false
commit_build_key:
description: 'public_linux_ubuntu_20_04_release | public_linux_ubuntu_22_04_release |
public_linux_ubuntu_24_04_release | public_windows_vs2019_release'
required: false
commit_share_path:
description: 'Path to share holding OpenVINO post-commit artifacts'
required: false
default: '/mount/build-artifacts'
nightly_build_key:
description: 'centos7_x86_64 | debian9_armhf | rhel8_x86_64 | ubuntu20_arm64 | ubuntu20_x86_64 | ubuntu22_x86_64 |
ubuntu24_x86_64 | macos_12_6_arm64 | macos_12_6_x86_64 | windows_x86_64'
required: false
nightly_package_source:
description: 'Source URL hosting OpenVINO nightly archives'
required: false
Expand All @@ -34,40 +39,44 @@ inputs:
description: 'Source URL hosting OpenVINO wheels, for passing to pip as --extra-index-url'
required: false
default: 'https://storage.openvinotoolkit.org/simple/wheels'
package_url:
description: 'Direct URL to OpenVINO archive to use, if neither post-commit nor nightly archives are required'
required: false

outputs:
ov_artifact_name:
description: "Name of the artifact to upload OpenVINO build artifacts under"
value: ${{ steps.openvino_download.outputs.ov_artifact_name || steps.openvino_commit_output.outputs.ov_artifact_name }}
ov_package_path:
description: "Path to OpenVINO core package relative to the root of uploaded artifacts"
value: ${{ steps.openvino_download.outputs.ov_package_path || steps.openvino_commit_output.outputs.ov_package_path }}
ov_wheel_input:
description: "Input for pip to install OpenVINO python wheel"
value: ${{ steps.openvino_wheel.outputs.ov_wheel_input || steps.openvino_commit_wheel.outputs.ov_wheel_input }}
value: ${{ steps.openvino_s3_download.outputs.ov_artifact_name || steps.openvino_commit_output.outputs.ov_artifact_name }}
ov_wheel_source:
description: "Pip option for custom OV wheel location (--find-links or --extra-index-url)"
value: ${{ steps.openvino_wheel.outputs.ov_wheel_source || steps.openvino_commit_wheel.outputs.ov_wheel_source }}
value: ${{ steps.openvino_s3_wheel.outputs.ov_wheel_source || steps.openvino_commit_wheel.outputs.ov_wheel_source }}
ov_version:
description: "OpenVINO product version"
value: ${{ steps.openvino_s3_wheel.outputs.ov_version || steps.openvino_commit_wheel.outputs.ov_version }}

runs:
using: "composite"
steps:
- name: Verify parameters
if: ${{ !inputs.package_url && !inputs.nightly_build_key && !inputs.commit_build_key }}
- name: Identify artifacts source
id: get_source
shell: bash
run: |
echo "Please, set one parameter for action - nightly_build_key, commit_build_key or package_url"
exit 1
artifacts_source="postcommit"
if [[ "${{ inputs.revision }}" == "latest_nightly" || "${{ inputs.revision }}" =~ ^[0-9]{4}\.[0-9]+\.*[0-9]* ]]; then
artifacts_source="s3"
fi
echo "artifacts_source=$artifacts_source" >> $GITHUB_OUTPUT
- name: Get action revision
id: get_action_revision
run: echo "action_ref=${{ env.ACTION_REF }}" >> $GITHUB_OUTPUT
shell: bash
env:
ACTION_REF: ${{ github.action_ref || github.base_ref || github.event.merge_group.base_ref || github.ref }}

- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
repository: 'openvinotoolkit/openvino'
ref: ${{ inputs.branch_name || github.base_ref || github.event.merge_group.base_ref || github.ref }}
ref: ${{ steps.get_action_revision.outputs.action_ref }}
sparse-checkout: .github/actions

# --- Post-commit case ---
- name: Clone OpenVINO to get HEAD commit
if: inputs.revision == 'HEAD'
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
Expand All @@ -84,18 +93,25 @@ runs:
head=$(git -C openvino rev-parse HEAD)
echo "head=$head" >> $GITHUB_OUTPUT
- name: Create install dir
if: inputs.install_dir
shell: bash
run: |
mkdir -p ${{ inputs.install_dir }}
# --- Post-commit case ---
- name: Download post-commit OpenVINO archives
id: openvino_commit_download
if: inputs.commit_build_key
if: steps.get_source.outputs.artifacts_source == 'postcommit'
uses: ./.github/actions/restore_artifacts
with:
storage_dir: ${{ inputs.commit_build_key }}
platform: ${{ inputs.platform }}_${{ inputs.arch }}
storage_root: ${{ inputs.commit_share_path }}
event_name: "commit"
trigger_repo_sha: ${{ env.OV_REVISION }}
branch_name: ${{ inputs.branch_name }}
artifacts_key: ${{ inputs.ov_artifact_name || inputs.commit_build_key }}
to_restore: ${{ inputs.commit_packages_to_provide }}
target_dir: ${{ inputs.install_dir }}
env:
OV_REVISION: ${{ inputs.revision == 'HEAD' && steps.get_openvino_head_commit.outputs.head || inputs.revision }}

Expand All @@ -104,79 +120,98 @@ runs:
if: steps.openvino_commit_download.outcome == 'success'
shell: bash
run: |
ov_artifact_name=${{ inputs.ov_artifact_name || steps.openvino_commit_download.outputs.restored_artifacts_key }}
cd ${{ steps.openvino_commit_download.outputs.artifacts_workspace_path }}
ov_artifact_name=${{ inputs.ov_artifact_name || inputs.platform }}
workspace_path=${{ steps.openvino_commit_download.outputs.artifacts_path }}
cd $workspace_path
ov_package_path=$(find . -name 'openvino_package*' -printf "%P\n" -quit)
[ -z "$ov_package_path" ] && echo "No openvino_package found in restored artifacts" && exit 1
mv $ov_package_path/* . && rm -rf $ov_package_path
echo "ov_artifact_name=$ov_artifact_name" >> $GITHUB_OUTPUT
echo "ov_package_path=$ov_package_path" >> $GITHUB_OUTPUT
echo "ov_package_path=$workspace_path" >> $GITHUB_OUTPUT
- name: Get commit wheel
id: openvino_commit_wheel
if: steps.openvino_commit_download.outcome == 'success'
shell: bash
run: |
cd ${{ steps.openvino_commit_download.outputs.artifacts_workspace_path }}
artifacts_path=${{ steps.openvino_commit_download.outputs.artifacts_path }}
cd $artifacts_path
version=$(yq eval '.components.dldt.custom_params.wheel_product_version' manifest.yml)
wheel_path=${{ steps.openvino_commit_output.outputs.ov_package_path }}/tools
default_find_links_cmd="--find-links=./$wheel_path"
wheel_path=${{ inputs.install_dir && '$artifacts_path/tools' || './tools' }}
default_find_links_cmd="--find-links=$wheel_path"
find_links_cmd=$([[ -n "$PIP_FIND_LINKS" ]] && echo "" || echo "$default_find_links_cmd")
wheel_input="$find_links_cmd openvino==$version"
echo "ov_wheel_input=$wheel_input" >> $GITHUB_OUTPUT
echo "ov_version=$version" >> $GITHUB_OUTPUT
echo "ov_wheel_source=$default_find_links_cmd" >> $GITHUB_OUTPUT
- name: Upload commit OpenVINO archives
if: steps.openvino_commit_download.outcome == 'success' && !inputs.install_dir
uses: actions/upload-artifact@v4
with:
name: ${{ steps.openvino_commit_output.outputs.ov_artifact_name }}
path: ${{ steps.openvino_commit_output.outputs.ov_package_path }}
if-no-files-found: 'error'

# --- S3 case (nightly or direct URL) ---
- name: Verify parameters
if: (inputs.nightly_build_key || inputs.package_url) && inputs.revision != 'latest_available'
# --- S3 case ---
- name: Get a specific S3 package URL
id: get_s3_package
if: steps.get_source.outputs.artifacts_source == 's3' && inputs.revision != 'latest_nightly'
shell: bash
run: |
echo "Nightly and package_url build sources provide only 'latest' artifacts, specific revision not supported"
exit 1
run: >-
python3 -m venv venv && . venv/bin/activate &&
pip install -r $GITHUB_ACTION_PATH/requirements.txt &&
python $GITHUB_ACTION_PATH/get_s3_package.py --product openvino --platform ${{ inputs.platform }} --arch ${{ inputs.arch }}
--version ${{ inputs.revision }}
- name: Download OpenVINO archives
id: openvino_download
if: inputs.nightly_build_key || inputs.package_url
- name: Download nightly OpenVINO archives
id: openvino_s3_download
if: steps.get_source.outputs.artifacts_source == 's3'
shell: bash
run: |
if [ ${{ inputs.package_url }} ]; then
ov_package_url=${{ inputs.package_url }}
if [ ${{ steps.get_s3_package.outcome }} != 'skipped' ]; then
ov_package_url=${{ steps.get_s3_package.outputs.package_url }}
else
ov_package_url=$(curl -s ${{ inputs.nightly_package_source }} | jq -r '.${{ inputs.nightly_build_key }}')
ov_package_url=$(curl -s ${{ inputs.nightly_package_source }} | jq -r '.${{ inputs.platform }}_${{ inputs.arch }}')
fi
cd ${{ inputs.install_dir || env.GITHUB_WORKSPACE }}
package_basename=$(basename $ov_package_url)
wget $ov_package_url --progress=bar:force:noscroll -O $package_basename
package_folder=${package_basename%.*} && mkdir $package_folder
[[ "$package_basename" == *.zip ]] && unzip "$package_basename" -d $package_folder || tar -xvf "$package_basename" -C $package_folder
package_folder=${package_basename%.*}
[[ "$package_basename" == *.zip ]] && unzip "$package_basename" || tar -xvf "$package_basename"
rm $package_basename
if [ ${{ inputs.install_dir }} ]; then
mv $package_folder/* . && rm -rf $package_folder
fi
ov_artifact_name=$([ ${{ inputs.ov_artifact_name }} ] && echo ${{ inputs.ov_artifact_name }} || echo $package_folder)
echo "ov_package_url=$ov_package_url" >> $GITHUB_OUTPUT
echo "ov_artifact_name=$ov_artifact_name" >> $GITHUB_OUTPUT
echo "ov_package_path=$package_folder" >> $GITHUB_OUTPUT
- name: Upload OpenVINO archives
if: steps.openvino_download.outcome == 'success'
if: steps.openvino_s3_download.outcome == 'success' && !inputs.install_dir
uses: actions/upload-artifact@v4
with:
name: ${{ steps.openvino_download.outputs.ov_artifact_name }}
path: ${{ steps.openvino_download.outputs.ov_package_path }}
name: ${{ steps.openvino_s3_download.outputs.ov_artifact_name }}
path: ${{ steps.openvino_s3_download.outputs.ov_package_path }}
if-no-files-found: 'error'

- name: Get wheel
id: openvino_wheel
if: steps.openvino_download.outcome == 'success'
id: openvino_s3_wheel
if: steps.openvino_s3_download.outcome == 'success'
shell: bash
run: |
rc_version=$(echo ${{ inputs.package_url }} |
rc_version=$(echo ${{ steps.openvino_s3_download.outputs.ov_package_url }} |
grep -oP '(?<=pre-release\/)\d{4}\.\d+\.\d+rc\d+' || true)
release_version=$(echo ${{ steps.openvino_download.outputs.ov_package_path }} |
release_version=$(echo ${{ steps.openvino_s3_download.outputs.ov_package_path }} |
grep -oP '_\K\d{4}\.\d+\.\d+')
dev_version=$(echo ${{ steps.openvino_download.outputs.ov_package_path }} |
dev_version=$(echo ${{ steps.openvino_s3_download.outputs.ov_package_path }} |
grep -oP "${release_version//./\\.}\.dev\d{8}" || true)
version=${rc_version:-${dev_version:-$release_version}}
extra_index="--pre --extra-index-url ${{ inputs.pip_extra_url }}/nightly \
--extra-index-url ${{ inputs.pip_extra_url }}/pre-release"
wheel_input="$extra_index openvino==$version"
echo "ov_wheel_input=$wheel_input" >> $GITHUB_OUTPUT
echo "ov_version=$version" >> $GITHUB_OUTPUT
echo "ov_wheel_source=$extra_index" >> $GITHUB_OUTPUT
77 changes: 77 additions & 0 deletions .github/actions/openvino_provider/get_s3_package.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
import requests
import re
import argparse
import logging
import sys

from pathlib import Path

sys.path.append(str(Path(__file__).parents[1]))
from common import action_utils


# Function to download the JSON file
def load_json_file(url):
response = requests.get(url)
if response.status_code == 200:
return response.json()
else:
raise Exception(f"Failed to download the file, status code: {response.status_code}")


# Function to recursively gather all file paths from the JSON structure
def gather_all_files(node, base_path=''):
all_files = []
if 'children' in node:
for child in node['children']:
new_base = f"{base_path}/{child['name']}"
if 'children' in child:
all_files += gather_all_files(child, new_base)
else:
all_files.append(new_base)
return all_files


# Function to filter files based on the product, version, platform, architecture, and folder
def filter_files_by_criteria(files, product, version_pattern, platform, arch, folder):
matching_files = []
for file in files:
if re.search(
fr"{product}/packages/{folder}(/?.*/)?{version_pattern}/.*{platform}.*{arch}.*\.(tar\.gz|tgz|zip)$",
file):
matching_files.append(file)
return matching_files


# Main function to load the JSON, gather file paths, and filter based on criteria
def main(product, version_pattern, platform, arch, folder):
action_utils.init_logger()
logger = logging.getLogger(__name__)

url = 'https://storage.openvinotoolkit.org/filetree.json'
filetree = load_json_file(url)
all_files = gather_all_files(filetree)
matching_files = filter_files_by_criteria(all_files, product, version_pattern, platform, arch, folder)
if matching_files:
logger.info(f"Matching packages: {sorted(matching_files)}")
package_url = f"https://storage.openvinotoolkit.org{sorted(matching_files)[-1]}"
logger.info(f"Returning package URL: {package_url}")
action_utils.set_github_output("package_url", package_url)
else:
logger.error("No matching files found.")
sys.exit(1)


if __name__ == "__main__":
parser = argparse.ArgumentParser(
description='Search OpenVINO archives based on product, version, platform, architecture, and folder.')
parser.add_argument('--product', required=True, choices=['openvino', 'openvino_genai', 'openvino_tokenizers'],
help='Product name')
parser.add_argument('--version', required=True, help='Version pattern (supports regex)')
parser.add_argument('--platform', default='ubuntu22', help='Platform (default: ubuntu22)')
parser.add_argument('--arch', default='x86_64', help='Architecture (default: x86_64)')
parser.add_argument('--folder', default='(.*/)?', help='Folder type (e.g., pre-release, nightly, default: (.*)?')

args = parser.parse_args()

main(args.product, args.version, args.platform, args.arch, args.folder)
1 change: 1 addition & 0 deletions .github/actions/openvino_provider/requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
requests~=2.32
Loading

0 comments on commit 4bfe725

Please sign in to comment.