-
Notifications
You must be signed in to change notification settings - Fork 423
200 lines (177 loc) · 8.48 KB
/
macos-job.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
name: Set up conda environment for testing
description: Clean workspace and check out PyTorch
inputs:
python-version:
description: If set to any value, dont use sudo to clean the workspace
required: false
type: string
default: "3.9"
miniconda-version:
description: Miniconda version to install
required: false
type: string
default: "23.1.0-1"
environment-file:
description: Environment file to install dependencies from
required: false
type: string
default: ""
pip-requirements-file:
description: An optional pip requirements file to be installed in the conda environment
required: false
type: string
default: ""
runs:
using: composite
steps:
# Use the same trick from https://github.com/marketplace/actions/setup-miniconda
# to refresh the cache daily. This is kind of optional though
- name: Get date
id: get-date
shell: bash
run: |
echo "today=$(/bin/date -u '+%Y%m%d')d" >> "${GITHUB_OUTPUT}"
- name: Setup miniconda cache
id: miniconda-cache
uses: actions/cache@v3
with:
path: ${{ runner.temp }}/miniconda
key: miniconda-${{ runner.os }}-${{ runner.arch }}-${{ inputs.python-version }}-${{ steps.get-date.outputs.today }}
- name: Install miniconda (${{ inputs.miniconda-version }})
if: steps.miniconda-cache.outputs.cache-hit != 'true'
env:
MINICONDA_VERSION: ${{ inputs.miniconda-version }}
shell: bash -l {0}
run: |
MINICONDA_INSTALL_PATH="${RUNNER_TEMP}/miniconda"
mkdir -p "${MINICONDA_INSTALL_PATH}"
case ${RUNNER_OS}-${RUNNER_ARCH} in
Linux-X64)
MINICONDA_ARCH="Linux-x86_64"
;;
macOS-ARM64)
MINICONDA_ARCH="MacOSX-arm64"
;;
macOS-X64)
MINICONDA_ARCH="MacOSX-x86_64"
;;
*)
echo "::error::Platform ${RUNNER_OS}-${RUNNER_ARCH} currently unsupported using this action"
exit 1
;;
esac
MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-py39_${MINICONDA_VERSION}-${MINICONDA_ARCH}.sh"
curl -fsSL "${MINICONDA_URL}" -o "${MINICONDA_INSTALL_PATH}/miniconda.sh"
bash "${MINICONDA_INSTALL_PATH}/miniconda.sh" -b -u -p "${MINICONDA_INSTALL_PATH}"
rm -rf "${MINICONDA_INSTALL_PATH}/miniconda.sh"
- name: Update GitHub path to include miniconda install
shell: bash
run: |
set -x
MINICONDA_INSTALL_PATH="${RUNNER_TEMP}/miniconda"
echo "${MINICONDA_INSTALL_PATH}/bin" >> $GITHUB_PATH
# NB: GITHUB_PATH has a lower priority than PATH, so also set the path
# here to make sure that the correct conda is used
echo "PATH=${MINICONDA_INSTALL_PATH}/bin:${PATH}" >> $GITHUB_ENV
# When the environment-file or pip-requirements-file inputs are not set or are set to invalid paths, the hashFiles
# function will return an empty string without failing the step. This works out nicely and we can have a various
# combination of cache key such as:
# - Both are missing or invalid: miniconda-env-macOS-ARM64-20221022d--
# - Both are set: miniconda-env-macOS-ARM64-20221022d-HASH(environment-file)-HASH(pip-requirements-file)
# - The first one is missing or invalid: miniconda-env-macOS-ARM64-20221022d--HASH(pip-requirements-file)
# - The second one is missing or invalid: miniconda-env-macOS-ARM64-20221022d-HASH(environment-file)-
#
# There is no need to skip or run actions/cache with complicated logic
- name: Setup miniconda env cache
id: miniconda-env-cache
uses: actions/cache@v3
with:
path: ${{ runner.temp }}/conda-python-${{ inputs.python-version }}
key: miniconda-env-${{ runner.os }}-${{ runner.arch }}-${{ inputs.python-version }}-${{ steps.get-date.outputs.today }}-${{ hashFiles(inputs.environment-file) }}-${{ hashFiles(inputs.pip-requirements-file) }}
- name: Setup conda environment with python (v${{ inputs.python-version }})
if: steps.miniconda-env-cache.outcome == 'success' && steps.miniconda-env-cache.outputs.cache-hit != 'true'
shell: bash
env:
PYTHON_VERSION: ${{ inputs.python-version }}
ENV_FILE: ${{ inputs.environment-file }}
PIP_REQUIREMENTS_FILE: ${{ inputs.pip-requirements-file }}
run: |
set -x
CONDA_BASE_ENV="${RUNNER_TEMP}/conda-python-${PYTHON_VERSION}"
ENV_FILE_FLAG=""
if [[ -f "${ENV_FILE}" ]]; then
ENV_FILE_FLAG="--file ${ENV_FILE}"
elif [[ -n "${ENV_FILE}" ]]; then
echo "::warning::Specified env file (${ENV_FILE}) not found, not going to include it"
fi
CONDA_EXTRA_FLAGS=""
if [[ "${PYTHON_VERSION}" == "3.11" ]]; then
CONDA_EXTRA_FLAGS=" -c pytorch-nightly"
fi
# Print the conda we are using here in case we need debugging information
CONDA_RUNTIME=$(which conda)
"${CONDA_RUNTIME}" --version
"${CONDA_RUNTIME}" create \
--yes --quiet \
--prefix "${CONDA_BASE_ENV}" \
${ENV_FILE_FLAG} \
python="${PYTHON_VERSION}" \
cmake=3.22 \
ninja=1.10 \
pkg-config=0.29 \
wheel=0.37 \
${CONDA_EXTRA_FLAGS}
if [[ -f "${PIP_REQUIREMENTS_FILE}" ]]; then
"${CONDA_RUNTIME}" run -p "${CONDA_BASE_ENV}" --no-capture-output python3 -mpip install -r "${PIP_REQUIREMENTS_FILE}"
elif [[ -n "${PIP_REQUIREMENTS_FILE}" ]]; then
echo "::warning::Specified pip requirements file (${PIP_REQUIREMENTS_FILE}) not found, not going to include it"
fi
- name: Clone the base conda environment and update GitHub env
shell: bash
env:
PYTHON_VERSION: ${{ inputs.python-version }}
CONDA_BASE_ENV: ${{ runner.temp }}/conda-python-${{ inputs.python-version }}
PIP_REQUIREMENTS_FILE: ${{ inputs.pip-requirements-file }}
run: |
set -x
# Print the conda we are using here in case we need debugging information
CONDA_RUNTIME=$(which conda)
"${CONDA_RUNTIME}" --version
CONDA_ENV="${RUNNER_TEMP}/conda_environment_${GITHUB_RUN_ID}"
"${CONDA_RUNTIME}" create \
--yes --quiet \
--prefix "${CONDA_ENV}" \
--clone "${CONDA_BASE_ENV}"
set +e
# NB: Cloning sometimes doesn't copied pip dependencies (untracked files) over. If this
# happens, let's attempt to install the pip requirements directly on top of the cloned
# environment. This is to make sure that no dependency is missing.
UNTRACKED_FILES_COUNT=$("${CONDA_RUNTIME}" package -p "${CONDA_ENV}" -u | grep -v "^#" | wc -l | xargs)
set -e
if [[ -z "${UNTRACKED_FILES_COUNT}" ]] || [[ "${UNTRACKED_FILES_COUNT}" == "0" ]]; then
if [[ -f "${PIP_REQUIREMENTS_FILE}" ]]; then
# NB: Force reinstall and don't use the cache, as the installation would still fail
# when reporting that all requirements have been satisfied
"${CONDA_RUNTIME}" run -p "${CONDA_ENV}" --no-capture-output python3 -mpip install --ignore-installed --no-cache-dir -r "${PIP_REQUIREMENTS_FILE}"
elif [[ -n "${PIP_REQUIREMENTS_FILE}" ]]; then
echo "::warning::Specified pip requirements file (${PIP_REQUIREMENTS_FILE}) not found, not going to include it"
fi
fi
echo "CONDA_ENV=${CONDA_ENV}" >> "${GITHUB_ENV}"
echo "CONDA_RUN=${CONDA_RUNTIME} run -p ${CONDA_ENV} --no-capture-output" >> "${GITHUB_ENV}"
if [[ "${PYTHON_VERSION}" == "3.11" ]]; then
# TODO: Remove me, when more packages will be available on default channel
echo "CONDA_INSTALL=${CONDA_RUNTIME} install --yes --quiet -p ${CONDA_ENV} -c pytorch-nightly" >> "${GITHUB_ENV}"
else
echo "CONDA_INSTALL=${CONDA_RUNTIME} install --yes --quiet -p ${CONDA_ENV}" >> "${GITHUB_ENV}"
fi
- name: Reset channel priority
shell: bash
run: |
CONDA_RUNTIME=$(which conda)
set -euxo pipefail
"${CONDA_RUNTIME}" config --set channel_priority false
- name: Setup upterm session
uses: owenthereal/action-upterm@v1
with:
limit-access-to-actor: true