diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 00000000..e38afff6 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: Apache-2.0 + +name: E2E test + +on: + push: + branches: + - "main" + - "release-**" + paths: + - '**.py' + - 'pyproject.toml' + - 'requirements*.txt' + - '.github/workflows/e2e.yml' + pull_request: + branches: + - "main" + - "release-**" + paths: + - '**.py' + - 'pyproject.toml' + - 'requirements*.txt' + - '.github/workflows/e2e.yml' + +jobs: + e2e: + runs-on: ubuntu-gpu + + permissions: + pull-requests: write + + steps: + # No step-security/harden-runner since this is a self-hosted runner + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + # https://github.com/actions/checkout/issues/249 + fetch-depth: 0 + + - name: Install Packages + run: | + sudo apt-get install -y cuda-toolkit git cmake build-essential virtualenv + nvidia-smi + ls -l /dev/nvidia* + + - name: Setup Python 3.11 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + with: + python-version: 3.11 + cache: pip + cache-dependency-path: | + **/pyproject.toml + **/requirements*.txt + + - name: Remove llama-cpp-python from cache + run: | + pip cache remove llama_cpp_python + + - name: Cache huggingface + uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: ~/.cache/huggingface + # config contains DEFAULT_MODEL + key: huggingface-${{ hashFiles('src/instructlab/configuration.py') }} + + - name: Install instructlab and libraries + run: | + export PATH="/home/runner/.local/bin:/usr/local/cuda/bin:$PATH" + python3 -m venv venv + . venv/bin/activate + git clone https://github.com/instructlab/instructlab.git + cd instructlab + sed 's/\[.*\]//' requirements.txt > constraints.txt + python3 -m pip cache remove llama_cpp_python + CMAKE_ARGS="-DLLAMA_CUBLAS=on" python3 -m pip install --no-binary llama_cpp_python -c constraints.txt llama_cpp_python + # needed for --4-bit-quant option to ilab train + python3 -m pip install bitsandbytes + # install instructlab + python3 -m pip install . + cd .. + # Install instructlab-sdg + python3 -m pip install . + + - name: Run e2e test + run: | + . venv/bin/activate + ./instructlab/scripts/basic-workflow-tests.sh -cm + + - name: Remove llama-cpp-python from cache + if: always() + run: | + pip cache remove llama_cpp_python diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml new file mode 100644 index 00000000..9f585f6f --- /dev/null +++ b/.github/workflows/spellcheck.yml @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: Apache-2.0 + +name: Spellcheck + +on: + push: + branches: + - "main" + paths: + - '**.md' + - '.github/workflows/spellcheck.yml' # This workflow + pull_request: + branches: + - "main" + paths: + - '**.md' + - '.github/workflows/spellcheck.yml' # This workflow + +env: + LC_ALL: en_US.UTF-8 + +defaults: + run: + shell: bash + +permissions: + contents: read + +jobs: + spellcheck: + name: Spellcheck (en_US) + runs-on: ubuntu-latest + steps: + - name: "Harden Runner" + uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + with: + egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs + + - name: "Checkout" + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 + + - name: Spellcheck + uses: rojopolis/spellcheck-github-actions@d354a4dc525c8067555c7481b60416cedb0060ff # v0.38.0