From 87ea53022c39f3adead2688fbae5698147b279d8 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 9 Jun 2020 15:10:11 -0400 Subject: [PATCH 01/18] Add setuptools and wheel as pip dependencies setuptools usually comes along with pip, but wheel does not. Using wheel where possible to build python extensions is more modern and more security conscious than using setup.py. --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0a8547b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +setuptools +wheel From 7f8d430a309b22a3410949e64ccd60ad5a1dc79f Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 9 Jun 2020 15:19:05 -0400 Subject: [PATCH 02/18] Pull in requirements.txt from requirements-test.txt --- requirements-test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-test.txt b/requirements-test.txt index 416634f..66f74db 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1 +1,2 @@ +--requirement requirements.txt pre-commit From 370f2a2d8b7375c6e529565aa49a41d6499c028e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 15 Jun 2020 17:35:25 -0400 Subject: [PATCH 03/18] Remove requirements that already exist in setup.py to align with a Single Source of Truth. --- requirements-test.txt | 1 - requirements.txt | 1 - 2 files changed, 2 deletions(-) diff --git a/requirements-test.txt b/requirements-test.txt index 47e4ffd..0a6d7d6 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,3 +1,2 @@ --requirement requirements.txt -e .[test] -pre-commit diff --git a/requirements.txt b/requirements.txt index e9d0dcc..d08662f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ -e . -setuptools wheel From d2d05000b14b1d8bffc83040d268f30faf03a9d4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jun 2020 21:46:17 -0400 Subject: [PATCH 04/18] Switch test and build workflows to use a matrix of Python versions. We claim to support Python versions 3.6, 3.7, and 3.8 in setup.py, but we only test and build on Python 3.8. If we claim support for a version then I believe we should be testing and building on that version to ensure "advertised" compatibility. --- .github/workflows/build.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7237b5a..fe5337e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -49,11 +49,14 @@ jobs: run: pre-commit run --all-files test: runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: ${{ matrix.python_version }} - name: Store installed Python version run: | echo "::set-env name=PY_VERSION::"\ @@ -84,11 +87,14 @@ jobs: build: runs-on: ubuntu-latest needs: [lint, test] + strategy: + matrix: + python-version: [3.6, 3.7, 3.8] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: ${{ matrix.python-version }} - name: Store installed Python version run: | echo "::set-env name=PY_VERSION::"\ @@ -112,5 +118,5 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v1 with: - name: dist + name: dist-${{ matrix.python-version }} path: dist From 6f6f5972fac972fc150286d571eeb5aabb44a2fc Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jun 2020 21:56:58 -0400 Subject: [PATCH 05/18] Fix typo and use matrix.python-version instead of env.PY_VERSION. In workflows using the Python version matrix we should just use the version for that run instead of extracting the Python version from the environment. --- .github/workflows/build.yml | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fe5337e..8bf777b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -56,19 +56,15 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: ${{ matrix.python_version }} - - name: Store installed Python version - run: | - echo "::set-env name=PY_VERSION::"\ - "$(python -c "import platform;print(platform.python_version())")" + python-version: ${{ matrix.python-version }} - name: Cache pip test requirements uses: actions/cache@v1 with: path: ${{ env.PIP_CACHE_DIR }} - key: "${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}-\ + key: "${{ runner.os }}-pip-test-py${{ matrix.python-version }}-\ ${{ hashFiles('**/requirements-test.txt') }}" restore-keys: | - ${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}- + ${{ runner.os }}-pip-test-py${{ matrix.python-version }}- ${{ runner.os }}-pip-test- ${{ runner.os }}-pip- - name: Install dependencies @@ -95,18 +91,14 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - name: Store installed Python version - run: | - echo "::set-env name=PY_VERSION::"\ - "$(python -c "import platform;print(platform.python_version())")" - name: Cache pip build requirements uses: actions/cache@v1 with: path: ${{ env.PIP_CACHE_DIR }} - key: "${{ runner.os }}-pip-build-py${{ env.PY_VERSION }}-\ + key: "${{ runner.os }}-pip-build-py${{ matrix.python-version }}-\ ${{ hashFiles('**/requirements.txt') }}" restore-keys: | - ${{ runner.os }}-pip-build-py${{ env.PY_VERSION }}- + ${{ runner.os }}-pip-build-py${{ matrix.python-version }}- ${{ runner.os }}-pip-build- ${{ runner.os }}-pip- - name: Install dependencies From af06915b05674695cb5d9bd8418c6b37f77ec3bc Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 14:57:04 -0400 Subject: [PATCH 06/18] Add tool and docs to automate development environment setup. --- CONTRIBUTING.md | 11 +++ setup-env | 175 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100755 setup-env diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dacaaad..2bd2451 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,6 +46,17 @@ There are a few ways to do this, but we prefer to use create and manage a Python virtual environment specific to this project. +If you already have `pyenv` and `pyenv-virtualenv` configured you can +take advantage of the `setup-env` tool in this repo to automate the +entire environment configuration process. + +```bash +./setup-env +``` + +Otherwise, follow the steps below to manually configure your +environment. + #### Installing and using `pyenv` and `pyenv-virtualenv` #### On the Mac, installation is as simple as `brew install pyenv diff --git a/setup-env b/setup-env new file mode 100755 index 0000000..65c5b6f --- /dev/null +++ b/setup-env @@ -0,0 +1,175 @@ +#!/usr/bin/env bash + +set -o nounset +set -o errexit +set -o pipefail + +USAGE=$(cat << 'END_OF_LINE' +This script is used to configure a developement environment for this repo. + +It does the following: + - Verifies pyenv and pyenv-virtualenv are installed. + - Creates a Python virtual environment. + - Configures the activation of the virtual enviroment for the repo directory. + - Installs the requirements required for development. + - Installs git pre-commit hooks. + - Configures git upstream remote "lineage" repositories. + +usage: setup-env [--force] [--help] [virt_env_name] + +END_OF_LINE +) + +# Flag to force deletion and creation of virtual environment +FORCE=0 + +# Positional parameters +PARAMS="" + +# Parse command line arguments +while (( "$#" )); do + case "$1" in + -f|--force) + FORCE=1 + shift + ;; + -h|--help) + echo "${USAGE}" + exit 0 + ;; + -*) # unsupported flags + echo "Error: Unsupported flag $1" >&2 + exit 1 + ;; + *) # preserve positional arguments + PARAMS="$PARAMS $1" + shift + ;; + esac +done + +# set positional arguments in their proper place +eval set -- "$PARAMS" + +# Check to see if pyenv is installed +if [ -z "$(which pyenv)" ] || [ -z "$(which pyenv-virtualenv)" ]; then + echo "pyenv and pyenv-virtualenv are required." + if [[ "$OSTYPE" == "darwin"* ]]; then + cat << 'END_OF_LINE' + + On the Mac, installation is as simple as "brew install pyenv + pyenv-virtualenv" and adding this to your profile: + + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" + +END_OF_LINE + + fi + cat << 'END_OF_LINE' + For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you don't want + to use "brew") you can use https://github.com/pyenv/pyenv-installer to install + the necessary tools. Before running this ensure that you have installed the + prerequisites for your platform according to the pyenv wiki page, + https://github.com/pyenv/pyenv/wiki/common-build-problems. + + On WSL you should treat your platform as whatever Linux distribution you've + chosen to install. + + Once you have installed "pyenv" you will need to add the following lines to + your ".bashrc": + + export PATH="$PATH:$HOME/.pyenv/bin" + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" +END_OF_LINE + exit 1 +fi + +set +o nounset +# Determine the virtual environment name +if [ "$1" ]; then + # Use the user-provided environment name + env_name=$1 +else + # Set the environment name to the last part of the working directory. + env_name=${PWD##*/} +fi +set -o nounset + +# Remove any lingering local configuration. +if [ $FORCE -ne 0 ]; then + rm -f .python-version + pyenv virtualenv-delete --force "${env_name}" || true +elif [[ -f .python-version ]]; then + cat << 'END_OF_LINE' + An existing .python-version file was found. Either remove this file yourself + or re-run with --force option to have it deleted along with the associated + virtual environment. + + rm .python-version + +END_OF_LINE + exit 1 +fi + +# Create a new virutal environment for this project +if ! pyenv virtualenv "${env_name}"; then + cat << END_OF_LINE + An existing virtual environment named $env_name was found. Either delete this + environment yourself or re-run with --force option to have it deleted. + + pyenv virtualenv-delete ${env_name} + +END_OF_LINE + exit 1 +fi + +# Activate the new virtual environment +pyenv local "${env_name}" + +# Upgrade pip and friends +python -m pip install --upgrade pip setuptools wheel + +# Find a requirements file (if possible) and install +for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt"; do + if [[ -f $req_file ]] + then + pip install -r $req_file + break + fi +done + +# Install git pre-commit hooks +pre-commit install + +# Setup git remotes from lineage configuration +# This could fail if the remotes are already setup, but that is ok. +set +o errexit + +eval "$(python3 << 'END_OF_LINE' +from pathlib import Path +import yaml +import sys + +LINEAGE_CONFIG = Path(".github/lineage.yml") + +if not LINEAGE_CONFIG.exists(): + print('No lineage configuration found.', file=sys.stderr) + sys.exit(0) + +with LINEAGE_CONFIG.open("r") as f: + lineage = yaml.safe_load(stream=f) + +if lineage["version"] == "1": + for parent_name, v in lineage["lineage"].items(): + remote_url = v["remote-url"] + print(f"git remote add {parent_name} {remote_url};") + print(f"git remote set-url --push {parent_name} no_push;") +else: + print(f'Unsupported lineage version: {lineage["version"]}', file=sys.stderr) +END_OF_LINE +)" + +# Qapla +echo "Success!" From c50094a8dee2573462c355e82e01eba13fb1fb14 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 15:15:18 -0400 Subject: [PATCH 07/18] Fix typo. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 65c5b6f..5015beb 100755 --- a/setup-env +++ b/setup-env @@ -113,7 +113,7 @@ END_OF_LINE exit 1 fi -# Create a new virutal environment for this project +# Create a new virtual environment for this project if ! pyenv virtualenv "${env_name}"; then cat << END_OF_LINE An existing virtual environment named $env_name was found. Either delete this From 3c2781c8cc3d87d4e1433100e243679e899d7fe1 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 15:25:59 -0400 Subject: [PATCH 08/18] Correct syntax on fence. Closes CVE-2020-3.14159 --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2bd2451..8635fea 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,7 +50,7 @@ If you already have `pyenv` and `pyenv-virtualenv` configured you can take advantage of the `setup-env` tool in this repo to automate the entire environment configuration process. -```bash +```console ./setup-env ``` From 23ef29c8a301a6851664ae0ccd8633363f95648f Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:30:46 -0400 Subject: [PATCH 09/18] Clarify comment with text from `pyenv local --help` --- setup-env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5015beb..5f404b3 100755 --- a/setup-env +++ b/setup-env @@ -125,7 +125,8 @@ END_OF_LINE exit 1 fi -# Activate the new virtual environment +# Set the local application-specific Python version(s) by writing the +# version name to a file named `.python-version'. pyenv local "${env_name}" # Upgrade pip and friends From 5d1d134c88aa203d69d4baf88e9b1e5d4b7a2eab Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:34:53 -0400 Subject: [PATCH 10/18] Change to python3 to standardize how we call python. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5f404b3..a767b3d 100755 --- a/setup-env +++ b/setup-env @@ -130,7 +130,7 @@ fi pyenv local "${env_name}" # Upgrade pip and friends -python -m pip install --upgrade pip setuptools wheel +python3 -m pip install --upgrade pip setuptools wheel # Find a requirements file (if possible) and install for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt"; do From 8a105ca04010ca92cd066a525b840fe1c6de66f5 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:50:42 -0400 Subject: [PATCH 11/18] Respect customs. --- setup-env | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup-env b/setup-env index a767b3d..713687a 100755 --- a/setup-env +++ b/setup-env @@ -134,9 +134,8 @@ python3 -m pip install --upgrade pip setuptools wheel # Find a requirements file (if possible) and install for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt"; do - if [[ -f $req_file ]] - then - pip install -r $req_file + if [[ -f $req_file ]]; then + pip install --requirement $req_file break fi done From ae21af108405bde6ea772d804f71ecf3cd677895 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:55:01 -0400 Subject: [PATCH 12/18] Improve usage message. --- setup-env | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/setup-env b/setup-env index 713687a..7bc99f9 100755 --- a/setup-env +++ b/setup-env @@ -5,17 +5,25 @@ set -o errexit set -o pipefail USAGE=$(cat << 'END_OF_LINE' -This script is used to configure a developement environment for this repo. +Configure a developement environment for this repository. It does the following: - Verifies pyenv and pyenv-virtualenv are installed. - Creates a Python virtual environment. - Configures the activation of the virtual enviroment for the repo directory. - - Installs the requirements required for development. + - Installs the requirements needed for development. - Installs git pre-commit hooks. - Configures git upstream remote "lineage" repositories. -usage: setup-env [--force] [--help] [virt_env_name] +Usage: + setup-env [--force] [virt_env_name] + setup-env (-h | --help) + +Options: + -f --force Delete virtual enviroment if it already exists. + -h --help Show this message. + -i --install-hooks Install hook environments for all environments in the + pre-commit config file. END_OF_LINE ) From c5e7edf46b736965216b258ac562cbf6ad6e1218 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:55:34 -0400 Subject: [PATCH 13/18] Add flag to install pre-commit hooks now. --- setup-env | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/setup-env b/setup-env index 7bc99f9..5b6dacd 100755 --- a/setup-env +++ b/setup-env @@ -37,6 +37,10 @@ PARAMS="" # Parse command line arguments while (( "$#" )); do case "$1" in + -i|--install-hooks) + INSTALL_HOOKS=1 + shift + ;; -f|--force) FORCE=1 shift @@ -148,8 +152,8 @@ for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt fi done -# Install git pre-commit hooks -pre-commit install +# Install git pre-commit hooks now or later. +pre-commit install ${INSTALL_HOOKS:+"--install-hooks"} # Setup git remotes from lineage configuration # This could fail if the remotes are already setup, but that is ok. From 482a370679570e55233c13ef6df644ee3437694e Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:02:03 -0400 Subject: [PATCH 14/18] Switch to a POSIX compliant command... command. This is the way. I learned something today. https://unix.stackexchange.com/questions/85249/why-not-use-which-what-to-use-then --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5b6dacd..9d1ed6a 100755 --- a/setup-env +++ b/setup-env @@ -64,7 +64,7 @@ done eval set -- "$PARAMS" # Check to see if pyenv is installed -if [ -z "$(which pyenv)" ] || [ -z "$(which pyenv-virtualenv)" ]; then +if [ -z "$(command -v pyenv)" ] || [ -z "$(command -v pyenv-virtualenv)" ]; then echo "pyenv and pyenv-virtualenv are required." if [[ "$OSTYPE" == "darwin"* ]]; then cat << 'END_OF_LINE' From 466ef0cce560fc18b4d657caef943f9c07626fc8 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:05:19 -0400 Subject: [PATCH 15/18] Sort command line flag case statement. --- setup-env | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup-env b/setup-env index 9d1ed6a..0b44a24 100755 --- a/setup-env +++ b/setup-env @@ -37,10 +37,6 @@ PARAMS="" # Parse command line arguments while (( "$#" )); do case "$1" in - -i|--install-hooks) - INSTALL_HOOKS=1 - shift - ;; -f|--force) FORCE=1 shift @@ -49,6 +45,10 @@ while (( "$#" )); do echo "${USAGE}" exit 0 ;; + -i|--install-hooks) + INSTALL_HOOKS=1 + shift + ;; -*) # unsupported flags echo "Error: Unsupported flag $1" >&2 exit 1 From 996ae2677a33e1631c8ac80a6e9c682be22e6ac9 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:13:51 -0400 Subject: [PATCH 16/18] Add recommendation to install brew, instead of assuming it is available. --- CONTRIBUTING.md | 5 +++-- setup-env | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8635fea..6434d30 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,8 +59,9 @@ environment. #### Installing and using `pyenv` and `pyenv-virtualenv` #### -On the Mac, installation is as simple as `brew install pyenv -pyenv-virtualenv` and adding this to your profile: +On the Mac, we recommend installing [brew](https://brew.sh/). Then +installation is as simple as `brew install pyenv pyenv-virtualenv` and +adding this to your profile: ```bash eval "$(pyenv init -)" diff --git a/setup-env b/setup-env index 0b44a24..46620e0 100755 --- a/setup-env +++ b/setup-env @@ -69,8 +69,9 @@ if [ -z "$(command -v pyenv)" ] || [ -z "$(command -v pyenv-virtualenv)" ]; then if [[ "$OSTYPE" == "darwin"* ]]; then cat << 'END_OF_LINE' - On the Mac, installation is as simple as "brew install pyenv - pyenv-virtualenv" and adding this to your profile: + On the Mac, we recommend installing brew, https://brew.sh/. Then installation + is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your + profile: eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" From 31980ed4a8624baa682735d0ecfef611582e55a1 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:18:19 -0400 Subject: [PATCH 17/18] Clean up usage syntax since we now have multiple options. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 46620e0..57abe56 100755 --- a/setup-env +++ b/setup-env @@ -16,7 +16,7 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env [--force] [virt_env_name] + setup-env [options] [virt_env_name] setup-env (-h | --help) Options: From db362b912512418280e8b74891de812016e4fb9f Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:37:34 -0400 Subject: [PATCH 18/18] Normalize quotes. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 57abe56..4d822c4 100755 --- a/setup-env +++ b/setup-env @@ -168,7 +168,7 @@ import sys LINEAGE_CONFIG = Path(".github/lineage.yml") if not LINEAGE_CONFIG.exists(): - print('No lineage configuration found.', file=sys.stderr) + print("No lineage configuration found.", file=sys.stderr) sys.exit(0) with LINEAGE_CONFIG.open("r") as f: