From af06915b05674695cb5d9bd8418c6b37f77ec3bc Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 14:57:04 -0400 Subject: [PATCH 01/13] Add tool and docs to automate development environment setup. --- CONTRIBUTING.md | 11 +++ setup-env | 175 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100755 setup-env diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dacaaad..2bd2451 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,6 +46,17 @@ There are a few ways to do this, but we prefer to use create and manage a Python virtual environment specific to this project. +If you already have `pyenv` and `pyenv-virtualenv` configured you can +take advantage of the `setup-env` tool in this repo to automate the +entire environment configuration process. + +```bash +./setup-env +``` + +Otherwise, follow the steps below to manually configure your +environment. + #### Installing and using `pyenv` and `pyenv-virtualenv` #### On the Mac, installation is as simple as `brew install pyenv diff --git a/setup-env b/setup-env new file mode 100755 index 0000000..65c5b6f --- /dev/null +++ b/setup-env @@ -0,0 +1,175 @@ +#!/usr/bin/env bash + +set -o nounset +set -o errexit +set -o pipefail + +USAGE=$(cat << 'END_OF_LINE' +This script is used to configure a developement environment for this repo. + +It does the following: + - Verifies pyenv and pyenv-virtualenv are installed. + - Creates a Python virtual environment. + - Configures the activation of the virtual enviroment for the repo directory. + - Installs the requirements required for development. + - Installs git pre-commit hooks. + - Configures git upstream remote "lineage" repositories. + +usage: setup-env [--force] [--help] [virt_env_name] + +END_OF_LINE +) + +# Flag to force deletion and creation of virtual environment +FORCE=0 + +# Positional parameters +PARAMS="" + +# Parse command line arguments +while (( "$#" )); do + case "$1" in + -f|--force) + FORCE=1 + shift + ;; + -h|--help) + echo "${USAGE}" + exit 0 + ;; + -*) # unsupported flags + echo "Error: Unsupported flag $1" >&2 + exit 1 + ;; + *) # preserve positional arguments + PARAMS="$PARAMS $1" + shift + ;; + esac +done + +# set positional arguments in their proper place +eval set -- "$PARAMS" + +# Check to see if pyenv is installed +if [ -z "$(which pyenv)" ] || [ -z "$(which pyenv-virtualenv)" ]; then + echo "pyenv and pyenv-virtualenv are required." + if [[ "$OSTYPE" == "darwin"* ]]; then + cat << 'END_OF_LINE' + + On the Mac, installation is as simple as "brew install pyenv + pyenv-virtualenv" and adding this to your profile: + + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" + +END_OF_LINE + + fi + cat << 'END_OF_LINE' + For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you don't want + to use "brew") you can use https://github.com/pyenv/pyenv-installer to install + the necessary tools. Before running this ensure that you have installed the + prerequisites for your platform according to the pyenv wiki page, + https://github.com/pyenv/pyenv/wiki/common-build-problems. + + On WSL you should treat your platform as whatever Linux distribution you've + chosen to install. + + Once you have installed "pyenv" you will need to add the following lines to + your ".bashrc": + + export PATH="$PATH:$HOME/.pyenv/bin" + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" +END_OF_LINE + exit 1 +fi + +set +o nounset +# Determine the virtual environment name +if [ "$1" ]; then + # Use the user-provided environment name + env_name=$1 +else + # Set the environment name to the last part of the working directory. + env_name=${PWD##*/} +fi +set -o nounset + +# Remove any lingering local configuration. +if [ $FORCE -ne 0 ]; then + rm -f .python-version + pyenv virtualenv-delete --force "${env_name}" || true +elif [[ -f .python-version ]]; then + cat << 'END_OF_LINE' + An existing .python-version file was found. Either remove this file yourself + or re-run with --force option to have it deleted along with the associated + virtual environment. + + rm .python-version + +END_OF_LINE + exit 1 +fi + +# Create a new virutal environment for this project +if ! pyenv virtualenv "${env_name}"; then + cat << END_OF_LINE + An existing virtual environment named $env_name was found. Either delete this + environment yourself or re-run with --force option to have it deleted. + + pyenv virtualenv-delete ${env_name} + +END_OF_LINE + exit 1 +fi + +# Activate the new virtual environment +pyenv local "${env_name}" + +# Upgrade pip and friends +python -m pip install --upgrade pip setuptools wheel + +# Find a requirements file (if possible) and install +for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt"; do + if [[ -f $req_file ]] + then + pip install -r $req_file + break + fi +done + +# Install git pre-commit hooks +pre-commit install + +# Setup git remotes from lineage configuration +# This could fail if the remotes are already setup, but that is ok. +set +o errexit + +eval "$(python3 << 'END_OF_LINE' +from pathlib import Path +import yaml +import sys + +LINEAGE_CONFIG = Path(".github/lineage.yml") + +if not LINEAGE_CONFIG.exists(): + print('No lineage configuration found.', file=sys.stderr) + sys.exit(0) + +with LINEAGE_CONFIG.open("r") as f: + lineage = yaml.safe_load(stream=f) + +if lineage["version"] == "1": + for parent_name, v in lineage["lineage"].items(): + remote_url = v["remote-url"] + print(f"git remote add {parent_name} {remote_url};") + print(f"git remote set-url --push {parent_name} no_push;") +else: + print(f'Unsupported lineage version: {lineage["version"]}', file=sys.stderr) +END_OF_LINE +)" + +# Qapla +echo "Success!" From c50094a8dee2573462c355e82e01eba13fb1fb14 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 15:15:18 -0400 Subject: [PATCH 02/13] Fix typo. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 65c5b6f..5015beb 100755 --- a/setup-env +++ b/setup-env @@ -113,7 +113,7 @@ END_OF_LINE exit 1 fi -# Create a new virutal environment for this project +# Create a new virtual environment for this project if ! pyenv virtualenv "${env_name}"; then cat << END_OF_LINE An existing virtual environment named $env_name was found. Either delete this From 3c2781c8cc3d87d4e1433100e243679e899d7fe1 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 15:25:59 -0400 Subject: [PATCH 03/13] Correct syntax on fence. Closes CVE-2020-3.14159 --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2bd2451..8635fea 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,7 +50,7 @@ If you already have `pyenv` and `pyenv-virtualenv` configured you can take advantage of the `setup-env` tool in this repo to automate the entire environment configuration process. -```bash +```console ./setup-env ``` From 23ef29c8a301a6851664ae0ccd8633363f95648f Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:30:46 -0400 Subject: [PATCH 04/13] Clarify comment with text from `pyenv local --help` --- setup-env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5015beb..5f404b3 100755 --- a/setup-env +++ b/setup-env @@ -125,7 +125,8 @@ END_OF_LINE exit 1 fi -# Activate the new virtual environment +# Set the local application-specific Python version(s) by writing the +# version name to a file named `.python-version'. pyenv local "${env_name}" # Upgrade pip and friends From 5d1d134c88aa203d69d4baf88e9b1e5d4b7a2eab Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:34:53 -0400 Subject: [PATCH 05/13] Change to python3 to standardize how we call python. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5f404b3..a767b3d 100755 --- a/setup-env +++ b/setup-env @@ -130,7 +130,7 @@ fi pyenv local "${env_name}" # Upgrade pip and friends -python -m pip install --upgrade pip setuptools wheel +python3 -m pip install --upgrade pip setuptools wheel # Find a requirements file (if possible) and install for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt"; do From 8a105ca04010ca92cd066a525b840fe1c6de66f5 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:50:42 -0400 Subject: [PATCH 06/13] Respect customs. --- setup-env | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup-env b/setup-env index a767b3d..713687a 100755 --- a/setup-env +++ b/setup-env @@ -134,9 +134,8 @@ python3 -m pip install --upgrade pip setuptools wheel # Find a requirements file (if possible) and install for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt"; do - if [[ -f $req_file ]] - then - pip install -r $req_file + if [[ -f $req_file ]]; then + pip install --requirement $req_file break fi done From ae21af108405bde6ea772d804f71ecf3cd677895 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:55:01 -0400 Subject: [PATCH 07/13] Improve usage message. --- setup-env | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/setup-env b/setup-env index 713687a..7bc99f9 100755 --- a/setup-env +++ b/setup-env @@ -5,17 +5,25 @@ set -o errexit set -o pipefail USAGE=$(cat << 'END_OF_LINE' -This script is used to configure a developement environment for this repo. +Configure a developement environment for this repository. It does the following: - Verifies pyenv and pyenv-virtualenv are installed. - Creates a Python virtual environment. - Configures the activation of the virtual enviroment for the repo directory. - - Installs the requirements required for development. + - Installs the requirements needed for development. - Installs git pre-commit hooks. - Configures git upstream remote "lineage" repositories. -usage: setup-env [--force] [--help] [virt_env_name] +Usage: + setup-env [--force] [virt_env_name] + setup-env (-h | --help) + +Options: + -f --force Delete virtual enviroment if it already exists. + -h --help Show this message. + -i --install-hooks Install hook environments for all environments in the + pre-commit config file. END_OF_LINE ) From c5e7edf46b736965216b258ac562cbf6ad6e1218 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:55:34 -0400 Subject: [PATCH 08/13] Add flag to install pre-commit hooks now. --- setup-env | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/setup-env b/setup-env index 7bc99f9..5b6dacd 100755 --- a/setup-env +++ b/setup-env @@ -37,6 +37,10 @@ PARAMS="" # Parse command line arguments while (( "$#" )); do case "$1" in + -i|--install-hooks) + INSTALL_HOOKS=1 + shift + ;; -f|--force) FORCE=1 shift @@ -148,8 +152,8 @@ for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt fi done -# Install git pre-commit hooks -pre-commit install +# Install git pre-commit hooks now or later. +pre-commit install ${INSTALL_HOOKS:+"--install-hooks"} # Setup git remotes from lineage configuration # This could fail if the remotes are already setup, but that is ok. From 482a370679570e55233c13ef6df644ee3437694e Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:02:03 -0400 Subject: [PATCH 09/13] Switch to a POSIX compliant command... command. This is the way. I learned something today. https://unix.stackexchange.com/questions/85249/why-not-use-which-what-to-use-then --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5b6dacd..9d1ed6a 100755 --- a/setup-env +++ b/setup-env @@ -64,7 +64,7 @@ done eval set -- "$PARAMS" # Check to see if pyenv is installed -if [ -z "$(which pyenv)" ] || [ -z "$(which pyenv-virtualenv)" ]; then +if [ -z "$(command -v pyenv)" ] || [ -z "$(command -v pyenv-virtualenv)" ]; then echo "pyenv and pyenv-virtualenv are required." if [[ "$OSTYPE" == "darwin"* ]]; then cat << 'END_OF_LINE' From 466ef0cce560fc18b4d657caef943f9c07626fc8 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:05:19 -0400 Subject: [PATCH 10/13] Sort command line flag case statement. --- setup-env | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup-env b/setup-env index 9d1ed6a..0b44a24 100755 --- a/setup-env +++ b/setup-env @@ -37,10 +37,6 @@ PARAMS="" # Parse command line arguments while (( "$#" )); do case "$1" in - -i|--install-hooks) - INSTALL_HOOKS=1 - shift - ;; -f|--force) FORCE=1 shift @@ -49,6 +45,10 @@ while (( "$#" )); do echo "${USAGE}" exit 0 ;; + -i|--install-hooks) + INSTALL_HOOKS=1 + shift + ;; -*) # unsupported flags echo "Error: Unsupported flag $1" >&2 exit 1 From 996ae2677a33e1631c8ac80a6e9c682be22e6ac9 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:13:51 -0400 Subject: [PATCH 11/13] Add recommendation to install brew, instead of assuming it is available. --- CONTRIBUTING.md | 5 +++-- setup-env | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8635fea..6434d30 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,8 +59,9 @@ environment. #### Installing and using `pyenv` and `pyenv-virtualenv` #### -On the Mac, installation is as simple as `brew install pyenv -pyenv-virtualenv` and adding this to your profile: +On the Mac, we recommend installing [brew](https://brew.sh/). Then +installation is as simple as `brew install pyenv pyenv-virtualenv` and +adding this to your profile: ```bash eval "$(pyenv init -)" diff --git a/setup-env b/setup-env index 0b44a24..46620e0 100755 --- a/setup-env +++ b/setup-env @@ -69,8 +69,9 @@ if [ -z "$(command -v pyenv)" ] || [ -z "$(command -v pyenv-virtualenv)" ]; then if [[ "$OSTYPE" == "darwin"* ]]; then cat << 'END_OF_LINE' - On the Mac, installation is as simple as "brew install pyenv - pyenv-virtualenv" and adding this to your profile: + On the Mac, we recommend installing brew, https://brew.sh/. Then installation + is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your + profile: eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" From 31980ed4a8624baa682735d0ecfef611582e55a1 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:18:19 -0400 Subject: [PATCH 12/13] Clean up usage syntax since we now have multiple options. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 46620e0..57abe56 100755 --- a/setup-env +++ b/setup-env @@ -16,7 +16,7 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env [--force] [virt_env_name] + setup-env [options] [virt_env_name] setup-env (-h | --help) Options: From db362b912512418280e8b74891de812016e4fb9f Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:37:34 -0400 Subject: [PATCH 13/13] Normalize quotes. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 57abe56..4d822c4 100755 --- a/setup-env +++ b/setup-env @@ -168,7 +168,7 @@ import sys LINEAGE_CONFIG = Path(".github/lineage.yml") if not LINEAGE_CONFIG.exists(): - print('No lineage configuration found.', file=sys.stderr) + print("No lineage configuration found.", file=sys.stderr) sys.exit(0) with LINEAGE_CONFIG.open("r") as f: