diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index c224d79..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,73 +0,0 @@ -version: 2 -jobs: - lint: - docker: - - image: circleci/python:3.7 - steps: - - checkout - - run: - name: black - command: | - pip install black - black --check --line-length 80 kf_update_dbgap_consent tests - - run: - name: flake8 - command: | - pip install flake8 - flake8 --ignore=E501,W503,E203 kf_update_dbgap_consent tests - - - test: - docker: - - image: circleci/python:3.7 - - image: postgres:11.1 - environment: - POSTGRES_USER: "postgres" - POSTGRES_DB: "test" - PG_USER: "postgres" - PG_NAME: "test" - - steps: - - checkout - - - run: - name: install dockerize - command: wget https://github.com/d3b-center/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz - environment: - DOCKERIZE_VERSION: v0.6.1 - - - run: - name: start local dataservice - command: | - git clone --depth 1 https://github.com/kids-first/kf-api-dataservice.git - cd kf-api-dataservice - python3 -m venv dataservice_venv - source dataservice_venv/bin/activate - pip install -r requirements.txt - pip install -e . - flask db upgrade - flask db migrate - flask run - background: true - environment: - FLASK_APP: "manage" - PG_USER: "postgres" - PG_NAME: "test" - - - run: - name: run tests - command: | - python3 -m venv client_venv - source client_venv/bin/activate - pip install -r dev-requirements.txt - pip install . - dockerize -wait http://localhost:5000 -timeout 5m - pytest tests - - -workflows: - version: 2 - build: - jobs: - - lint - - test diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..235210d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,61 @@ + +name: ✅ CI + +on: + pull_request: + types: [opened, reopened, edited, synchronize, closed] + +concurrency: + group: ci-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-20.04 + timeout-minutes: 10 + steps: + - name: 👩‍💻 Checkout code + uses: actions/checkout@v3 + + - name: 🐍 Setup Python + uses: actions/setup-python@v3 + with: + python-version: '3.8' + + - name: 📦 Install deps + run: | + pip install black + + - name: 🚨 Lint code + run: | + black --check --line-length 80 kf_update_dbgap_consent tests + + test: + runs-on: ubuntu-20.04 + timeout-minutes: 10 + steps: + - name: 👩‍💻 Checkout code + uses: actions/checkout@v3 + + - name: 🐳 Start Dataservice docker-compose + run: | + ./bin/setup_dataservice.sh + + - name: 🐍 Setup Python + uses: actions/setup-python@v3 + with: + python-version: '3.8' + + - name: 📦 Install deps + run: | + pip install --upgrade virtualenv + virtualenv -p python3 venv + source venv/bin/activate + pip install -r dev-requirements.txt + pip install -e . + + - name: ✅ Test + run: | + source venv/bin/activate + pytest tests + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ee93e72 --- /dev/null +++ b/.gitignore @@ -0,0 +1,165 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +### Ingest stuff +cached_schema.json +*.tsv +*.csv +*.xls +*.xlsx +*.txt + +# Ingest outputs +kf_ingest_packages/packages/**/output/* + +# Exclude uid cache db +!kf_ingest_packages/packages/**/output/*.db + +# Exclude count analysis report +!kf_ingest_packages/packages/**/counts*.log + +# Exclude ingest logs +!kf_ingest_packages/packages/**/ingest*.log + +# dot files and folders +.* + +# Exclude github workflows +!/.github + +# Exclude validation test results +!kf_ingest_packages/packages/**/validation_results/* + +# Validation Report Site +## Build dir +web/site + +# CI Artifacts +kf-api-dataservice diff --git a/bin/health-check.sh b/bin/health-check.sh new file mode 100755 index 0000000..dbb25b1 --- /dev/null +++ b/bin/health-check.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Dataservice Health Check + +set -e + +START_TIME=$SECONDS +DATASERVICE_ENDPOINT=${DATASERVICE_ENDPOINT:-http://localhost:5000} + +echo "⛑️ Waiting for service to become healthy ..." +until $(curl --output /dev/null --head --silent --fail $DATASERVICE_ENDPOINT/status) +do + echo -n "." + sleep 2 +done + +ELAPSED=$((( SECONDS - START_TIME ) / 60 )) +FORMATTED_ELAPSED=$(printf "%.2f" $ELAPSED) +echo "" +echo "Elapsed time $FORMATTED_ELAPSED minutes" + +echo "✅ --- Development environment setup complete! ---" + diff --git a/bin/setup_dataservice.sh b/bin/setup_dataservice.sh new file mode 100755 index 0000000..1786a5b --- /dev/null +++ b/bin/setup_dataservice.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Dataservice Setup + +set -e + +if [ -d "./kf-api-dataservice" ]; +then + cd kf-api-dataservice + git pull + cd .. +else + git clone --depth 1 https://github.com/kids-first/kf-api-dataservice.git +fi +cp kf-api-dataservice/.env.sample kf-api-dataservice/.env +cp docker-compose.yml kf-api-dataservice/ +docker-compose -f kf-api-dataservice/docker-compose.yml up -d --build +./bin/health-check.sh diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..32413b8 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,36 @@ +version: '3' + +services: + dataservice_pg: + container_name: dataservice_pg + image: postgres:11.1 + env_file: + - .env + environment: + POSTGRES_DB: "${DATASERVICE_DB}" + POSTGRES_USER: "${DATASERVICE_DB_USER}" + POSTGRES_PASSWORD: "${DATASERVICE_DB_PASSWORD}" + ports: + - "5432:5432" + dataservice: + container_name: dataservice + build: + context: . + target: test + command: /bin/ash -c "sleep 5; ./bin/run.sh" + volumes: + - .:/app + ports: + - "5000:80" + env_file: + - .env + environment: + FLASK_CONFIG: "${FLASK_CONFIG}" + FLASK_APP: "${FLASK_APP}" + PG_HOST: "${DATASERVICE_PG_HOST}" + PG_PORT: "${DATASERVICE_PG_PORT}" + PG_NAME: "${DATASERVICE_DB}" + PG_USER: "${DATASERVICE_DB_USER}" + PG_PASS: "${DATASERVICE_DB_PASSWORD}" + depends_on: + - dataservice_pg